diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4f507c7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,92 @@ +# http://help.github.com/ignore-files/ +# Compiled source # +################### +*.com +*.class +*.dll +*.exe +*.o +*.so +*.a + +# Packages # +############ +# it's better to unpack these files and commit the raw source +# git has its own built in compression methods +*.7z +*.dmg +*.gz +*.iso +*.jar +*.rar +*.tar +*.zip + +# Logs and databases # +###################### +*.log +*.sql +*.sqlite + +# OS generated files # +###################### +.DS_Store* +ehthumbs.db +Icon? +Thumbs.db + +# Backup Files and autoconf nonsense +# http://stackoverflow.com/questions/3290908/which-files-generated-by-autotools-should-i-keep-in-git-repository +autom4te.cache +*.aff +*.obj +*.sig +*~ +Makefile +Makefile.in +Makefile.in +TAGS +a.out.dSYM +aclocal.m4 +affconfig.h +affconfig.h.in +afflib-*.tar.gz +afflib.lib +afflib.pc +afflib.spec +autom4te.cache +config.guess +config.log +config.status +config.sub +configure +install-sh +libtool +ltmain.sh +m4 +missing +stamp-h* + +# Executables +.deps +.libs +TAGS +tools/affdiskprint +tools/affcat +tools/affcompare +tools/affconvert +tools/affcopy +tools/affcrypto +tools/affinfo +tools/affix +tools/affrecover +tools/affsegment +tools/affsign +tools/affstats +tools/affuse +tools/affverify +tools/affxml + +lib/version.h +*.lo +*.la diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..9f42eb3 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,30 @@ +The primary author of AFF is: + +Simson L. Garfinkel + +Special thanks also to those who have contributed bug reports and fixes: + +Jean-Francois Beckers +Sebastien Coutaz +Eric C. +Timothy Lyons +Nicolas (kwizart) +Joachim Metz +David Collett (PyFlag), for the PyAFF integration +Phillip Hellewell (AccessData) for port to VC2008 and catching a few bugs. + +The AFF Project received substantial financial support in 2005 and +2006 from Basis Technology Corporation. For that reason some of the +some of the files (as indicated) have the 4-part Berkeley Copyright +License. + +We would also like to thank the following organizations for their +interest and support: + +- Naval Postgraduate School +- National Institutes of Standards and Technology +- I.D.E.A.L. Technology Corporation +- US Department of Defense +- AccessData + + diff --git a/BUGLIST.txt b/BUGLIST.txt new file mode 100644 index 0000000..9f0cec3 --- /dev/null +++ b/BUGLIST.txt @@ -0,0 +1,75 @@ +AFFLIB Buglist and roadmap. + + +PERFORMANCE IMPROVEMENTS: +* afconvert should calculate MD5, SHA256 and SHA1 +* Store TOC sorted and find entries with binary search. + (A 1TB TOC has 62,500 pages or 100K segments; with fixed size records that would be 8,000,000, which is not much.) +* aff_write_seg() shouldn't be calling af_rewind_seg after every call +* af_write() shouldn't be calling lseek() and write() so many times. + - move to scatter/gather writing. + - look at the TOC to know how much space is available. + - Make the TOC always used. Trust the TOC. + +BUGS: +* Test all programs with file://-style URLs. You can't test for existence of a file with + access(). This is an issue for -z in afcopy. +* image_size not set for AFM files; does it need to be set, or is it generated on the fly? +* afcrypto should note if a file can't be decrypted. +* library doesn't automatically write SHA1s and SHA256s; need to be selectable. +* combine affix and afrecover into a single executable. +* combine afconvert and afcopy into a single executable. +* better error handling in aimage? +* Graacefully handle expat not installed? +* improve performance of + - cache memory selection size - * Automatically increase cache to use 1/2 of physical RAM? + - cache algorithm + +FEATURES: + +library: +* Add support for http://mingw.org/ +* badflag needs to be reimplemented as a bitfield. +* Since we have the TOC, is there EVER a case in which we need to scan the whole file? + If not, perhaps we can take out those scan routines. +* Keep track of sectors that have been touched and output. +* Some kind of sector map array. (Bit array? Byte array?) +* A 1TB disk has 2 billion sectors and 62,500 AFF pages; do we need to store this in a better + data structure than an unsorted linear list? + +aimage: + * Have aimage update more often + * Final report on bytes written is wrong. + * Fix imaging over the network + * Keep a log of everything imaged, # of bad blocks, MD5, etc. + * Image from an SSL tcp connection. + * notify user of kernel error messages; write them to segment + * Report if drive is wiped + * -A mode needs some work; it sometimes compresses too much. + * Digitally sign the AFFs + +afcat: + * option - BADFLAG or BLANK errors + * Option to output a specific sector or a range of sectors + * Multiple ranges of bytes + +afcompare: + * Make SIGINFO work + * -d - just compare data + * -s3 - just check to see if the named segments got written. + - Just compare MD5s! + +afcopy: + * don't copy the file if it won't fit. + * preserve timestamps + * have the file names .aff.new, and rename to .aff after complete. + * if there are _md5/_sha1/_sha256 segments, read all of those segments first and verify them. + * if AFFLIB_PASSPHRASE is set and a file is copied, the resultant file is broken. + +afconvert: + * This didn't work: afconvert -q -o /project3/1028.raw /project2/p3/1028.aff + +afupload: + * Uploading shouldn't decrypt first. + + diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..7d06192 --- /dev/null +++ b/COPYING @@ -0,0 +1,142 @@ +AFFLIB is covered under two copyright regimes. + +Items that were created in 2005 and 2006 are covered under the +four-part Berkeley copyright license. The advertising clause is +intentional included and can only be removed with the permission of +Basis Technology Corp. In the past Basis has been willing to waive the +advertising requirement when requested: + +/* + * AFFLIB(tm) + * + * Copyright (c) 2005-2006 Simson L. Garfinkel and Basis Technology Corp. + * All rights reserved. + * + * This code is derived from software contributed by + * Simson L. Garfinkel + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by Simson L. Garfinkel + * and Basis Technology Corp. + * 4. Neither the name of Simson L. Garfinkel, Basis Technology, or other + * contributors to this program may be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY SIMSON L. GARFINKEL, BASIS TECHNOLOGY, + * AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL SIMSON L. GARFINKEL, BASIS TECHNOLOGY, + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * AFF and AFFLIB is a trademark of Simson Garfinkel and Basis Technology Corp. + */ + +AFFLIB 2.0a14, dated December 8, 2006, was the last version of AFFLIB +to be exclusively licensed under the four-part Berkeley license. + +On December 24, 2006, Simson Garfinkel became a full-time employee of +the US Government. Works of US Government employees developed as part +of their employment are not subject to copyright. Therefore all +files contributed to AFFLIB after December 24, 2006 developed by +Simson Garfinkel, any copyright notice notwithstanding, are not +covered under US copyright law and are in the public domain. + +The files in AFFLIB 2.0a14 that are covered under the four-part +Berkeley license are: + + 714 Dec 8 2006 . + 29821 Dec 1 2006 ./aimage/aimage.cpp + 2319 Nov 18 2006 ./aimage/aimage.h + 14399 Oct 20 2006 ./aimage/aimage_os.cpp + 13961 Nov 21 2006 ./aimage/gui.cpp + 231 Oct 20 2006 ./aimage/gui.h + 8087 Nov 15 2006 ./aimage/ident.cpp + 1719 Oct 20 2006 ./aimage/ident.h + 23327 Nov 25 2006 ./aimage/imager.cpp + 3729 Nov 25 2006 ./aimage/imager.h + 301 Nov 8 2006 ./aimage/Makefile.am + 13466 Dec 8 2006 ./aimage/Makefile.in + 461 Dec 8 2006 ./AUTHORS + 26608 Nov 28 2006 ./ChangeLog + 7765 Dec 8 2006 ./config.h.in + 427990 Dec 8 2006 ./configure + 5351 Dec 8 2006 ./configure.ac + 1909 Oct 19 2006 ./COPYING + 15205 Oct 20 2006 ./depcomp + 152 Oct 22 2006 ./INSTALL + 9212 Oct 20 2006 ./install-sh + 1292 Dec 8 2006 ./lib + 4831 Nov 15 2006 ./lib/aff_db.cpp + 105 Oct 19 2006 ./lib/aff_db.h + 2517 Nov 15 2006 ./lib/aff_toc.cpp + 12907 Nov 25 2006 ./lib/afflib.cpp + 17363 Nov 25 2006 ./lib/afflib.h + 11721 Nov 25 2006 ./lib/afflib_i.h + 3932 Dec 1 2006 ./lib/afflib_os.cpp + 18713 Nov 25 2006 ./lib/afflib_pages.cpp + 9462 Nov 25 2006 ./lib/afflib_stream.cpp + 5725 Nov 15 2006 ./lib/afflib_util.cpp + 1925 Nov 15 2006 ./lib/aftimer.cpp + 836 Oct 31 2006 ./lib/aftimer.h + 10819 Oct 20 2006 ./lib/base64.cpp + 692 Oct 19 2006 ./lib/base64.h + 4652 Nov 15 2006 ./lib/lzma_glue.cpp + 1978 Nov 1 2006 ./lib/Makefile.am + 73313 Dec 8 2006 ./lib/Makefile.in + 12649 Dec 8 2006 ./lib/s3.cpp + 19190 Dec 7 2006 ./lib/s3_glue.cpp + 3841 Dec 1 2006 ./lib/s3_glue.h + 14470 Nov 27 2006 ./lib/vnode_afd.cpp + 98 Oct 19 2006 ./lib/vnode_afd.h + 17078 Nov 21 2006 ./lib/vnode_aff.cpp + 13654 Dec 6 2006 ./lib/vnode_afm.cpp + 76 Oct 19 2006 ./lib/vnode_afm.h + 7305 Nov 15 2006 ./lib/vnode_evd.cpp + 54 Oct 19 2006 ./lib/vnode_evd.h + 47317 Nov 15 2006 ./lib/vnode_evf.cpp + 8156 Oct 19 2006 ./lib/vnode_evf.h + 7301 Nov 15 2006 ./lib/vnode_raw.cpp + 156 Oct 19 2006 ./lib/vnode_raw.h + 8371 Nov 25 2006 ./lib/vnode_s3.cpp + 52 Oct 19 2006 ./lib/vnode_s3.h + 17982 Dec 6 2006 ./lib/vnode_split_raw.cpp + 84 Oct 19 2006 ./lib/vnode_split_raw.h + 690 Nov 27 2006 ./Makefile.am + 18908 Dec 8 2006 ./Makefile.in + 10678 Oct 20 2006 ./missing + 333 Nov 18 2006 ./NEWS + 9977 Oct 19 2006 ./README + 578 Dec 8 2006 ./tools + 9067 Nov 28 2006 ./tools/afcat.cpp + 22564 Nov 28 2006 ./tools/afcompare.cpp + 19102 Nov 28 2006 ./tools/afconvert.cpp + 14755 Nov 28 2006 ./tools/afcopy.cpp + 4216 Nov 28 2006 ./tools/affix.cpp + 16266 Nov 28 2006 ./tools/afinfo.cpp + 7498 Nov 28 2006 ./tools/afsegment.cpp + 5044 Nov 28 2006 ./tools/afstats.cpp + 6573 Nov 28 2006 ./tools/afxml.cpp + 502 Nov 14 2006 ./tools/Makefile.am + 17176 Dec 8 2006 ./tools/Makefile.in + 382 Oct 20 2006 ./tools/quads.cpp + 116 Oct 20 2006 ./tools/quads.h + 541 Nov 21 2006 ./tools/utils.cpp + 1004 Nov 14 2006 ./tools/utils.h diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 0000000..db6b46d --- /dev/null +++ b/ChangeLog @@ -0,0 +1,2587 @@ +2012-03-31 Simson Garfinkel + + * configure.ac: increased version number to 3.7.0 + + * ALL FILES: Audited for copyright. + + * COPYING: Clarified that Basis copyright only applies to + + * .gitignore: created file in preparation for move to github + +2012-03-04 Simson Garfinkel + + * configure.ac: increased version number to 3.6.16 + + * Changed license from BSD with advertising to BSD without advertising. + +2011-12-02 Simson Garfinkel + + * configure.ac: increased version number to 3.6.15 + +2011-11-26 Simson Garfinkel + + * configure.ac: incorporated Greg Freemyer's patch to address some + ncurses problems. + +2011-11-24 Simson Garfinkel + + * lib/utils.h: created virtual destructore + +2011-11-21 Simson Garfinkel + + * configure.ac: added AC_PROG_CXX + +2011-11-18 Simson Garfinkel + + * tools/affxml.cpp (main): removed unused bflag variable + + * lib/vnode_afd.cpp: max() is now always defined, even on windows. + +2011-11-01 Simson Garfinkel + + * lib/aftimer.h (aftimer::eta_time): added std:: before string + + * configure.ac: updated version counter to 3.6.14 + +2011-11-01 Simson Garfinkel + + * lib/utils.h: added virtual destructor for seginfo + +2011-09-28 Simson Garfinkel + + * configure.ac: incremented version counter to 3.6.13 + + * lib/afflib_pages.cpp (af_get_pagesize): having af_page_size() as + a macro was causing problems, so now there is both af_page_size + and af_get_pagesize, both of which do the same. Please use af_get_pagesize. + +2011-09-27 Simson Garfinkel + + * aftimer.h (timestamp): updated aftimer for higher resolution on win32 + +2011-09-18 Simson Garfinkel + + * lib/afflib.cpp (af_get_pagesize): created af_get_pagesize, which + for some reason was missing. + + * lib/afflib.h: added int af_get_pagesize(AFFILE *af); + + +2011-05-04 Simson Garfinkel + + * configure.ac: increased version number to 3.6.11, overcomes recent problems. + +2011-05-02 Simson Garfinkel + + * configure.ac: added back libcrypto (needed on MacOS) + + * tools/affinfo.cpp (color): explicity tests for tigetstr(), putp() and tparm() + +2011-04-27 Simson Garfinkel + + * configure.ac: removed dependency on libcrypto (as we have + libssl-dev dependency) + +2011-04-25 Simson Garfinkel + + * lib/Makefile.am (EXTRA_PROGRAMS): java re-added to distribution; + bulk_extractor.jar is now listed as an EXTRA program so it doesn't + always get built. + +2011-04-14 Simson Garfinkel + + * configure.ac: increased version to 3.6.10 + + * lib/vnode_split_raw.cpp (split_raw_increment_fname): revised + splitraw so that it goes 000 to 999 then A00 through ZZ9 (that is, + the first and second character are base 36, the third remains base + 10) + +2011-04-06 Simson Garfinkel + + * configure.ac: incremented to version 3.6.9 + + * lib/aftest.cpp (bugs_test): added tests + + * lib/vnode_split_raw.cpp (split_raw_identify_file): now + returns 1 for filename.000 or filename.001 or filename.AAA or filename.aaa. + + * (split_raw_close): Now only closes files that are open. + + * (increment_fname) changed to vnode_split_raw_increment_fname so it + can be externally tested. + +2011-03-20 Simson Garfinkel + + * configure.ac: increased version number to 3.6.8 + +2011-03-18 Simson Garfinkel + + * configure.ac: version number incremented to 3.6.7 + + * lib/afflib_i.h: removed typedefs for uint and int64 and uint32_t + + +2011-03-17 Simson Garfinkel + + * ALL OF AFFLIB: All "unsigned long" changed to uint32_t *. + +2011-02-16 Simson Garfinkel + + * lib/afflib_i.h: removed all u_int, etc. for Cstandard uint8_t, + uint32_t, etc. + +2011-01-29 Simson Garfinkel + + * tools/affsegment.cpp ("C"): added extern "C" around regex.h include + + * lib/utils.h: changed all u_int to "unsigned int". + (aff): changed u_long to unsigned long + + * lib/afflib.h: changed all u_int to "unsigned int" and all + "u_char" to unsigned char to ease compiling in certain environments. + +2011-01-14 Simson Garfinkel + + * lib/afflib.cpp (af_make_badflag): changed int to unsigned int to + avoid compiler warning. + + * configure.ac: removed AC_MSG_NOTICE([LIBEWF: ${enable_libewf} (requires uuid-dev on Linux)]) + +2011-01-12 Simson Garfinkel + + * aftimer.h (aftimer::hms): fixed plural for days. + +2011-01-01 Simson Garfinkel + + * Makefile.am (pkginclude_HEADERS): removed threaded_hash.h from package. + + * lib/vnode_aff.cpp: incorporated kernel modifications from kernel mounter. + +2010-12-22 Simson Garfinkel + + * tools/affverify.cpp (hash_verify): changed aftimer to return string + + * aftimer.h (class aftimer): modified to return string rather than + deal with silly buffers. + +2010-12-16 Simson Garfinkel + + * configure.ac: removed IN_AFFLIB. I wonder what it was used for. + +2010-12-16 Simson Garfinkel + + * configure.ac: version number increased to 3.6.6 + + * lib/Makefile.am (AFFLIB_SOURCES): threaded_hash.h removed from + afflib distribution. + +2010-12-04 Simson Garfinkel + + * lib/afflib_i.h: only define ENOTSUP if it is not already defined + + * configure.ac: version number increased to 3.6.5 + +2010-12-04 Simson Garfinkel + + * configure.ac: disable qemu with Msys + + * configure.ac: ssl3_new in AC_CHECK_LIB changed to SSL_library_init + +2010-11-10 Simson Garfinkel + + * tools/affdiskprint.cpp: changed HAVE_EXPAT to HAVE_LIBEXPAT + +2010-10-25 Simson Garfinkel + + * tools/affxml.cpp (xml_info): was adding a NULL to data instead + of b64size. + +2010-10-14 Simson Garfinkel + + * lib/afflib.cpp (af_open): added a warning that support for E01 + was removed. + +2010-10-08 Simson Garfinkel + + * configure.ac: increased to 3.6.3 + + * lib/vnode_split_raw.cpp (split_raw_get_seg): added ability to + query AF_PAGESIZE, AF_IMAGESIZE, AF_SECTORSIZE and AF_DEVICE_SECTORS + + * lib/vnode_raw.cpp (raw_get_seg): fixed AF_DEVICE_SECTORS to make + sure that there are 8 bytes available for the data (previously was 0) + + * lib/afflib_pages.cpp (af_get_page): wasn't setting *bytes if a + page was requested and data was null. Now it does. This caused + mis-reporting in the size of split_raw files. + + * lib/vnode_split_raw.cpp (split_raw_open_internal): replaced + fprintf with call to stderr. + +2010-08-09 Simson Garfinkel + + * make_mingw.sh: renamed bootstrap_mingw.sh to make_mingw.sh for consistency. + +2010-08-01 Simson Garfinkel + + * configure.ac: increased version to 3.6.2 + + * lib/vnode_afd.cpp (afd_identify_file): added file:// logic to + afd_identify_file from aff_identify_file + + * lib/vnode_split_raw.cpp (split_raw_write_internal2): when + storing data in AFM-files the mode set by af_open is only used for + the AFM-file, but not for the .000 which contains the raw data. In + case of mode 0600 for af_open, the mode for the .000 raw-data + file is set to 0644, which enables read-access for everyone. This + is a possible security risk on multi-user systems. To correct + this, mode in open() is now taken from af->openmode, and not + hard-coded to be 0644. + + +2010-07-29 Simson Garfinkel + + * win32/afflib.mak (LINK_OPTS): updated for AFFLIB3.6 naming scheme + + * lib/afflib.h: added #ifdef for stdint.h + + * BUGLIST.txt: removed note about afd encryption not working. + +2010-07-20 Simson Garfinkel + +2010-07-15 Simson Garfinkel + + * lib/vnode_afd.cpp (vnode_afd): removed AF_VNODE_NO_SEALING from vnode_afd->flags. + (afd_add_file): ap->num_afs>0 changed to ap->num_afs>1 for copying + over the metadata for the second file. + +2010-07-10 Simson L. Garfinkel + + * lib/afflib.h (struct af_vnode_info): changed imagesize in + af_vnode_info structure from int64_t to uint64_t. + +2010-07-10 Simson L. Garfinkel + + * tools/Makefile.am: all commands now begin aff rather than af. + + * lib/vnode_ewf.cpp (ewf_vstat): removed EWF support from AFF + +2010-07-06 Simson L. Garfinkel + + * lib/vnode_aff.cpp (aff_get_seg): fixed assertion in ret. + +2010-06-28 Simson Garfinkel + + * configure.ac: increased version number to 3.5.13. + + * tools/Makefile.am (bin_PROGRAMS): renamed afconvert to + affconvert because of name collission with MacOS + +2010-06-27 Simson Garfinkel + + * lib/Makefile.am (EXTRA_PROGRAMS): I read on a debian complaint + list that "aftest" and "s3" shouldn't be installed, but nobody + thought to drop me a note explaining this or telling me how to do + it. So after about 15 minutes with the automake manual I moved + aftest and s3 from bin_PROGRAMS to EXTRA_PROGRAMS and now they + shouldn't get installed. It would be really nice if those Debian + folks to send their comments back to package authors. + +2010-04-25 Simson L. Garfinkel + + * configure.ac: version increased to 3.5.12 + + * Makefile.am (EXTRA_DIST): misc expat distribution removed. + +2010-04-25 Simson Garfinkel + + * configure.ac: version increased to 3.5.11 + +2010-04-18 Simson Garfinkel + + * substantial modifications to compile under mingw + + * tools/afcrypto.cpp (check_file): replaced index() with strchr() + for WIN32 + + * lzma443/C/Common/Alloc.cpp: removed MidAlloc(), MidFree(), + SetLargePageSize(), BigAlloc() and BigFree() when compiling under mingw + +2010-04-17 Simson Garfinkel + + * configure.ac: patched for conditional inclusion of the win32 + directory. + +2010-04-14 Simson Garfinkel + + * debian/ directory removed at the request of the debian project. + +2010-04-12 Ryan Mayer + + * lib/aftest.cpp (xmltest): now compiles without threaded_hash.h + (simson apparently forgot to add threaded_hash.h to the subversion repository....) + + * tools/afinfo.cpp (print_info): now checks to see if fwrite() doesn't + write all the characters. + + * tools/afcat.cpp (afcat): now checks to see if fwrite() doesn't + write all the characters. + + * tools/afcrypto.cpp (usage): changed -d (debug) to -D. Added -e + option (decrypt.) + +2010-04-10 Simson Garfinkel + + * applied 03-bashism.patch, fixes a so-called bashism in a test script. + * applied 04-fix-afdiskprint.patch, fixes a grave bug which prevents + afdiskprint from starting at all. + + +2010-04-09 Simson Garfinkel + + * tools/affuse.c (main): added explicit instructions to install fuse-devl + +2010-03-19 Simson L. Garfinkel + + * configure.ac: updated version to 3.5.10 + + * tools/aff_bom.h: removed non-PD terms in copyright statement. + + * tools/aff_bom.cpp: removed non-PD terms in copyright statement. + + * tools/aff_bom.cpp: corrected spelling of postgraduate + +2010-03-14 Simson Garfinkel + + * configure.ac: increased version number to 3.5.9 + + * lib/vnode_raw.cpp: now sets af->imagesectorsize when + AF_SECTORSIZE is requested. + + * (raw_filesize): moved added support for af_figure_media to + raw_filesize(). + + * (raw_open): modified to call raw_filesize() to figure size of + the media + + * bulk_extractor should now work with raw devices. + +2010-02-23 Simson L. Garfinkel + + * configure.ac: increased version to 3.5.8 + + * tools/test_signing.sh (echo): removed ./ from command names. + + * tools/afconvert.cpp (main): removed TERM dependency from + afconvert. + +2010-01-17 Simson L. Garfinkel + + * lib/afflib_pages.cpp: removed 'shouldfree' comment, because the + variable is gone. + +2010-01-16 Simson L. Garfinkel + + * configure.ac: updated version to 3.5.6 + +2010-01-15 simsong + + * lib/vnode_ewf.cpp (ewf_get_seg): ewf_get_seg had segnum as an uint64_t; should have been a uint_64_t. + +2010-01-03 Simson L. Garfinkel + + * configure.ac: now properly handles linux systems that have + libewf installed but not uuid-dev + +2009-12-24 Simson L. Garfinkel + + * configure.ac: incremented version to 3.5.5 + + * tools/afverify.cpp (process): modified to only complain of an + unsigned file if the unsigned segments are data segments. + + * tools/afverify.cpp (verify_bom_signature): fixed bug in + verification of XML signatures. I have no idea why this was + here. Added better error checking. + +2009-12-24 Simson L. Garfinkel + + * tools/test_*.sh: modified to use mkstemp to create proper + temporary files. + + * lib/afflib_pages.cpp (af_read_sizes): corrected to properly + calculate the size of the last page + +2009-12-23 simsong + + * lib/afflib.cpp (af_get_imagesize): added memset(&vni,0,...) to af_get_imagesize() + +2009-12-20 Simson L. Garfinkel + + * lib/afflib.cpp (af_invalidate_vni_cache): created new function, + since invalidation was happening in more than one place + +2009-12-19 simsong + + * tools/afcrypto.cpp (main): added -E option which just prints the number of segments that would be encrypted. + +2009-12-16 Simson Garfinkel + + * lib/afflib.cpp (af_open_with): modified so that a valid AFFILE + is returned even if the passphrase is invalid + +2009-12-14 Simson Garfinkel + + * lib/vnode_aff.cpp (aff_vstat): wasn't setting cannot_decrypt:1; fixed. + + * tools/afconvert.cpp (convert): fixed so that it handles file:// notation. + +2009-12-13 Simson L. Garfinkel + + * configure.ac: upgraded version count to 3.5.4 + + * tools/afverify.cpp (main): added OpenSSL_add_all_digests(). + (usage): added debug to print if SHA256 isn't working. + +2009-12-13 Simson Garfinkel + + * tools/test_recovery.sh (PATH): now stores tempfiles in /tmp + + * tools/test_afsegment.sh (echo): now stores tempfiles in /tmp/ + + * tools/test_passphrase.sh: now stores tempfiles in /tmp, rather + than in the current directory. + + * tools/afconvert.cpp (convert): fixed utimes() so that it is + applied to the af_output_filename. + + * tools/afcrypto.cpp (main): added option -j to print total number + of encrypted segments and -J to print total number of unencrypted segments. + +2009-12-05 Simson Garfinkel + + * lib/vnode_aff.cpp (aff_close): forgot to close aff_toc_free, + causing memory leak. Fixed. + +2009-12-05 Simson Garfinkel + + * lib/vnode_ewf.cpp (ewf_get_seg): same bug as below + + * lib/vnode_qemu.cpp (qemu_get_seg): bug in qemu_get_seg should + have set *datalen to 8 when called with *datalen=0. Fixed. + + * tools/afcompare.cpp (compare_aff_aff): fixed handling when + images have different pagesize but same sectorsize. + + * lib/vnode_raw.cpp (raw_get_seg): added check for seek beyond end + of file + +2009-11-26 Simson Garfinkel + + * MASSIVE UPDATE: fixed many compiler warnings from signness and + bad printf formats. + + * lib/afflib.h: af_read now returns ssize_t + +2009-11-26 Simson Garfinkel + + * tools/afcrypto.cpp: now reports the number of encrypted pages + + * lib/vnode_aff.cpp (aff_vstat): now calculates the number of + encrypted pages. + + * lib/afflib.cpp (af_seek): reworked so that there would be no + sign problems + + * configure.ac (AFFUSE_BIN): turned on -Wall. enabled + _FORTIFY_SOURCE=2 in the Makefiles. What happened to that? + +2009-11-25 Simson Garfinkel + + * configure.ac: updated to version 3.5.3 + +2009-11-25 Simson Garfinkel + + * lib/crypto.cpp (af_get_affkey_using_keyfile): fixed swapped + memset arguments. + + * lib/vnode_s3.cpp (s3_get_next_seg): fixed swapped memset arguments. + + * lib/s3_glue.cpp (encode): fixed swapped memset arguments. + +2009-11-01 Simson L. Garfinkel + + * configure.ac: also checks ${prefix} for installed libraries + +2009-10-22 Simson Garfinkel + + * configure.ac: updated to version 3.5.2 + +2009-10-20 Simson Garfinkel + + * win32/Makefile.am (the_release): added libeay32.dll to zipfile + +2009-10-09 Simson Garfinkel + + * Removed AFF_SIGS since it is now determined on the fly. + + + * configure.ac: remove SHA256 detection from configure script; this is now done at runtime. + + * lib/afflib_pages.cpp (af_update_page): modified write_sha256 block to use the EVP system. + + * lib/afflib.cpp (af_initialize): removed SHA256 validation vectors from af_initialize() because AFFLIB now uses the EVP system for SHA256. + +2009-10-04 Simson Garfinkel + + * tools/afinfo.cpp (print_info): fixed bug in which auto_decrypt + was being set even if it was not previously set. + + * lib/afflib.cpp (af_set_option): now returns the previous version + of the option that is set. + +2009-10-03 Simson Garfinkel + + * tools/afdiskprint.cpp: added O_BINARY define for Windows. + +2009-09-30 Simson Garfinkel + + * configure.ac: updated version number to 3.4.2 + + * tools/afdiskprint.cpp: added #include because STL + isn't very S on Linux. + +2009-09-22 Simson Garfinkel + + * lib/vnode_aff.cpp (aff_open): failure to obtain exclusive lock + now generates warning rather than error. + + * tools/afdiskprint.cpp (main): added #ifdef HAVE_SRANDOMDEV for + compilation on linux + +2009-09-17 Simson Garfinkel + + * configure.ac: increased version number to 3.4.0 + + * tools/afdiskprint.cpp (diskprint_verify): added diskprint. + +2009-08-22 Simson Garfinkel + + * configure.ac: increased version number to 3.3.8 + +2009-08-20 Simson Garfinkel + + * lib/*.cpp and tools/*.cpp (info_file): warnings removed for GCC 4.2.1 + * Updated many format strings to handle GCC 4.2.1 + +2009-08-08 Simson Garfinkel + + * tools/affix.cpp (fix): changed truncate() to ftruncate(), as we + have a valid win32 cover for ftruncate. + + * tools/afconvert.cpp (convert): made call to utimes() conditional + because we don't always have it. + + * lib/Makefile.am (AFFLIB_SOURCES): removed aftimer.cpp, since the + class is now entirely defined in the aftimer.h + +2009-07-22 Simson Garfinkel + + * configure.ac: now specifically tells the user to install an up-to-date version of OpenSSL if SHA256 is missing. + + Version number upped to 3.3.7 + +2009-05-19 Simson Garfinkel + + * configure.ac: upped version number to 3.3.6 + + * Fixed bug in handling of fixed-size AFD files. + +2009-05-03 Simson Garfinkel + + * changed version number to 3.3.5 + + * Modified configure.ac so lack of SHA256 is no longer fatal + +2009-03-17 Simson L. Garfinkel + + * tools/afxml.cpp (xml_info): changed name= to image_filename= in affinfo attribute. + +2009-03-05 Simson Garfinkel + +2009-02-21 Simson Garfinkel + + * configure.ac: added AC_CHECK_LIB([rt],[aio_error64]) to configure.ac + +2009-02-11 Simson Garfinkel + + * configure.ac: support added for VMDK and DMG files. + Version number will be incremented to to 3.4 when they are stable. + +2009-02-10 Simson Garfinkel + + * lib/afflib.cpp (af_open_with): Now, if it can't allocate + AFFLIB_CACHE_PAGES_DEFAULT, try to allocate just 2 pages. + + * lib/afflib.h (AFFLIB_CACHE_PAGES_DEFAULT): Default page cache + size increased from 2 pages to 32, increasing memory requirements + from 32MB to 512MB. This will make AFFLIB run dramatically faster + in most situations. It will, however, require lowering when AFF is + running in reduced memory configurations. + +2009-01-27 Simson Garfinkel + + * lib/afflib.cpp: created function to set acquisition date in a + standardized way. + + * tools/afconvert.cpp (usage): removed -Z option from usage + because it was never implemented. + +2008-12-30 Simson Garfinkel + + * lib/Makefile.am (AFFLIB_SOURCES): moved display_as_quad to + af_display_as_quad() and added to library. + +2008-10-28 Simson L. Garfinkel + + * lib/afflib_stream.cpp (af_read): if reading through a short page, return (0) for attempts to read additional bytes. (Previously generated a crash...) What's the correct behavior? SHould we advance to the beginning of the next page? I don't know. + +2008-10-12 Simson Garfinkel + + * man/Makefile.am: created man page for afcat + +2008-09-29 Simson L. Garfinkel + + * configure.ac: incremented version counter to 3.3.4 + + * pyaff/pyaff.c: changed #include from afflib/afflib.h to lib/afflib.h + +2008-09-23 Simson Garfinkel + + * lib/afflib_i.h (struct af_figure_media_buf): incremented + max_read_blocks from 4 bytes to 8 bytes because that's what Apple returns!!! + +2008-09-10 Simson Garfinkel + + * configure.ac: upped version counter to 3.3.3 + + * Added include to all CPP files that had include + +2008-09-04 Simson Garfinkel + + * configure.ac (AFFUSE_BIN): removed printing of modified cflags + + * lib/s3.cpp (s3_df): removed af_commas, as it was already in afflib_utils.cpp + (s3_ls): changed af_commas to PRId64 + +2008-09-03 Simson L. Garfinkel + + * configure.ac: fixed support for QEMU under Linux (added -lrt --- + what a weird place to put it), and also fixed autoconf problem of + qemu support being required even when it was turned off. + +2008-08-25 Simson L. Garfinkel + + * lib/afflib.h (struct af_vnode_info): + + * lib/afflib.cpp (af_get_imagesize): updated for new af_vnode_info name + + * lib/afflib.h (struct af_vnode_info): encrypted_count and + signed_count changed to segment_count_encrypted and + segment_count_signed; page_count_total and segment_count_total added. + +2008-08-26 Simson Garfinkel + + * configure.ac: removed QEMU support by default; it needs more work. + +2008-08-17 Simson Garfinkel + + * configure.ac: made OpenSSL error on Mac fatal. + + * lib/Makefile.am (libafflib_la_SOURCES): s3_glue is now not included + in AFFLIB SOURCES if MAYBE_S3 is false. + +2008-08-17 Simson Garfinkel + + * lib/afflib_stream.cpp (af_set_maxsize): changed call from + af_get_imagesize() to access of af->image_size. + + * lib/afflib.cpp (af_seek): changed called to af_get_imagesize() + to af->image_size + (af_get_imagesize): removed locking, as the individual elements of + the af structure are not accessed. + +2008-08-17 Simson Garfinkel + + * lib/afflib.cpp (af_update_segf): modified to clear af->vni_cache + + * lib/afflib.cpp (af_vstat): added support for caching results. + + * lib/afflib_stream.cpp (af_write): modified to clear af->vni_cache + + * lib/afflib_stream.cpp (af_read): modified to use af->image_size + rather than a calculated image_size. + +2008-08-13 Simson Garfinkel + + * lib/vnode_ewf.cpp (ewf_get_seg): now takes imagesize from + af->image_size, since it was already set. + (ewf_bytes_per_sector): refactored all libewf_get_bytes_per_sector + into this function. + +2008-08-11 Simson Garfinkel + + * configure.ac: increased version number to 3.2.7 + + * lib/s3.cpp: changed include err.h to include afflib_i.h for + cases where users want s3 but don't have err.h (like Solaris) + +2008-08-07 Simson Garfinkel + + * configure.ac: version increased to 3.2.6 + + * lib/vnode_ewf.cpp (ewf_close): modified to use the second + argument for libewf if LIBEWF_VERSION>=20080501 + +2008-07-22 Simson Garfinkel + + * configure.ac: upped version to 3.2.5, as 3.2.4 wouldn't compile + on machines without EVP_SHA256. + +2008-07-20 Simson Garfinkel + + * lib/afflib.cpp (af_make_gid): returns -1 if an error, 0 if GID + exists, and if one is made. + + * lib/crypto.cpp: (af_sign_all_unsigned_segments): Now returns the + number of unsigned segments that were signed. + + * tools/afsign.cpp (afsign): + + * lib/crypto.cpp (af_is_signature_segment): af_is_signed_segment + changed to af_is_signature_segment because that's what it is doing. + +2008-07-18 Simson Garfinkel + + * lib/vnode_ewf.cpp (ewf_get_seg): fixed case of data==0 and + datalen==0 generating an error. + (ewf_get_seg): fixed requesting of invalid pages + +2008-07-18 Simson Garfinkel + + * tools/afcopy.cpp (main): changed -s option to -k for consistency + with afsign. + +2008-07-16 Simson Garfinkel + + * lib/vnode_aff.cpp (aff_vstat): removed has_encrypted and + has_signed from the vni information; use the encrypted_count and + signed_count. Sorry about breaking backward compatibility with the + binary API; nobody is using this yet. + + * tools/afsign.cpp (afsign): bug in signing of signature pages fixed. + +2008-07-15 Simson L. Garfinkel + + * configure.ac: increased version number to 3.2.4 + +2008-07-14 Simson Garfinkel + + * lib/vnode_aff.cpp (aff_open): added flock to vnode_aff, so that + multiple writers to an AFF file will now block rather + than corrupt the file + +2008-07-10 Simson Garfinkel + + * tools/afinfo.cpp (figure_media): added -A option to print XML + output of media params. + +2008-07-09 Simson L. Garfinkel + + * tools/afxml.cpp: suppreses XML output of encrypted segments + + * tools/afinfo.cpp (info_file): color was too hard to do; we just use bold now for decrypted data + +2008-07-08 Simson L. Garfinkel + + * tools/afxml.cpp (xml_info): fixed bug in afxml where last 2 chars of md5_hex were not reported because not large enough buffer was allocated + + * lib/afflib_util.cpp (af_hexbuf): + +2008-07-07 Simson Garfinkel + + * tools/afsegment.cpp (main): added -x to print segment as hex string + +2008-07-05 Simson Garfinkel + + * lib/Makefile.am (#aftest_LDFLAGS): lib/s3 binary no longer made + if S3 is not enabled. + +2008-07-03 Simson Garfinkel + + * tools/afsegment.cpp (process): better error messages when + segment doesn't exist + +2008-07-03 Simson L. Garfinkel + + * tools/afinfo.cpp (info_file): fixed printing so color is left as + red, not black + (info_file): only reports missing pages can't be computed if -d + flag is present. + +2008-07-02 Simson Garfinkel + + * lib/afflib_i.h (AF_VNODE_NO_SEALING): added flags for + AF_VNODE_NO_SIGNING and AF_VNODE_NO_SEALING so that AFF + implementations that do not support encryption can declare + this. Added appropriate macros for checking these variables. + +2008-06-29 Simson Garfinkel + + * configure.ac: incremented version to 3.2.3 + + * tools/afconvert.cpp (convert): fixed bug in which converting .aff file + to a .afd file caused crash + +2008-06-27 Simson Garfinkel + + * lib/afflib.cpp: added locking to all of the functions; still + need to add it to afflib_streams. + + * lib/afflib_util.cpp: moved eff, af_err, errx, warn and warnx to + this file. + +2008-06-24 Simson Garfinkel + + * tools/affix.cpp (fix): adds an AF_IMAGE_GID if one doesn't exist. + (fix): fixed error in which O_RDWR flag was being passed as a mode + to af_open_with(). This prevented the -y option from operating + properly. + + * tools/afconvert.cpp (convert): adds an AF_IMAGE_GID if one doesn't exist. + + * tools/afcopy.cpp (afcopy): adds an AF_IMAGE_GID if one doesn't exist. + + + * lib/afflib.cpp (af_make_gid): added af_make_gid (removed from aimage) + +2008-06-20 Simson Garfinkel + + * configure.ac: incremented version number to 3.2.2 + +2008-06-18 Simson Garfinkel + + * Makefile.am: added threaded_hash.h to the list of installed .h files + +2008-06-10 Simson Garfinkel + + * lib/aff_toc.cpp: simplified to make toc mandatory. + +2008-06-02 Simson Garfinkel + + * lib/afflib.cpp (af_open_with): fixed error in wiping password + when open mode is read-only and password is provided. (If no + encryption segment is present, do not use a password.) + +2008-05-28 Simson L. Garfinkel + + * tools/afinfo.cpp (info_file): now states that bold is something that was decrypted + +2008-05-30 Simson Garfinkel + + * tools/afinfo.cpp (info_file): decrypted data now shows in red + and bold + + * lib/afflib_i.h (struct _AFFILE): fixed definition of struct + _AFFILE to be compatible with C (was just compatible with C++). + +2008-05-29 Simson L. Garfinkel + + * lib/vnode_ewf.cpp (ewf_get_seg): modifid once again to work with + wide char + +2008-05-21 Simson Garfinkel + + * lib/vnode_afd.cpp (afd_open): now refuses to open a .afd + directory with no aff files. + + * tools/afcrypto.cpp (main): AFCRYPTO will now only encrypt AFD + and AFM files. + +2008-05-20 Simson Garfinkel + + * tools/*.cpp (recover): changed err() to af_err() after every af_open(). + + * lib/vnode_aff.cpp (aff_get_next_seg): detects truncated files + and will not allowed them to be opened. + +2008-05-19 Simson Garfinkel + + * tools/affix.cpp (usage): removed -b option; wasn't using it. + +2008-05-17 Simson Garfinkel + + * lib/afflib.cpp (af_update_segf): changed padding to 01 for 1 pad + byte, 02 02 for two pad bytes, etc., in keeping with PKCS7. + + * lib/afflib.h (AFFLIB_TRACEFILE): changed AFFLIB_TRACE to AFFLIB_TRACEFILE + +2008-05-13 Simson L. Garfinkel + + * tools/afxml.cpp (xml_info): for some reason, XML sometimes had a ^C at the end. Changed malloc to calloc to zero buffer and avoid the problem. + +2008-05-12 Simson Garfinkel + + * lib/aftest.cpp (lzma_test): changed more char *'s to const char *. + + * lib/afflib_i.h (struct af_vnode): changed char *name to const + char *name; wonder why I didn't see this before. + +2008-05-08 Simson Garfinkel + + * configure.ac: fixed script so that setting the environment + variable AFF_NOOPT to any value will prevent optimization + +2008-05-07 Simson Garfinkel + + * lib/crypto.cpp (af_get_aes_key_from_passphrase): fixed error in + kversion checking; additional work on encryption compatibility + between Mac and Linux + +2008-04-28 Simson Garfinkel + + * tools/afinfo.cpp: updated copyright notice. + + * tools/aff_bom.cpp: added public domain disclaimer + + * tools/aff_bom.h: added public domain disclaimer + + * tools/afcrypto.cpp: corrected copyright notice + + * tools/afcompare.cpp: updated copyright notice + + * tools/afconvert.cpp: updated copyright notice. + +2008-04-26 Simson Garfinkel + + * tools/afcopy.cpp (afcopy): removed debugging. + +2008-04-25 Simson Garfinkel + + * lib/afflib.h (AF_XML_AFFBOM): AF_XML_CUSTODY_CHAIN renamed + AF_XML_AFFBOM and changed from "custody_chain" to "affbom" + +2008-04-23 Simson L. Garfinkel + + * lib/crypto.cpp (af_get_aes_key_from_passphrase): added code to + fix compiler problem on Linux/GCC 4.0/64-bit machines in which + unsigned long foo:32 took up 8 bytes (instead of 4). + + * Added to code to handle reading incorrectly-generated 56-byte affkey segments + +2008-04-21 Simson L. Garfinkel + + * lib/vnode_ewf.cpp (ewf_get_seg): changes made to support the + circa 2007 version of libewf + + * tools/afcrypto.cpp (main): changed output to make easier to parse and more useful. + + * lib/vnode_aff.cpp (aff_vstat): added signed_count and encrypted_count. + +2008-04-14 Simson Garfinkel + + * lib/vnode_raw.cpp (raw_read): replaced fseek() with fseeko(). + + * lib/vnode_aff.cpp (af_truncate_blank): replaced fseek() with + fseeko() + + * lib/vnode_aff.cpp (aff_update_seg): replaced fseek() with fseeko(). (Shouldn't + matter here, but it might). + + * lib/vnode_raw.cpp (raw_write): changed fseek() to fseeko(). + +2008-03-27 Simson Garfinkel + + * Added debian system provided by Joachim Metz + + * configure.ac: incremented version counter to 3.1.5 + + * lib/afflib.cpp (af_get_imagesize): now sets errno=1 if it can't + determine imagesize because of encryption. + + * afflib.pc.in (prefix): added file + +2008-03-24 Simson Garfinkel + + * lib/vnode_ewf.cpp (ewf_get_seg): updated to handle with libewf + returning -1 for error and 1 for success. + +2008-03-12 Simson Garfinkel + + * afflib.spec.in (BuildRequires): fixed subdirectory problem in specfile. + +2008-03-09 Simson Garfinkel + + * configure.ac: + +2008-03-09 Simson Garfinkel + + * configure.ac: updated to AFFLIB-3.1.3 + +2008-03-09 System Administrator + + * lib/Makefile.am (install-exec-hook): fixed shell script syntax error + +2008-03-05 Simson Garfinkel + + * lib/vnode_ewf.cpp: updated for libewf 20080305 + +2008-03-02 Simson Garfinkel + + * afflib.spec.in (BuildRequires): added missing include files + + * Makefile.am (pkginclude_HEADERS): removed duplicate afflib_sha256.h + +2008-02-28 Simson L. Garfinkel + + * tools/afrecover.cpp (recover): fixed bug in printf statement. + +2008-02-26 Simson L. Garfinkel + + * Here's how to turn on executable property in the SVN repository: + svn propset svn:executable ON test_tools.sh + More at http://svnbook.red-bean.com/en/1.0/re23.html + +2008-02-26 Simson Garfinkel + + * lib/afflib.cpp (af_update_segf): fixed annoying bug which caused + signing of encrypted data being written to generate segmentation + errors under some conditions. + + * configure.ac: better reporting of dependencies for aff signatures + +2008-02-26 Simson L. Garfinkel + + * tools/afsegment.cpp (usage): usage fixed, removing space after + -s and -p options (apparently getopt on Linux can't handle it.) + + * lib/crypto.cpp (af_sig_verify_seg): #ifdef USE_AFFSGIS changed + to #ifdef USE_AFFSIGS + + * lib/utils.cpp: C++ utilities moved into the aff namespace. + + * tools/afverify.cpp (main): version flag changed from -v to -V + for consistency with other tools. + + * lib/afflib_os.cpp (af_figure_media): fixed another int64 reference + +2008-02-26 Simson Garfinkel + + * tools/afverify.cpp (usage): corrected usage. + +2008-02-25 Simson Garfinkel + + * configure.ac: now properly handles libexpat once again + + * lib/afflib_i.h: removed lots of the #ifdefs for openssl/*.h and replaced if #ifdef HAVE_LIBSSL + + * lib & tools: moved utils.cpp from tools directory to lib directory + + * Makefile.am (pkginclude_HEADERS): added lib/aftimer.h to the list of installed headers + +2008-02-24 Simson Garfinkel + + * lib/afflib.cpp (af_get_imagesize): changed int64 to int64_t and + uint64 to uint64_t for compliance with C99 standard. + + * aimage removed for AFF; it is now going to be its own distribution + +2008-02-12 SImson L. Garfinkel + + * aimage/gui.cpp (my_refresh): changed screen update so it now happens every 128K bytes. + +2008-02-10 SImson L. Garfinkel + + * tools/afcat.cpp (sig_info): added SIGINFO (^t) to afcat to print current page + +2008-02-10 Charlie Root + + * lib/vnode_raw.cpp (raw_open): modified raw so that it will write as well + +2008-02-20 Simson Garfinkel + + * lib/afflib_i.h ("C"): put everything inside an extern "C".; removed legacy uint64 and int64 types. + + * tools/Makefile.am (bin_PROGRAMS): added test_tools to being to get some more automated testing. + + * tools/afsign.cpp (afsign): improved error message when certificate file cannot be loaded + + * tools/afsegment.cpp (usage): fixed usage so that version number is -V (as the option was implemented) + +2008-02-13 Simson Garfinkel + + * configure.ac: revamped to allow you to specify where expat is located + +2008-02-08 Simson L. Garfinkel + + * tools/afcat.cpp (afcat): now warns if it can't cat because it + doesn't have the encryption keyx + + * lib/afflib.h: moved af_vstat() into afflib.h. + * lib/afflib.h: moved actually AFFILE definition into afflib_i.h + +2008-02-06 SImson L. Garfinkel + + * aimage/gui.cpp (my_refresh): Now prints names of input and + output files on display. + + * aimage/aimage.cpp: You can now specify "%d" or "%02d" or any + %...d format in the output filename; %d is incremented until the + file doesn't exist. + +2008-01-29 Simson L. Garfinkel + + * lib/aftest.cpp (main): made default to run all tests + (main): now erases the test files after creating them. + + * Makefile.am (prerelease): added "make distcheck" to prelease target + +2008-01-28 Simson Garfinkel + * Released 3.0.5 - doesn't compile due to noinst_ issue in + Makefile.am + * Released 3.0.6, which is the same as 3.0.5 but doesn't install + libraries. + +2007-12-24 Simson Garfinkel + + * afflib.spec.in (Name): Joachim Metz contributed this file. + +2007-12-18 Simson Garfinkel + + * lib/vnode_afd.h (AFD_DEFAULT_MAXSIZE): Changed from 600 to 608 + (so that it is now a multiple of 16; not strictly needed, but it + makes things a bit cleaner.) + + * lib/afflib.cpp (af_update_segf): fixed crasher on update when + af->crypto->encrypt was not present. + + * tools/affuse.c (main): improved error messages for when affuse + can't run. + +2007-12-14 Simson Garfinkel + + * configure.ac: changed config.h to affconfig.h to minimize + name space collisions. + + * lib/Makefile.am (lib_LTLIBRARIES): libafflib.la is now an + installed library. + +2007-11-27 Simson L. Garfinkel + + * tools/afxml.cpp (okay_to_print): added ', ", and & to the list + of characters that cannot be output in XML + + * lib/afflib.cpp (af_open_with): opening a read-only AFF file that + has no encryption works, even if you specify an encryption password. + + * tools/afinfo.cpp (display_as_hex): displayed image GID in hex + +2007-11-26 Simson L. Garfinkel + + * REMOVED lib/sha256.c --- the implementation had bugs. (That's + what I get for including a non-validated crypto algorithm.) AFF + now requires SHA256 but it should compile without it + + * tools/afsegment.cpp (main): now handles case when optarg==0 + (which happened with afsegment -d seg instead of affsegment -dseg) + (usage): added : to "d" in getopt. + +2007-11-24 Simson Garfinkel + + * lib/crypto.cpp: refactored AFF compilation error message + +2007-11-16 Simson L. Garfinkel + + * lib/vnode_aff.cpp (aff_identify_file): fixed parsing of file:// URLs. + +2007-11-14 Simson L. Garfinkel + + * lib/afflib.h: af_is_badsector now takes const unsigned char + *buf; taking non-const was an error + +2007-11-13 Simson L. Garfinkel + + * lib/afflib_sha256.h: minor tweaks to deal with the problem with + SHA256() and EVP_sha256() are present but not in the #include file. + +2007-11-07 Simson L. Garfinkel + + * tools/afsegment.cpp (main): -q renamed to be -Q (because -q + should be quiet) + (main): only prints filename if there is more than one file + (main): only prints segname if there is more than one seg + +2007-10-31 Simson Garfinkel + + * tools/utils.cpp: added #ifdef AFF_SIGS so that bom class won't + be compiled if not compiling with AFFSIGS + added #include to get #ifdef AFF_SIGS + +2007-10-29 Simson Garfinkel + + * tools/afinfo.cpp (info_file): aes_segs now initialized to + 0. (Strangely, the uninitialized value was causing problems when + running with MSC.) + + * lib/vnode_s3.cpp (s3_cantopen): added "return -1" to s3_cantopen() + + * lib/vnode_aff.cpp (aff_get_next_seg): ENOBUFS error on + incomplete reads changed to E2BIG, since there is no ENOBUFS on windows. + +2007-10-23 Simson Garfinkel + + * tools/afverify.cpp: moved #include "expat.h" to after the #ifdef USE_AFFSIGS + +2007-10-25 Simson Garfinkel + + * lib/afflib_pages.cpp (af_update_page): lots of code that + required signatures made #ifdef USE_AFFSIGS for compiling under + Windows without OpenSSL + + * lib/afflib_util.cpp (af_parse_url): index() changed to strchr() + for Win32. + +2007-10-24 Simson Garfinkel + + * lib/afflib.h: significant changes for compiling under win32. + - added u_int and others for compiling under win32. + - affkey_aes256 structure had an element with the same name; + structure renamed to affkey. + +2007-10-20 Simson Garfinkel + + * lib/vnode_afm.cpp (afm_split_raw_setup): modified so that + ap->aff->image_size only needs to equal ap->sr->image_size if + ap->aff->image_size has already been set + +2007-10-18 Simson Garfinkel + + * lib/vnode_split_raw.cpp (split_raw_get_seg): fixed bug in that + *datalen was not set properly if it was smaller than the size of + the segment, even if data==0. Fixed. + +2007-10-16 Simson Garfinkel + + * lib/vnode_split_raw.cpp (split_raw_update_seg): was returning + the number of bytes written; now returns 0 if write is successful, + -1 if it is not. + +2007-10-14 Simson Garfinkel + + * aimage/aimage.cpp (usage): --sign and -s now used to specify + private and public key used for signing. + + * aimage/aimage.cpp (usage): --setseg used to be -s; changed to -g + so that we can use -s for signing. + +2007-09-25 Simson Garfinkel + + * tools/utils.cpp (get_seglist): fixed uninitialized variables in + get_seglist which was causing indeterminate behavior in get_seglist. + +2007-09-16 Simson Garfinkel + + * BUGLIST.txt (library): removed need to check for af->writing, + because it's gone. + +2007-09-15 Simson Garfinkel + + * lib/afflib.h: removed af->writing, because it wasn't good for + POSIX compatibility. + +2007-09-14 Simson L. Garfinkel + + * tools/afcopy.cpp (main): -c option (verify compression) removed + from usage because it was never implemented. New -c will specify + an X.509 certificate. + +2007-09-12 Simson L. Garfinkel + + * MULTIPLE FILES: changed most "const void *" pointers on update + and write routines to "const u_char *". + +2007-09-02 Simson L. Garfinkel + + + * lib/aff_toc.cpp (aff_toc_del): fixed double-del problem; thanks + to Jason Shiffer for bringing this to our attention. + +2007-08-29 Simson Garfinkel + + * lib/Makefile.am, tools/Makefile.am, configure.ac (affuse_LDADD): + first steps for using libtool and shared libraries + + * lib/afflib.cpp (af_open_with): added support for AFFLIB_PASSPHRASE + +2007-08-27 Simson Garfinkel + + * aimage/aimage.cpp (process_option): made opt_maxsize an int64; + added support for dvd and dvddl from luca at palazzo.name. + +2007-08-20 Simson Garfinkel + + * lib/aftest.cpp (aestest): now sets page size in test files. + + * lib/afflib.cpp (af_update_seg): af_update_seg() didn't check to + make sure that writing was enabled. Now it does. + +2007-08-20 Simson Garfinkel + + * lib/afflib_stream.cpp (af_write): Now sets pagesize to + AFF_DEFAULT_PAGESIZE (1024*1024*16) if a write is attempted + without a pagesize being previously set. + +2007-08-18 Simson Garfinkel + + * configure.ac: increased version to 2.4.0 + + * lib/aff_db.cpp (af_backspace): af_backspace will no longer + backspace from the first segment in the file. + + * lib/vnode_aff.cpp (aff_write_ignore): BUG --- if first segment + was deleted, entire file got wiped. Whoops! fixed this. + + * tools/afcopy.cpp (afcopy): bugfix - removed af_close(out.af); + when zapping; file wasn't open. + +2007-08-17 Simson Garfinkel + + * lib/vnode_s3.cpp (s3_open): revised for URL parsing in afflib.cpp + + * lib/vnode_aff.cpp (aff_create): moved af_make_badflag to afflib.cpp + (aff_get_next_seg): now sets errno to ENOBUFS if buffer isn't + large enough to hold requested data. + +2007-08-16 Simson Garfinkel + + * lib/vnode_aff.cpp (aff_open): changed all occurances of af_toc + to aff_toc since the TOC is an AFF-file specific thing. + + * tools/afinfo.cpp (usage): changed -p option (validate the hash + of each page) to -v + +2007-08-12 Simson Garfinkel + + * tools/afinfo.cpp (print_info): added two spaces to the "data + length" printout. + +2007-08-11 Simson Garfinkel + + * tools/afinfo.cpp (info_file): fixed printing of missing pages + with -a when there was no device sector size. + + * lib/afflib.cpp: added support for openssl/fips_sha.h + + * S3 support is now disabled by default. + +2007-08-11 Simson Garfinkel + + * lib/s3_glue.cpp: removed main() if USE_S3 was not defined. What + was I thinking? + +2007-07-29 Simson Garfinkel + + * tools/afcopy.cpp (main): Removed -p option; now just has -X1 + through -X9 and -L for recompressing; these are the same flags as aimage. + + * lib/afflib.cpp (af_open): attempts to open a file for writing + with an invalid name now sets EINVAL. + + +2007-07-26 Simson Garfinkel + + * tools/afxml.cpp (xml_info): removed use of af_backspace() so + this will now work with S3. + (xml_info): hex is now printed with coding='base16', decimal + numbers now printed coding='base10' + (xml_info): printable data now just printed, rather than encoded + +2007-07-25 Simson Garfinkel + + * lib/afflib_stream.cpp (af_write): added check for valid + image_pagesize in af_write if the write bypass is not used. + + * lib/vnode_aff.cpp (aff_update_seg): removed stale "append" comment. + +2007-07-19 Simson Garfinkel + + * lib/afflib.cpp (af_seek): af_seek w/ SEEK_CUR didn't work. It + was doing SEEK_CUR as SEEK_SET. This is now fixed. SEEK_END was + working. + (af_seek): added automatic detection of random access, which now + changes the af_update_seg algorithm. + + * lib/vnode_aff.cpp (aff_write_ignore): modified so that if an + IGNORE segment is written before another IGNORE, they are + combined. Likewise, if one is written after an IGNORE, they are + combined. + (aff_update_seg): modified so that new objects smaller than 4096 + will not be dropped into deleted segments larger than 4096 bytes + to prevent fragmentation. + +2007-07-18 Simson Garfinkel + + * lib/vnode_aff.cpp (aff_update_seg): if the last segment is NULL, + it is now erased before writing. + + (aff_del_seg): now we use this on Windows, since _chsize_s() is a + good analog for ftruncate() + + * lib/aftest.cpp (reverse_test): reverse test now reads data to + verify it is correct + +2007-07-17 Simson Garfinkel + + * lib/s3.cpp: applied patches from Ted Romer. The 'cat' and 'cp' + commands in s3.cpp had a bug where they neglect to skip over the + command name in argv, with the result that cat always tries to + look for the key 'cat' and 'cp' always tries to copy the file + 'cp'. + + This patch fixes that, and also modifies cat and rmdir to accept + multiple arguments. + +2007-07-16 Simson L. Garfinkel + + * lib/aftest.cpp (sparse_test): added read-at-end and + read-beyond-end of sparse file tests to aftest -6 + + * lib/afflib_stream.cpp (af_read): removed break so that reads on + pages that don't exist return NULs rather than errors. + +2007-07-14 Simson L. Garfinkel + + * lib/vnode_s3.cpp: changed AF_IDENTIFY_AFD to AF_IDENTIFY_S3 + Modified system so that if s3 support is not compiled in, s3:/// + names generate an error. + + * lib/afflib_i.h: changed from int64 to int64_t and from uint64 to uint64_t. + +2007-07-13 Simson L. Garfinkel + + * Removed #define UNIX. Now we have #ifdef HAVE_POPEN (which is + what we really wanted, anyway) + + * Added support for uint64_t; uint64 now is a typedef for it. + * Added support for int64_t; int64 now is a typedef for it. + +2007-07-11 Simson L. Garfinkel + + * tools/afcopy.cpp (afcopy): cleaned up printing a bit + (afcopy): fixed -y option, which apparently didn't work + + * lib/afflib_pages.cpp (af_read_sizes): removed &arg from + af_get_next_seg call. This change makes af_read_sizes() + dramatically faster on s3 because the calls can be satisfied by + reading the directory, rather than the need to read every + individual segment. + + +2007-07-10 Simson L. Garfinkel + + * configure.ac: S3 support is now disabled by default. + + * lib/afflib.cpp (af_version): added af_version() + + * lib/afflib.h: moved #include to its own major + block, as prelude to moving over to C99 standards. + +2007-07-04 Simson Garfinkel + + * configure.ac: added isatty() + + * lib/afflib_i.h: moved af_identify_file_type() and + af_identify_file_name() from afflib.h to afflib_i.h so that people + wouldn't be tempted to call them . + +2007-07-01 Simson Garfinkel + + * tools/afinfo.cpp (info_file): changed %qd to %"I64d" for Win32 compat. + + * tools/afstats.cpp (main): changed final return() to exit() for + Microsoft happieness. + + * tools/affix.cpp (main): added final exit(). + + * tools/afconvert.cpp: changed final return() to exit() for + microsoft happiness + + * tools/afcopy.cpp (afcopy): changed final return to exit() for + Microsoft happieness. + + * win32/openssl/sha.c: replaced sha1 implementation with + pass-through to Microsoft CryptoAPI. + + * win32/openssl/md5.c: replaced md5 implementation with + pass-through to Microsoft CryptoAPI. + +2007-06-29 Simson L. Garfinkel + + * aimage/gui.cpp (gui_startup): corrected bug of + #ifdef HAVE_LIBCURSES; should have been + #ifdef HAVE_LIBNCURSES + + * aimage/imager.cpp, aimage/gui.cpp, aimage/aimage.cpp: added support for SHA256 + + * lib/afflib.h (AF_SHA256): added AF_SHA256 + +2007-06-26 Simson L. Garfinkel + + * Makefile.am (SUBDIRS): added win32 back to the configure.ac and + Makefile, so that the win32 stuff would be included in the distribution + +2007-06-21 Simson L. Garfinkel + + * aimage/gui.cpp (comma_printw): modifications to gui.cpp and + aimage.cpp so that it will compile and run in batch if ncurses and + termcap are not installed. + +2007-06-10 U-AMD64\Simson Garfinkel + + * aimage/imager.cpp (start_recover_scan): added #ifdef for + HAVE_SRANDOMDEV to allow for complication under Cygwin (and other + systems that don't have srandomdev() + +2007-05-30 Simson L. Garfinkel + + * configure.ac: added -D_FILE_OFFSET_BITS=64 to CPPFLAGS when FUSE + is enabled. fixed compilation for AFFUSE + +2007-05-30 Simson Garfinkel + + * lib/vnode_aff.cpp (aff_get_seg): no longer scans the whole file + looking for a segment. If it is not in the ToC, it doesn't + exist. Not that this means that we can't have simultaneous access + to a single AFF file by a writer and a reader anymore. + + * tools/afinfo.cpp (print_info): added -X flag to print segment names without data contents. + + * lib/afflib_pages.cpp (af_get_page): only fills buffer with "bad flag" if a buffer is provided. + +2007-05-29 Simson Garfinkel + + * aimage/imager.cpp (start_imaging): only writes out AF_IMAGE_GID if segment doesn't exist. + Appends to AF_ACQUISITION_DATE segment if it already exists. + Added \n to AF_ACQUISITION_DATE segment contents. + +2007-05-25 Simson L. Garfinkel + + * tools/afinfo.cpp (info_file): fixed rounding error in + calculation of missing pages + +2007-05-23 Simson L. Garfinkel + + * tools/afinfo.cpp (info_file): added Missing Pages to printout + +2007-05-14 Simson L. Garfinkel + + * tools/afsegment.cpp (main): modified output format so that it + prints filename:segname and then the contents on every line + +2007-05-11 Simson L. Garfinkel + + * tools/afsegment.cpp (usage): changed afcat to afsegment in usage() + (main): added -q option which prints value as 64-bit quad + +2007-05-02 Simson L. Garfinkel + + * lib/afflib.h: changed af_ext_is to return "int" instead of bool. + +2007-04-30 Simson L. Garfinkel + + * lib/s3.cpp (s3_bandwidth): removed s3_debug call from s3 + bandwidth test + +2007-04-29 Simson L. Garfinkel + + * aimage/aimage.cpp: minor update to getlock(), although this code + is still not being called. + +2007-04-29 Simson L. Garfinkel + + * lib/vnode_raw.cpp (raw_popen): moved hasmeta() to af_hasmeta() + and put in afflib_util.cpp. + +2007-04-29 Simson Garfinkel + + * tools/afcopy.cpp (afcopy): command string parameter fname no + longer used as format string in warn(). + + * tools/afxml.cpp (xml_info): command string no longer used as + format string in warn(). + + * aimage/aimage.cpp (getlock): command line parameter no longer + used as format string in err() + + * tools/afinfo.cpp (main): command line parameter no longer used + as format string in err(). + + * lib/aftest.cpp (figure): err(1,fn) changed to err(1,"%s",fn) in + multiple places + + * lib/s3.cpp (s3_cp): fixed * Format String Injection in s3 * + reported by V3Security + + * lib/vnode_raw.cpp (raw_popen): added metacharacter testing here + as well + +2007-04-25 Simson L. Garfinkel + + * configure.ac: version 2.2.8 + + * aimage/aimage_os.cpp (ident): fixed minor compile-under-linux problems + +2007-04-17 Simson Garfinkel + + * aimage/imager.cpp (write_data): doesn't seek if we don't know offset + + * tests/verify.py (runtest): modified test to put file in /tmp + + * aimage/ident.cpp (get_params): added metacharacter checking + filename before popen. + + * tools/afconvert.cpp (convert): added meta character filtering + per VSecurity. + +2007-04-16 Simson Garfinkel + + * tools/afxml.cpp (xml_info): added name sanitization to AFF XML + names per VSecurity report + + * aimage/imager.cpp (image_loop): fixed important bug when imaging + from /dev/random (or presumably a network connection) which caused + page 0 to be repeatedly rewritten. + + * lib/vnode_raw.cpp: added |AF_VNODE_TYPE_RELIABLE to vnode type + + * lib/vnode_aff.cpp: added |AF_VNODE_TYPE_RELIABLE to vnode type + + * lib/vnode_afd.cpp: added |AF_VNODE_TYPE_RELIABLE to vnode type + + * lib/vnode_split_raw.cpp: added + |AF_VNODE_TYPE_RELIABLE|AF_VNODE_MAXSIZE_MULTIPLE to vnode type + + * lib/vnode_afm.cpp: added + |AF_VNODE_TYPE_RELIABLE|AF_VNODE_MAXSIZE_MULTIPLE to vnode type + + * configure.ac: cleaned up enable_fuse so that it doesn't generate + errors on Macintosh; removed OPENSSL alert + +2007-04-11 Simson L. Garfinkel + + * lib/afflib.h: prototypes for af_read() and the like changes from + "unsigned count" to "size_t count" + + * lib/s3_glue.cpp (quote_plus): added quoting of '%' + +2007-04-04 Simson L. Garfinkel + + * aimage/ident.cpp (get_params): fixed strlcat() & strlcpy() order + +2007-04-03 Simson L. Garfinkel + + * lib/s3.cpp (strlcat): added strlcpy and strlcat to s3.cpp + +2007-04-02 Simson Garfinkel + + * aimage/ident.cpp (get_params): changed occurances of strcpy() to + strlcpy() and strcat() to strlcat() per VSecurity report. + + * lib/vnode_ewf.cpp (ewf_open): Now checks to make sure that there + are at least 4 characters available in the fname buffer after the + trailing '.', per VSecurity report. + + * lib/s3.cpp (s3_bandwidth): all occurances of sprintf() changed + to snprintf per VSecurity report. + + * tools/afinfo.cpp (main): changed err(1,infile) to + err(1,"%s",infile) to defend against string format vulnerability + per VSecurity report. + + * tools/afcopy.cpp (afcopy): changed err(1,infile) to + err(1,"%s",infile) to defend against string format vulnerability + per VSecurity report. + + * tools/afconvert.cpp (convert): changed err(1,infile) to + err(1,"%s",infile) to defend against string format vulnerability + per VSecurity report. + + * tools/afcompare.cpp (main): replaced all occurances of sprintf() + with snprintf(), all occurances of strcpy() with strlcpy(), and + all occurances of strcat() with strlcat(). + + * lib/vnode_ewf.cpp (ewf_get_next_seg): replaced both strcpy() + calls with strlcpy() per VSecurity report + + * lib/vnode_s3.cpp (s3_open): added maximal size to both memcpy() + calls, per VSecurity report. + + * lib/s3.cpp (s3_ls): replaced strcpy() with strlcpy() per + VSecurity report. + +2007-03-28 Simson L. Garfinkel + + * aimage/aimage.cpp: added option "skip" / -k to the longopts[] + array. It wasn't there before, and it should have been. + + * lib/afflib_pages.cpp (af_update_page): moved acbi.compressed=1 + out of the if statement in the event that the compress2() call fails. + +2007-03-20 + + * lib/s3.cpp (s3_bandwidth): s3 bandwidth test now verifies MD5 returned + +2007-03-07 Simson L. Garfinkel + + * tools/affuse.cpp (main): Olivier Castan provided a fuse + implementation for us. Thanks! We also have changes of his in + configure.ac, Makefile.am. + +2007-03-01 Simson L. Garfinkel + + * configure.ac: Now compiles under stock MacOS without expat installed + +2007-02-05 Simson L. Garfinkel + + * tools/afcopy.cpp (usage): fixed formatting + +2007-02-05 Simson Garfinkel + + * aimage/imager.cpp: patches to aimage to fix imaging over a + network provided by Suessmilch Bernd + +2007-02-01 Simson Garfinkel + + * fixed bugs in the caching system; caching now works. + +2007-01-24 Simson Garfinkel + + * lib/afflib.h: moved #include from afflib_i.h to afflib.h + + * lib/afflib.h: Moved #ifndef PACKAGE_TARNAME from afflib.h to afflib_i.h + + * lib/afflib.cpp (af_eof): uses results from af_vstat() instead of + calling af_imagesize() again (which would be another call to vstat) + +2007-01-23 Simson Garfinkel + + * lib/aff_toc.cpp (af_toc_build): changed malloc(0) to + malloc(sizeof(af_toc_mem)) as on Borland C++, malloc(0) returns a + NULL pointer. + + * lib/afflib.cpp (af_open_with): fixed memory leak in error condition. + + * lib/vnode_aff.cpp (aff_open): added "b" to opens for running + under windows. + + * lib/vnode_raw.cpp (raw_get_seg): added support for phantom + segments AF_PAGESIZE and AF_IMAGESIZE. This fixes the bug with + afconvert that had been previously reported. + +2007-01-11 Simson L. Garfinkel + + * lib/vnode_afd.cpp: moved definitions of F_OK and R_OK to this file. + + * lib/afflib_pages.cpp: added #include to malloc.h for + valloc. Perhaps it should go in afflib_i.h? + + * aimage/aimage_os.cpp (ident): fixed handling if ident is not defined + + * tools/unix4win32.cpp: removed this file because it shouldn't be + needed anymore; warn, warnx, err and errx are now provided by + afflib. + + * lib/afflib_i.h: created proper configure macros for HAVE_ERR, + HAVE_ERRX, HAVE_WARN and HAVE_WARNX to handle the presence of + absence of these on multiple platforms. + +2006-12-27 Simson L. Garfinkel + + * lib/s3_glue.cpp (request): Added CURLOPT_TIMEOUT 60*60 (1 hour) + to S3 transactions. + +2006-12-14 Simson L. Garfinkel + + * lib/afflib_pages.cpp (af_read_sizes): modified af_read_sizes so + that if an image file doesn't have an IMAGESIZE segment, the + imagesize is determined by reading all of the segments + +2006-11-28 Simson L. Garfinkel + +2006-11-27 Simson L. Garfinkel + + * lib/s3_glue.cpp (canonical_string): added support for BAD_STL, + so that on pair, where STL is out-of-date, we can still compile + + * lib/vnode_afd.cpp (afd_add_file): wasn't setting sectorsize on + AFD subfiles. This caused problems with files that had sector + sizes of 1024 bytes... + + * (afd_add_file): files now added to array before new file is set up. + +2006-11-25 Simson L. Garfinkel + + * lib/vnode_afd.cpp (afd_update_seg): corrected multiple + definitions of AFFILE *af2, which might cause compiling problems + under Windows + + * lib/afflib_i.h: Added AFFLIB_TRACE environment variable; right + now it just traces seek and read + + * lib/afflib.h: retyped af_seek to take an int64 instead of a uint64 + + * lib/vnode_s3.cpp (s3_open): All S3 environment variables are now + defined in s3_glue.h + +2006-11-18 Simson L. Garfinkel + + * aimage/aimage.cpp (debug_list): changed -L option (Log file) to + -G so that -L could be used for LZMA compression. + +2006-10-31 Simson L. Garfinkel + + * lib/s3_glue.cpp (request): changed CURLOPT_INFILESIZE_LARGE to + CURLOPT_INFILESIZE. This limits maximum requests to 2GB in length, + but I don't think that this will be a significant problem, and the + LARGE was causing problems on Linux. + +2006-10-27 Simson L. Garfinkel + + * lib/afflib_pages.cpp (af_set_pagesize): no longer complains if + pagesize it being set to what it already is + + * tools/afcopy.cpp: added -z (opt_zap) to overwrite existing files. + +2006-10-26 Simson L. Garfinkel + + * lib/afflib_pages.cpp: fixed bytes==0 bug in af_get_page. Now if + bytes==0, it probes for the page existence and then returns. + + * lib/s3_glue.cpp (request): Added up to 3 retries for S3 "500" + internal errors. + +2006-10-22 Simson L. Garfinkel + + * lib/afflib.cpp (af_identify_type): Added flag to allow user to + specify if file must exist or not. + + * lib/vnode_s3.cpp (s3_get_seg): Now uses "HEAD" to probe length + of segment without downloading segment. Note that the + Content-Length: header needs to be decoded. + +2006-10-21 Simson L. Garfinkel + + * aimage/aimage.cpp (process_config_questions): Now has conditional detection + of readline; if it isn't present, defaults to fgets() + + +2006-10-20 Simson L. Garfinkel + + * Moved AFFLIB to gnu build environment (autoconf, automake, + configure, etc.). Apparently I can now edit this file by doing a + C-x 4 a. How cool! More information at: + http://gnu.j1b.org/software/emacs/manual/html_node/Change-Log.html + +Release 1.8.0: +- Added support for storing files on Amazon S3. + Set Environment variable AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY. + Set variable AWS_DEFAULT_BUCKET. + Filename syntax is s3://bucket/filename.aff + or: + s3:///foo/bar/baz/filename.aff + + Bucket is getenv("AWS_DEFAULT_BUCKET") unless otherwise specified. + Buckets are automatically created if they do not exist (and if nobody else has created + that bucket yet.) + + Remember, there is a limit of 100 buckets per customer. + +- Moved af_read_sizes(af) from each open to af_open(). This means that I need + to re-validate all of the various implementations. + +- API simplification: The "append" flag has been dropped from af_update_seg() and + af_update_segq(). None of my code ever had append==0, and to make S3 handle it + sensibly would have required a second round-trip to the server. + +- Removed errors from LZMA routines + +Release 1.7.1: +- Fixed isdigit() call, which isn't available under Linux + + +Release 1.7.0: + +- Moved AFFINFO mailing list to Google Groups. Actualy, to two groups + on Google Groups: aff-users and aff-announce. Everybody on AFFINFO + has been subscribed to aff-announce. + +- Added a new compression algorithm, AF_PAGE_COMP_ALG_ZERO. This algorithm + detects a page of all 0s and simply notes that fact, rather than passing + the sectors through zlib. + +- Added a new comparession algorithm, AF_PAGE_COMP_ALG_LZMA, which implements + the LZMA compression algorithm. Validated LZMA against a terrabyte of compressed + page images and found no errors in decompression. LZMA averages are, + on average, 1/3 to 1/10th the size of the ZLIB-compressed images. + +library: +- AFFLIB now maintains a two-page cache for every open AFFILE; this + can be expanded at runtime to any number by setting + AFFLIB_CACHE_PAGES environment variable to the number of pages that + you want to keep in memory. Since pages are typically 16MB, setting + this variable to 16 will dedicate 256MB to each open AFF file + (assuming that you are reading the file through the page system; the + memory only gets dedicated when the bytes actually get read.) + +- You can also set the AFFLIB_CACHE_STATS environment variable to 1, + which will put on STDERR the cache information every time you close + a file. + +- Renamed "af_is_badblock" to "af_is_badsector". + +- Removed logfile from AFFILE structure. It was just being used for + debugging, and not in a particularly effective way. (af_seek and + af_write no longer reference it. ) + +- Fixed linking under MacOS for strlcpy problem. + +- Integrated LZMA library + +- Got AFFLIB (but not the tools) to compile under MS VC6++. Because the Microsoft + compiler is pretty picky about some things, a whole bunch of signed/unsigned errors + were found and fixed. (not that they would have ever mattered, but...) + LZMA is not included in the Windows release currently + +afcopy: +- Created new tool, afcopy, which copies an AFF file segment-by-segment, + and verifies each segment after it is copied by reading the resulting file. + afcopy thus validates all writes and has the side-effect of compressing files, + so the resulting file may be smaller than the original. + +- Added "-p" option to afcopy to preen (recompress with LZMA). + +afcompare: +- Added -p option to report on preening +- Added -r option to compare directories recurisvely +- Added -j option to just do the data + + +Release 1.6.33: +- Fixed to compile under Kubuntu 6.0.6 + +Release 1.6.32: +- pages in AFFILES now have MD5s stored automatically as well. So when +"page3" is written, "page3_md5" is also written. +- afinfo + - when it sees any segment with "md5" in its name that is 16-bytes long, the segment is printed + in hex. Likewise when a segment has "sha1" in its name and is 20-bytes long. + - New "-p" option checks the hash for every page. +- aimage: MacOS 10.4 demsg now requires root to run; aimage no longer sends dmesg error to stderr. + Instead, errors go to /dev/null, and if there is no dmesg output, no dmesg segment is created. + +Release 1.6.31: +Make release now clearly requires gmake and errors if Berkeley Make is used + +Release 1.6.30: +Removed dependency on #include from building the library. + +Release 1.6.29: +Fixed "reverse" option (-V) on aimage so that it actually works. + +Release 1.6.28: +Typedef for "u64" defined under linux so that afflib will compile under Slackware + +Release 1.6.27: +- Compiles under Cygwin +- Aimage now has a -Y (--batch) option which prints + line-by-line output, so that the program can be easily + incorporated into other programs. + +Release 1.6.24: +AFFLIB no longer assumes that is installed on cygwin. + +Release 1.6.24: +Library and some of the tools now compiles under CYGWIN. +Programs that rely on termcap don't compile. This includes +aimage and afconvert. + + +Release 1.6.22: +afxml now displays the same values without base64 encoding as afinfo + +Release 1.6.20: +Library should now compile on NetBSD and OpenBSD. Let me know if it doesn't + +Release 1.6.19: +Library (but nothing else) now compiles on solaris with GCC + +Release 1.6.18: +Now compiles with either Berkeley make or gmake + +Release 1.6.17: +af_read: +- returns 0 if attempt made to read a file with no bytes. +- returns -1 if attempt made to read a file with no pagesize defined. + +aimage: +- Erases screen junk of amount of freespace on disk goes from 1000MB to 999MB + +Release 1.6.16: +aimage: +- Fixed a bug in report printing without going back into \r\l mode. +- fixed error handling when disk fills up. + +Release 1.6.15: +- Removed aff file in tools directory. Sorry about that. + +Release 1.6.14: +afxml: + - metadata segments that just had an arg and no data are now reported as + ###, rather than +aimage: + - minor cosmetic improvements + + +Release 1.6.13: +lib: + - error message on split_raw_setup clarified + - af_open now returns 0 if it can't open the file + +aimage: + - now has flags to suppress inclusion of dmesg and ethernet address in AFF file. + - scaled_atoi now checks its parameters better. + - cleaned up display on 25x80 display + - cleaned up display when imaging multiple drives + - --setseg now works + - now handles improperly-mastered CDROMs + + +Release 1.6.12: +lib: + - fixed badflag handling (badflags were only 4 bytes due to + variable sector size changes.) + +Release 1.6.11: +vnode_afm: + af_callback now gets propigated from afm layer to split_raw layer + on af_write. +lib: + AFFLIB now has a "parent" pointer which is set for AFM and AFD. + callback now uses the parent instead of self if parent is set +aimage: + -v now prints the version number and exits. + -V now scans in reverse. + + +Release 1.6.9: +aimage: +- Whoops. Broke update logic in 1.6.8. Fixed + +Release 1.6.8: +aimage: +- remove foo.000 and foo.afm from release + +Release 1.6.7: +lib: +- Fixed release system so that release X can't be made if release X is already on the server +- Fixed bug in vnode_raw implementation resulting from move to 64-bit math + +aimage: +- modified aimage so that it only updates the screen (and queries the timer) + every million bytes read +- by default, no log file is written + +afinfo: +- prints hex digits as DDDD EEEE FFFF now +- fixed behavior of sha1 printing --- it breaks by two lines by default + + +Release 1.6.6: +lib: +- moved check on change of imagesize on close from aff_close to af_close. + Now check is done for AFM files in addition to AFF files. This fixes the + bug of the AF_IMAGESIZE segment not being written by aimage with AFM files. +- renamed af->image_size_orig to af->image_size_in_file to make it more clear what it does +- created new segment type "aff_file_type" which holds AFF for AFF files, AFM for AFM files, + and AFD for AFD files. +- created aff_create() and afm_create() functions which are automatically + called by the corresponding _open() functions when a file is created. +- removed af_set_maxsize64; af_set_maxsize now requires a 64-bit argument +- created new function, af_set_sectorsize() +- sectorsize no longer hard-coded to be 512. +- removed af_demand_sizes; sizes are now assumed read when the file is opened +- no longer calls err() in af_open +- Virus hiding claim removed from FAQ + +TOOLS: +- all tools take "-v" to print their version number. Tools that were using + -v for "verbose' have been changed + +afcat: +- checks for -1 return from af_read + +afconvert: +- all math involving pages changed to 64-bits +- now honors opt_quiet + +afinfo: +- -d command now returns pagesize in addition to page number +- -w command now implemented properly + +aimage: +- error message with -E omitted no longer misleading +- Checks for ifconfig in path before running ifconfig + +afcompare: +- -v no longer is verbose +- -a prints things that are the same +- -b prints sector numbers that were different +- -c prints contents of sectors that are different. + + +Release 1.6.5: +afconvert: +- 'm' and 'k' changed back to mod 1024, not mod 1000 +- imagesize segment now written when converting from raw to afm + +aimage: +- SIGINFO #ifdefed so that it only is included if SIGINFO is defined. + + +Release 1.6.4: +- afconvert and afinfo fixed to handle 0 read bytes from af_read() + +Release 1.6.3: +- af_set_sizes() call removed. +- fixed bug in af_cat +- several bugs in AFM implementation fixed. + + +Release 1.6.2 +- fixed bug in af_imagesize which was affecting afcat +- fixed bug in parsing of -M option in afconvert +- fixed af_eof() for AFF implementation. + +Release 1.6.1: +vnode_afd: +- Support for AFD was inadvertantly broken in 1.6.0; it's fixed in 1.6.1 + +Release 1.6.0: +vnode_split_raw.cpp: +- integrated code from Joel Weber with these significant changes: +- Removed dependency on AFF/AFD/AFM implementation +- changed implementation to use lseek() to write empty files, rather than repeated writes of buffers... +- modified split_raw_get_seg so that it called af_segname_page_number() rather than parsing by hand. +- created afm_update_seg() so that we could remove call to afm_identify_file in split_raw_update_seg +- fixed a bug whereby 256K of raw files were created if maxsize was smaller than ZERO_BUFFER_SIZE, + which was hard-coded to 256K +- removed raw_file_size from split_raw_private, as it was redundent with maxfilesize in the AFFILE structure. + +vnode_afm.cpp: +- integrated code form Joel Weber with these significant changes: +- Opens two sub-files, one for the AFM metadata (through a coerced aff_open()), and one + for the split_raw implementation. +- Note: there is still a bug that AFM files can't be opened for extending; + +lib: +- added af_has_pages() to allow higher-level programs to know if the underlying + vnode implementation supports pages or not. +- af_vnode_info: added has_pages flag so that an implementation can tell the higher + layers if it supports pages or not. +- vnode_raw: made it so that af_popen() now tells the higher layers that pages are not + supported. +- af_vnode_info: added use_eof and at_eof flags, to support EOF reporting on af_popen. +- awaiting af_read() & af_write() bypass, pending code from nemo + +afcat: +- Outputs NULs for bad blocks unless -b is given. + +afconvert: +- Added version to usage. +- Fixed conversion of gzip'ed and bzip'ed raw images +- Calls af_has_pages to see if the input vnode implementation supports pages or not. + If it doesn't, then it just reads from the beginning of the image to the end. +- Implemented -O (outdir) flag + +afinfo: +- Now honors -w flag +- Output squished to fit MD5s and SHA1s. + +aimage: +- Properly prints free space on FreeBSD, MacOS and Linux +- Prints MD5 & SHA1 as groups of 4 upper-case letters. +- Now stores sectorsize in "sectorsize" segment +- Now stores number of sectors that the device has in "devicesectors" segment +- --pagesize= option now allows suffix of b, k, m or g +- ACQUISTION_TIME now correct (it was being saved as a 64-bit number but reported as a + 32-bit number in afinfo.) + + + +Release 1.5.13: +Encase support works for multiple encase files. + .E01, .E02 ... .E99 + +Release 1.5.12: +aimage: +- now allows --maxsize=cd and --maxsize=bigcd +- --no_hash (-H) avoids hash calculation + +Release 1.5.10: +makefiles: +- Finally implemented a single makefile along the lines + of the "Recursive Make considered Harmful" paper, but improved, + with targets that work in both the subdirectories and in the root directory. + Currently it only works with gmake, which is only a problem under FreeBSD. + +Release engineering: +- added a "tests" directory and moved end-to-end testing there. + afconvert, afcat, and aimage are now automatically tested as part of the release + process. + +aimage: +- no longer dumps core when not given an output file. +- Doesn't waste left-most column of screen +- Displays elapsed time when running. +- Now displays elapsed time while imaging +- Now displays aimage version +- Saves total time to image a disk in the AFF file. +- Properly indicates free space left on capture disk. + +Release 1.5.9: +- Fixed bug in release system which was giving incorrect filenames +- fixed bug in AFD which wasn't setting image size properly in the sub AFF files +- fixed bug in AFD support where maxsize wasn't being set on default. + Now it defaults to 600M + +Release 1.5.8: +- Fixed progname bug (progname appears to only exist under + freeBSD) + +Release 1.5.7: +- fixed a few bugs regarding AFD performance + +afcompare: +- -v flag (verbose) implemented. When run without -v, doesn't report similarities. +- corrected a bug when a data segment was in one file but not another. + + +Release 1.5.6: +- More options added to afcat for handling segments. +aimage: +- aimage now puts now information into aff files. + + + +Release 1.5.5: +lib: +- Eliminated bug when an afd file was opened read-only of the library + giving error messages that it was unable to copy over metadata from + the first AFF file to the other AFF files. +- AFFLIB is now tolerant of a trailing / on .afd names. +- Stores AFFLIB version in the AFF file when new files are created. + +afcat: +- now prints AFFLIB version number. +- now prints a warning on STDOUT if a page is skipped; suppress with -q +- no longer clobbers output file without warning +- no longer give errors about not being able to "update" + information when it cats out an AFD file. + + +afcompare: +- now prints AFFLIB version number + +afinfo: +- Now prints AFFLIB version number. +- Option -v no longer validates hash codes +- Option -m validates MD5 codes +- Option -s validates SHA1 codes +- No longer prints the total compression efficiency + +afstats: +- Now prints AFFLIB version number + +afxml: +- Now prints AFFLIB version number +- Puts AFFLIB version number as a comment in XML output + +release: +- fixed bug of executables in release tar file + +aimage: +- ETA acronym removed; now says "Done in:" + + + + +Release 1.5.4: +afconvert: +- significant changes in option parsing to make it more reliable. +- Now properly handles creating files with non-standard AFF segment sizes +- change to "-z" option so that it zaps the output file, not the input file + (to make consistent with other af commands.) + +release engineering: +- auotmatic validation of afconvert and afcat as part of the release cycle + +Release 1.5.3: +aimage +- Minor patch so it should write afd files... + + +Release 1.5.1: +- fixed suffixing for -M command in aimage +aimage: +- Now reads serial numbers of USB memory sticks using sysctl -a. +- NOTE: Be sure that ther eis only one USB memory stick in the system when you do this, + because we can't figure out how to match the stick up with the S/N other than the + manufacturer's name. + +Release 1.5 +lib: +- Initial support of EVF file format. (Only works on Intel right now, not PPC) + +afconvert: +- rewrote to use vnode support + +aimage: +- Initial support for split files + +Release 1.4.1: +aimage: +- fixed bug when encountering errors and imaging from a device. + (The error is that additional bad blocks were being written into the file with + out proper BAD SECTOR headers. No real data was corrupted, but in some cases + random data appeared in the image file which should not have been written.) +- Improved error recovery. Previously, 5 attempts were made to read the sectors that failed, after + which the program gave up. Now it skips forward and tries to read the next set of sectors. It repeats + this whole process until there is a bad region that is equal 2.5 times the size of the + default read (256K). + +lib: +- fixed bug in which small pages were written out with too much data +- fixed bug in af_get_page in which invalid data was being left uninitialized rather than + being initialized with the bad_block flag + + +-Release 1.4 +Terminology change: data segments are renamed "pages" to eliminate confusion with AFF segments. + +lib: +- Implemented a "vnode" abstraction to make it easy to work with multiple file systems. +- Found and fixed at least 10 major bugs. None of them would have caused data corruption, + but some would have caused the library to go into an infinite loop or to crash. +- Support for "AFD" files --- these are directories of AFF files. This will allow AFF + to work with file systems that do not support files larger than 2^32 characters + (such as FAT32). + + +Known bugs: +AImage: bad block flags are not being properly written out in AFF files. + +afinfo: +- prints more information about AFF files in an easy-to-read format + +afcompare: +- Now compares two AFF files, or an AFF and a RAW file. + +-Release 1.3.4 +aimage:' +- Dramatic restructing of the program. +- You can now batch multiple aimage commands on a single command line. + The program surveys total amount of data to copy and a grand-total ETA. +- New -A option will write out a segment with compression, a segment without + compression, and then will go with the fastest approach. +- Better graphics +- Prints a certificate when the imaging is finished + +lib: +- Removed call to fpurge(), which was incorrect. +- renamed aff_ to aff_toc, becuase it really is building a + table of contents. + +-Release 1.3.3 +lib: +- Improved error handling in writing segments. +- More information is now provided to segment writing callback function. + +aimage: +- Removed the --info command, since it was just for debugging. +- Made --make-config a flag that creates the config file if it doesn't exist, + but it uses the name provided by --config. +- Revamped config file. +- removed some global variables that weren't being used; folded others into the + imager class. +- fixed bug in aimage when reading from devices whose length cannot be determined + (e.g. /dev/zero, (tapes, TCP connections) + +aconvert: +- No longer core-dumps if the file that it is asked to convert doesn't exist. + +January 9, 2006 - Release 1.3.3 +aimage: +- added a -A flag which makes aimage compress or not compress, whichever is faster. + + +January 9, 2006 - Release 1.3.2 +Overall: + +aimage: +- Some curses testing code was left in. It's been removed. + +January 8, 2006 - Release 1.3.1. +Overall: +- Moved source for aimage to aimage/ directory +- Move source for tools to tools/ directory + +aimage: +- On Linux, now reads the serial number of USB drives that are being imaged + using the /proc file system. + +tools: + + + +December 31, 2005 - Release 1.3.0 +overall: +- Moved library source code into lib/ subdirectory. + One big makefile still makes it all, see + "Recursive make considered harmful." + http://www.canb.auug.org.au/~millerp/rmch/recu-make-cons-harm.html + +afcat: +- Now only outputs the segments containing data. + It does this by first scanning and making a list of all the data segments; + It then seeks to the location of each one in the virtual file, reads + the segment worth of data, and sends it to stdout. + +afconvert: +- When outputing with the -o option, now outputs an AFF file if file ends + '.aff' and outputs raw if file does not end .aff +- The -x (no compression) flag now works. +- The -X (set compression level) flag has been implemented. +- As a result, you can now batch compress a set of uncompressed AFF files + +Release 1.2.9 +afconvert: +- fixed crashing bug when converting raw files to aff files +- fixed crashing bug when converting aff files to raw files. + + +December 21, 2005 - Release 1.2.8 + +library: +- AF_BADBLOCKS is now initialized to 8 bytes of NULs, rather than + a 0-length segment. +- fixed corruption bug in af_del_seg(). +- Added (tm). + +aimage: +- Fixed calculation of free space on the disk for FreeBSD. +- Made display more attractive. +- Added support for automatically rescanning scsi bus. + You can now scan the scsi bus and image by specifying "scsi1", for + example, to image from scsi bus 1. (FreeBSD only.) +- Added optional preview to let you see the data as it is recorded. + This is great for spotting drives that have been 'sanitized' or + cleared. +- Displays size of target disk in GB/MB/KB +- Doesn't compute hash for raw image if the image is invalid + (if aimage did an lseek.) +- Now captures ATA serial number even if disk can't be imaged. +- Prints final report of image results + +afconvert: +- fixed crashing bug in afconvert. +- Corrected incorrect "progname" in many utilities. + +afinfo: +- More tolerant of errors in aff file. + +affix: +- Initial Release +- Can fix a file that is not properly terminated by removing junk at end. + This is useful if a computer crashes while aimage is being run. + +December 6, 2005 - Release 1.2.7 +- Minor fixes for makedepend problems that people on various platforms had + +November 18, 2005 - Release 1.2.5 +- Implemented segment directory; list of all segments is now kept + in memory, rather than read on-the-fly from the disk. This + dramatically improves performance on random-access test. + Library falls back to old method of scanning the file for + each segment if the segment list is too large to hold in memory. + Time for aftest -3 improves on my laptop from TK seconds to 27 seconds. + + +November 18, 2005 - Release 1.2.4 +- redesigned afflib for improved performance. +- afconvert can now convert from one AFF to another AFF. + +October 21, 2005 - Release 1.2.1 +- Added a whole bunch of segment names. +- Found a nasty bug in updating of segments under certain circumstances +- Lots of new features in aimage + +October 20, 2005 - Release 1.2.0 +- created atest for heavy-duty testing of af_seek() +- support for reading drive SN under Linux and FreeBSD +- found bug in af_seek() + +October 11, 2005 - Release 1.1.1 +- Modified af_open() so it read data_segsize on open. +- changed af_setsize() to af_set_sizes(). Now it doesn't allow you to change the + data segment size if it has already been set. +- Modified convert to handle gzipped files with gzcat. +- Fixed but in aconvert where infiles were not being closed. +- removed fancy image size calculation from ainfo, as it wasn't right for compressed files. +- Added -Wall and found lots of signage bugs +- Cleanly compiles on Linux; supports > 2GB files + + +September 11, 2005 - Release 1.1 +- Added support for writing files with af_write(). +- New files now automatically get bad block flag created +- fixed afflib for AMD64 architectures. +- ainfo wasn't calculating SHA-1 properly for -v option on AMD64. Fixed. + ainfo wasn't printing compression overhead properly. Fixed. Also fixed + small buffer overflow in printing of SHA-1 values. +- changed "radio" typo to "ratio." +- aconvert now aborts (and deletes AFF file) if ^C is pressed or if + a write fails. +- aconvert now refuses to convert an AFF file. Added af_identify(fname) to + return true if a file is probably an AFF file and false if it is not. + +August 14, 2005 - Release 1.0.1 +- changed minor bug in aconvert in handling of multiple files + +August 13, 2005 - Release 1.0 + +Initial Release of AFFLIB. Includes library description, README, basic +library functionality, and four utilities. + + +================================================================ +Copyright 2005, 2006, 2007, 2008 Simson L. Garfinkel +Copying and distribution of this file, with or without modification, are +permitted provided the copyright notice and this notice are preserved. + +# +# Local Variables: +# mode: flyspell +# mode: auto-fill +# End: +# diff --git a/INSTALL b/INSTALL new file mode 100644 index 0000000..a02d9a0 --- /dev/null +++ b/INSTALL @@ -0,0 +1,12 @@ +Install this file with: + + ./configure && make && make install + + +To disable optimization, do this: + + ./configure 'CXXFLAGS=-O0 -g' 'CFLAGS=-O0 -g' + + +Optimization is automatically disabled if the environment variable +AFF_NOOPT is set. \ No newline at end of file diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 0000000..19be4e6 --- /dev/null +++ b/Makefile.am @@ -0,0 +1,93 @@ +# Distributed under the Berekeley 4-part license + +SUBDIRS = lib tools lzma443 tests doc pyaff man + +pkgconfigdir = $(libdir)/pkgconfig + +pkgconfig_DATA = afflib.pc + +include_HEADERS = +pkginclude_HEADERS = lib/afflib.h lib/afflib_i.h lib/aftimer.h lib/utils.h + +ACLOCAL_AMFLAGS = -I m4 + +RELEASE_USER = simsong@ +RELEASE_HOST = www.afflib.org +RELEASE_DIR = afflib.org/ + +RELEASE_LOC = $(RELEASE_DIR)/downloads/ +CHANGES_LOC = $(RELEASE_DIR)/ChangeLog.txt +RELEASE_FN = $(PACKAGE)_version.txt +RELEASE_PATH = $(RELEASE_LOC)/$(RELEASE_FN) +RELEASE_SSH = $(RELEASE_USER)$(RELEASE_HOST):$(RELEASE_LOC) +RELEASE = $(PACKAGE)-$(VERSION) + +AM_CFLAGS = -Wall +AM_CPPFLAGS = -Wall + + +release: + make prerelease + make distcheck + make the_release + make distribute_release + +prerelease: + @echo Making sure SVN is up to date + svn update + @echo update PACKAGE_VERSION in win32 directory + sed s/PACKAGE_VERSION.\*/PACKAGE_VERSION\ \"$(PACKAGE_VERSION)\"/ win32/affconfig.h > win32/affconfig.h.$$ + mv -f win32/affconfig.h.$$ win32/affconfig.h + @echo Commit back changes + svn commit -m 'make prerelease' + @echo Building and uploading the release... + @echo Making release $(RELEASE) + @echo + echo + wget http://$(RELEASE_HOST)/downloads/$(RELEASE_FN) + @echo Version `cat $(RELEASE_FN)` is on the server. + python checkversion.py $(RELEASE_FN) $(RELEASE) + /bin/rm $(RELEASE_FN) + +getrelease: + @echo testing release on your system + wget http://$(PACKAGE).tar.gz + tar xfz $(PACKAGE).tar.gz + (cd afflib-*;./configure;make;make test_crypto) + +the_release: + gpg --detach-sign $(RELEASE).tar.gz + +distribute_release: + scp $(RELEASE).tar.gz{,.sig} $(RELEASE_SSH) + ssh $(RELEASE_HOST) 'cd $(RELEASE_LOC);/bin/rm $(PACKAGE).tar.gz;ln -s $(RELEASE).tar.gz $(PACKAGE).tar.gz' + ssh $(RELEASE_HOST) 'echo $(RELEASE).tar.gz > $(RELEASE_PATH)' + @echo Release $(RELEASE) uploaded to server + +DOCS = BUGLIST.txt README_Linux.txt README_Win32.txt \ + doc/crypto_doc.txt \ + doc/crypto_design.txt \ + doc/announce_1.7.txt \ + doc/announce_1.8.txt \ + doc/announce_2.2.txt + +EXTRA_DIST = $(DOCS) \ + bootstrap.sh \ + make_mingw.sh \ + m4/acx_pthread.m4 \ + afflib.spec.in \ + afflib.spec \ + afflib.pc.in \ + afflib.pubkey.asc \ + tests/encrypted.iso \ + tests/encrypted.aff \ + win32/Changes.txt \ + win32/README_MSVC++.txt \ + win32/affconfig.h \ + win32/afflib.mak \ + win32/make.bat + + +# +# Note: don't forget to run autoreconf when significant changes are made +# diff --git a/NEWS b/NEWS new file mode 100644 index 0000000..c9d5947 --- /dev/null +++ b/NEWS @@ -0,0 +1,104 @@ +April 18, 2010 +Version 3.6.0 + +I am pleased to announce the release of AFFLIB 3.6.0. + +With this release, AFFLIB now compiles under "mingw" allowing us to +directly produce statically linked executables on Windows. The AFFLIB +release has been significantly shurnk as it no longer needs to include +libewf, openssl, and zlib. + +You can now create your own AFFLIB Windows release on a Macintosh using +mingw and the script in bootstrap_mingw.sh. Or you can download our +precompiled Windows version from http://afflib.org/download/afflib_windows.zip + + + + + +September 30, 2009 +Version 3.5.0 Released. +Major features: + - Simultaneous release of AFFLIB, Bloom, and bulk_extractor for MacOS, Linux and Windows + - bulk_extractor ported to Java. + - Single ZIP file for Windows Release + +May 20, 2008 +Version 3.2.1: Cleanup release of 3.2.0; important new features: + - Detects truncated files and does not allow them to be opened. + - affix now fixes truncated files + - afcrypto will only encrypt AFF and AFD file types. + - AFFLIB will no longer open a .afd directory containing no AFF files. + +April 14, 2008 +Version 3.1.6 released. Corrects bug in handling of raw files larger than 4GB. + +================================================================ +November 26, 2007 + +AFFLIB 3.0.2 is withdrawn +AFFLIB 3.0.3 is released + +Dear AFF Users: + +AFFLIB 3.0.3 has been released. This version fixes a bug in the AFF +encryption routines that was inadvertantly introduced between +AFFLIB 3.0.1 and AFFLIB 3.0.2. As a result, AFFLIB 3.0.2 has been withdrawn. + +The bug in AFFLIB 3.0.2 resulted from a bug in a version of SHA256 +that was bundled into AFFLIB between version 3.0.1 and 3.0.2. Both +SHA256 and AES256 are required for AFF encryption. Unfortunately, the +version of SHA256 that was bundled had a data-dependent bug. This bug +only affected systems which did not have a system-installed SHA256 +implementation. + +As a result of this bug, all private implementations of cryptographic +functions have been stripped from AFFLIB. If you do not have an +OpenSSL library that has SHA256, you will not be able to use AFF +encryption. + +The practical result is that some users will not be able to use AFF +encryption without first updating their openssl library. + +ATTENTION MACINTOSH USERS: APPLE'S 10.4 and 10.5 OPERATING SYSTEMS +SHIP WITH A VERSION OF OPENSSL THAT IS OUT-OF-DATE AND DOES NOT +INCLUDE SHA256. If you are using a Macintosh, you must download a copy +of OpenSSL that has SHA256 to use AFF encryption. You can easily +download a modern OpenSSL implementation using the Macports or fink +system. + +I apologize for this problem. To prevent it from happening, we've +improved the validation of AFFLIB that runs both when AFFLIB is built +and each time it runs. + + + + + +================================================================ +August 18, 2007 +AFFLIB Release 2.4.0 + +I'm pleased to announce the general availability of AFFLIB Release +2.4.0. This release + + + +================================================================ +OLD NEWS FOLLOWS +================================================================ +November 1, 2006 + +AFFLIB Maintenance Release 2.0.1 +* Fixes a number of errors when AFFLIB was being run under Cygwin or Borland C++. + +AFFLIB Release 2.0 + +Key features of this release include: + +* Now uses the GNU build tools (autoconf & automake) + - Will probe for readline and other features + - Special addition to search for libraries in multiple locations + +* aimage2.0 released + - -L now enables LZMA compression; use -G for logfile output diff --git a/README b/README index e69de29..c56e8e2 100644 --- a/README +++ b/README @@ -0,0 +1,108 @@ + The Advanced Forensic Format + Library and Tools + Version 3 + + Simson L. Garfinkel + Naval Postgraduate School + 2012 + + +The Advanced Forensic Format (AFF) is on-disk format for storing +computer forensic information. Critical features of AFF include: + + - AFF allows you to store both computer forensic data and associated + metadata in one or more files. + + - AFF allows files to be digital singed, to provide for + chain-of-custody and long-term file integrity. + + - AFF allows for forensic disk images to stored encrypted and + decrypted on-the-fly for processing. This allows disk images + containing privacy sensitive material to be stored on the Internet. + + - AFF is an open format unencumbered by copyright or patent + protection. The AFFLIB library that implements AFF is available + for use in both Open Source and proprietary tools. + +AFF Library and Toolkit is a set of programs for working with computer +forensic information. Using these tools you can: + + * Interconvert disk images between a variety of formats, including: + + - raw or "dd" + - splitraw (in which a single image is split between mulitple files) + - EnCase or "E01" format + - AFF format (in which the entire disk image is stored in a single file.) + - AFD format (in which a disk image is stored in mulitple AFF files + stored in a single directory.) + - AFM format (in which an AFF file is used to annotate a raw file.) + + * Compare disk images and report the data or metadata that is different. + + * Copy disk images from one location to another, with full + verification of data, metadata, and the automatic generation of a + chain-of-custody segment. + + * Find errors in an AFF file and fix them. + + * Print information about a file. + + * Print detailed statistics about a file + + * Generate an XML representation of a disk image's metadata (for + example, acquisition time or the serial number of the acquisition + device.) + + * Produce an XML "diskprint" which allows a disk image to be rapidly + fingerprinted without having the computer the SHA1 of the entire + disk. + +AFFLIBv3 implements version 3 of the AFF format. This version is +currently in maintenance mode while work on AFFv4 continues. Key +differences between AFFv3 and AFFv4 include: + + * Whereas AFFv3 uses a purpose-built container file format, AFFv4 is + based on ZIP64. + + * Whereas AFFv3 is licensed with a four-part Berkeley license, AFFv4 + is licensed an approved Open Source license. + + + +AFFLIB and Toolkit is provided in source code form for Linux, MacOS +and Windows. We have also created a Windows zipfile that contains: + + * precompiled versions of the AFFLIB tools and all of the libraries + necessary to run them. + + * bulk_extractor.jar - A Java port of our system that automatically + extracts email addresses, dates, and other information from a file + and produces a histogram of the contents. + + +The AFF library can be downloaded from http://afflib.org/. + +The pre-compiled AFF for Windows can be downloaded from + http://afflib.org/windows. + +==== +AFFLIB with SleuthKit: + +TSK officially supports a subset of the image formats that AFFLIB +supports. To use the other image formats, specify the image type as +"afflib". For example: + + +# fls -o 63 -i afflib foo.vmdk + + + +================ +Note: AFF and AFFLIB are trademarks of Simson L. Garfinkel and Basis +Technology, Inc. + + +# Local Variables: +# mode: auto-fill +# mode: flyspell +# End: diff --git a/README_Linux.txt b/README_Linux.txt new file mode 100644 index 0000000..36cb516 --- /dev/null +++ b/README_Linux.txt @@ -0,0 +1,36 @@ +#!/bin/sh +# INSTALLING ON Ubuntu/Kbuntu/Debian LINUX +# + +if [ -r /usr/bin/apt-get ] ; +then + + echo you are running on a system with apt-get. + + # Edit /etc/apt/sources.list and uncomment the lines with "universe" + apt-get update + + # General build tools: + apt-get -y install make gcc g++ + + # Libraries required for AFFLIB: + apt-get -y install zlib1g-dev libssl-dev libncurses5-dev + apt-get -y install libcurl4-openssl-dev libexpat1-dev libreadline5-dev + + # Libraries if you want to make a release: + apt-get -y install automake1.9 autoconf libtool + exit 0 +fi + +if [ -r /usr/bin/yum ] ; +then + #================================================================ + #INSTALLOING ON FEDORA CORE 6: + # + # When you build Linux, tell it that you want developer tools. + # + yum upgrade all + yum install libssl-dev libncurses5-dev + exit 0 +fi + diff --git a/README_Win32.txt b/README_Win32.txt new file mode 100644 index 0000000..1dd1b8c --- /dev/null +++ b/README_Win32.txt @@ -0,0 +1,182 @@ + Using AFF Tool Under Microsoft Windows (Win32) + + +There are two ways to use AFFLIB with Windows: you can download the +pre-compiled executables, or you can compile your own. The advantage +of the pre-compiled executables is that they work. The advantage of +compiling the executables yourself is that you can modify them. + +Downloading and Installing +========================== +You can download the current version of AFF Tools from: + + http://afflib.org/downloads/afflib_windows.zip + +The ZIP file contains: + * pre-compiled executables for AFF Tools + * lib32eay.dll, the OpenSSL DLL (cryptography support for AFFLIB) + * bulk_extractor jar and bat file. (Use the bat file to run the jar file) + +Install these tools by: + +1. Unzip the archive into the c:\afflib directory. +2. Add c:\afflib to your system PATH directory by: + a. Opening the System control panel. + b. Clicking the "Environment Variables" button. + c. Adding "c:\afflib;" to the beginning of the PATH environment variable. + + +******************************* +Compiling under Windows + +There are three ways to compile for Windows: +1 - Cross-compiling from a Linux or Mac system with mingw. +2 - Compiling natively on Windows using mingw. +3 - Compiling natively on Windows using cygwin (untested) + +Cross-compiling from Linux or Mac using MINGW: +********************************************* + +* Cross-compiling works fine, but it does not include the version 4.x + GCC compiler and pthreads does not appear to work properly. + +* We used to install with mingw cross-compiling, but that created problems with multi-threading + + +Compiling natively under Windows with MINGW: +******************************************* + + Download the Windows Resource Kit from: + http://www.microsoft.com/downloads/details.aspx?familyid=9d467a69-57ff-4ae7-96ee-b18c4790cffd&displaylang=en + + Download and run mingw-get-inst-20101030.exe (or whatever version is current), + selecting all options including these: + C Compiler, C++ Compiler. MSYS Basic System, MinGW Development Toolkit. + When selecting the installation path to MinGW, Do not define a path with spaces in it. + + Start the MinGW32 shell window. + + Download the latest repository catalog and update and install modules required by MinGW + by typing the following: + mingw-get update + mingw-get install g++ + mingw-get install pthreads + mingw-get install mingw32-make + mingw-get install zlib + mingw-get install libz-dev + + Install the libraries in this order: + * expat (http://sourceforge.net/projects/expat/) + * openssl (http://openssl.org) + + For each library: + - download + - ./configure --prefix=/usr/local/ --enable-winapi=yes + - make + - make install + + For openssl, run "./config --prefix=/usr/local" rather than configure. + + Don't make directories in your home directory if there is a space in it! + Libtool doesn't handle paths with spaces in them. + + If OpenSSL is installed in /usr/local/ssl, you may need to build other libraries with: + ./configure CPPFLAGS="-I/usr/local/include" -I/usr/local/ssl/include" \ + LDFLAGS="-L/usr/local/lib -L/usr/local/ssl/lib" + + Most libraries will install in /usr/local/ ; you may need to add -I/usr/local/include to CFLAGS + and -L/usr/local/lib to your make scripts + + Still problematic, though, is actually running what is produced. Unless you link -static you will have + a lot of DLL references. Most of the DLLs are installed in /usr/local/bin/*.dll and /bin/*.dll and elsewhere, + which maps typically to c:\mingw\msys\1.0\local\bin and c:\mingw\bin\ + + + + +Compiling your own copy: +======================= +We compile with mingw. Download and install MSys. + +Next you will need to download and i + + +Working with the tools +====================== + +If you are working with an encrypted disk image, set the environment +variable AFFLIB_PASSPHRASE to be the passphrase that should be used +for decryption. + + % set AFFLIB_PASSPHRASE="this_is_my_passphrase" + +Displaying the metadata with a disk image: + + % afinfo.exe filename.aff + +To convert an AFF file into a RAW file, use: + + % affconvert.exe -e raw filename.aff + + +To reliably copy an AFF file from one location to another: + + % afcopy.exe file1.aff d:\dest\path\file2.aff + + +To compare two AFF files: + + % afcompare file1.aff file2.aff + + +To fix a corrupted AFF file: + + % affix badfile.aff + + +To print statistics about a file: + + % afstats.exe filename.aff + + + +Diskprint +================= +An exciting feature in AFF 3.5 is the ability to rapidly calculate and +verify the "print" of a disk image. A print is constructed by +computing the SHA-256 of the beginning, end, and several randomly +chosen parts of the disk image. + +To calculate the diskprint and store it in a file: + + % afdiskprint myfile.iso > myfile.xml + +To verify a diskprint + + % afdiskprint -x myfile.xml myfile.iso + + + +Verifying the AFFLIB Digital Signature +=============================== +Some organizations require that dgital signatures be verified on programs that are downloaded. + +Some AFF distributions are now signed with the AFFLIB privat key. You +can verify the distribution by downloading a copy of the public key +from the AFFLIB website or the GPG key server. + +The public key can be downloaded from the website: + + http://afflib.org/pubkey.asc + +You can also download the key directly from the GPG keyserver with +this command: + + $ gpg --keyserver subkeys.pgp.net --recv-keys 805B3DB0 + gpg: requesting key 805B3DB0 from hkp server subkeys.pgp.net + gpg: /home/simsong/.gnupg/trustdb.gpg: trustdb created + gpg: key 805B3DB0: public key "AFFLIB Distribution (Simson L. Garfinkel)" imported + gpg: Total number processed: 1 + gpg: imported: 1 + $ + diff --git a/afflib.pc.in b/afflib.pc.in new file mode 100644 index 0000000..de281f8 --- /dev/null +++ b/afflib.pc.in @@ -0,0 +1,11 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@/afflib + +Name: afflib +Description: Library to support the Advanced Forensic Format +Version: @VERSION@ +Requires: openssl +Libs: -L${libdir} -lafflib -lstdc++ +Cflags: -I${includedir} diff --git a/afflib.pubkey.asc b/afflib.pubkey.asc new file mode 100644 index 0000000..435d3c0 --- /dev/null +++ b/afflib.pubkey.asc @@ -0,0 +1,17 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.7 (Darwin) + +mQGiBEg/lAwRBADbHamoonoDrsmYwKLgtOdle/MpfPTUp0Tk/1Bu+l8f2TlJ1XBZ +XaF0+X2uZ+S23T+bOPK1mjNXXGPQHLnMZZjbxvQQB7d6qjwk/rQCm2+uLbYN3jn2 +goG9CSGatQ57fyE/zb4K0FHKWxGckEZwrc/ixLHXfI2UCp6v+hWn/5vAEwCgueFQ +AvHwokrJZrEUe17qaFzfEm8D/iyfD0tg+yJju7+pka+JHybVfvUd64bDvm4tSdAn +Qk+SRIFi3jxPOMZsKoxJ/d9RYFZWM2wWhzr616z0CludgwKLQ3FC8pDY0IMcDcRR +bVsbeBnRebCO3imN9RHw5G6eZ/cSxzm81+kebAncIXraBXCOZPGJ8nrDw8rdOyhe +rE26A/9L4eZK0GLpLTg/ohx/U7IBJ//6gxgAJTM5kfiGZ4hfR4bwZ/ue/ayRyre1 +xupcNiPJuTvECOfYf7nXtxlrYPVzpkcjvecFy9VJUE/WIvXqts5R4EhC/ZlcuI6D +6+njBHlflc1JhTTG+SSCQ2EEK6Fibk96EymEQFYAfE0VVa4LDbQpQUZGTElCIERp +c3RyaWJ1dGlvbiAoU2ltc29uIEwuIEdhcmZpbmtlbCmIYAQTEQIAIAUCSD+UDAIb +AwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEN1M3GCAWz2wrlUAn1sTy3ZKyUPO +qoj5JBtz/ZdCtLPpAKCXkWfSHrddkx6c3Zhwbz3AlgpJ+g== +=WJZv +-----END PGP PUBLIC KEY BLOCK----- diff --git a/afflib.spec.in b/afflib.spec.in new file mode 100644 index 0000000..67884bf --- /dev/null +++ b/afflib.spec.in @@ -0,0 +1,78 @@ +Name: afflib +Version: @VERSION@ +Release: 1 +Summary: Library to support the Advanced Forensic Format +Group: System Environment/Libraries +License: BSD with advertising +Source: %{name}-%{version}.tar.gz +URL: http://www.afflib.org/ +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) +BuildRequires: curl-devel expat-devel ncurses-devel openssl-devel zlib-devel + +%description +afflib is library for support of the Advanced Forensic Format (AFF). +AFF(R) is an open and extensible file format designed to store disk images and +associated metadata. + +%package devel +Summary: Header files and libraries for developing applications for afflib +Group: Development/Libraries +Requires: afflib = %{version}-%{release} + +%description devel +Header files and libraries for developing applications for afflib. + +%package tools +Summary: Several tools for reading and writing AFF files +Group: Applications/System +Requires: openssl afflib = %{version}-%{release} +BuildRequires: openssl-devel + +%description tools +Several tools for reading and writing AFF files. + +%prep +%setup -q + +%build +%configure --prefix=/usr --libdir=%{_libdir} --mandir=%{_mandir} +make %{?_smp_mflags} + +%install +rm -rf ${RPM_BUILD_ROOT} +make DESTDIR=${RPM_BUILD_ROOT} install + +%clean +rm -rf ${RPM_BUILD_ROOT} + +%post -p /sbin/ldconfig + +%postun -p /sbin/ldconfig + +%files +%defattr(644,root,root,755) +%doc AUTHORS BUGLIST.txt ChangeLog COPYING NEWS README README_Linux.txt +%attr(755,root,root) %{_libdir}/*.so.* + +%files devel +%defattr(644,root,root,755) +%doc AUTHORS BUGLIST.txt ChangeLog COPYING NEWS README README_Linux.txt +%doc doc/crypto_design.txt doc/crypto_doc.txt +%{_libdir}/*.a +%{_libdir}/*.la +%{_libdir}/*.so +%{_includedir}/afflib/afflib.h +%{_includedir}/afflib/afflib_i.h +%{_includedir}/afflib/afflib_sha256.h +%{_includedir}/afflib/aftimer.h +%{_includedir}/afflib/utils.h + +%files tools +%defattr(644,root,root,755) +%doc AUTHORS BUGLIST.txt ChangeLog COPYING NEWS README README_Linux.txt +%attr(755,root,root) %{_bindir}/* + +%changelog +* Sun Dec 9 2007 Joachim Metz 3.0.4-1 +- Initial version + diff --git a/bootstrap.sh b/bootstrap.sh new file mode 100755 index 0000000..3a7af59 --- /dev/null +++ b/bootstrap.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# +# This file is a work of a US government employee and as such is in the Public domain. +# Simson L. Garfinkel, March 12, 2012 +# +echo Bootstrap script to create configure script using autoconf +echo +# use the installed ones first, not matter what the path says. +export PATH=/usr/bin:/usr/sbin:/bin:$PATH +touch NEWS README AUTHORS ChangeLog stamp-h +aclocal +LIBTOOLIZE=glibtoolize +if test `which libtoolize`x != "x" ; + then LIBTOOLIZE=libtoolize +fi +$LIBTOOLIZE -f +autoheader -f +autoconf -f +automake --add-missing -c +echo "Ready to run configure!" +if [ $1"x" != "x" ]; then + ./configure "$@" +fi + diff --git a/checkversion.py b/checkversion.py new file mode 100644 index 0000000..3ee8832 --- /dev/null +++ b/checkversion.py @@ -0,0 +1,10 @@ +# This file is a work of a US government employee and as such is in the Public domain. +# Simson L. Garfinkel, March 12, 2012 + +import sys +server_version = open(sys.argv[1],"r").read().strip() +print "Server Version:",server_version +if(server_version == sys.argv[2]): + print "\n\nVersion",sys.argv[1],"is already on the server.\n\n" + sys.exit(-1) +sys.exit(0) diff --git a/compile b/compile new file mode 100755 index 0000000..a81e000 --- /dev/null +++ b/compile @@ -0,0 +1,136 @@ +#! /bin/sh +# Wrapper for compilers which do not understand `-c -o'. + +scriptversion=2003-11-09.00 + +# Copyright (C) 1999, 2000, 2003 Free Software Foundation, Inc. +# Written by Tom Tromey . +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + +# This file is maintained in Automake, please report +# bugs to or send patches to +# . + +case $1 in + '') + echo "$0: No command. Try \`$0 --help' for more information." 1>&2 + exit 1; + ;; + -h | --h*) + cat <<\EOF +Usage: compile [--help] [--version] PROGRAM [ARGS] + +Wrapper for compilers which do not understand `-c -o'. +Remove `-o dest.o' from ARGS, run PROGRAM with the remaining +arguments, and rename the output as expected. + +If you are trying to build a whole package this is not the +right script to run: please start by reading the file `INSTALL'. + +Report bugs to . +EOF + exit 0 + ;; + -v | --v*) + echo "compile $scriptversion" + exit 0 + ;; +esac + + +prog=$1 +shift + +ofile= +cfile= +args= +while test $# -gt 0; do + case "$1" in + -o) + # configure might choose to run compile as `compile cc -o foo foo.c'. + # So we do something ugly here. + ofile=$2 + shift + case "$ofile" in + *.o | *.obj) + ;; + *) + args="$args -o $ofile" + ofile= + ;; + esac + ;; + *.c) + cfile=$1 + args="$args $1" + ;; + *) + args="$args $1" + ;; + esac + shift +done + +if test -z "$ofile" || test -z "$cfile"; then + # If no `-o' option was seen then we might have been invoked from a + # pattern rule where we don't need one. That is ok -- this is a + # normal compilation that the losing compiler can handle. If no + # `.c' file was seen then we are probably linking. That is also + # ok. + exec "$prog" $args +fi + +# Name of file we expect compiler to create. +cofile=`echo $cfile | sed -e 's|^.*/||' -e 's/\.c$/.o/'` + +# Create the lock directory. +# Note: use `[/.-]' here to ensure that we don't use the same name +# that we are using for the .o file. Also, base the name on the expected +# object file name, since that is what matters with a parallel build. +lockdir=`echo $cofile | sed -e 's|[/.-]|_|g'`.d +while true; do + if mkdir $lockdir > /dev/null 2>&1; then + break + fi + sleep 1 +done +# FIXME: race condition here if user kills between mkdir and trap. +trap "rmdir $lockdir; exit 1" 1 2 15 + +# Run the compile. +"$prog" $args +status=$? + +if test -f "$cofile"; then + mv "$cofile" "$ofile" +fi + +rmdir $lockdir +exit $status + +# Local Variables: +# mode: shell-script +# sh-indentation: 2 +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-end: "$" +# End: diff --git a/config.h.in b/config.h.in new file mode 100644 index 0000000..29d8294 --- /dev/null +++ b/config.h.in @@ -0,0 +1,446 @@ +/* config.h.in. Generated from configure.ac by autoheader. */ + +/* Define to one of `_getb67', `GETB67', `getb67' for Cray-2 and Cray-YMP + systems. This function is required for `alloca.c' support on those systems. + */ +#undef CRAY_STACKSEG_END + +/* Define to 1 if using `alloca.c'. */ +#undef C_ALLOCA + +/* Define to 1 if you have the `AES_encrypt' function. */ +#undef HAVE_AES_ENCRYPT + +/* Define to 1 if you have `alloca', as a function or macro. */ +#undef HAVE_ALLOCA + +/* Define to 1 if you have and it should be used (not on Ultrix). + */ +#undef HAVE_ALLOCA_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_ARPA_INET_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_ASSERT_H + +/* Define to 1 if you have the `beep' function. */ +#undef HAVE_BEEP + +/* Define to 1 if you have the `CC_SHA256_Init' function. */ +#undef HAVE_CC_SHA256_INIT + +/* Define to 1 if you have the header file. */ +#undef HAVE_COMMONCRYPTO_COMMONDIGEST_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_CTYPE_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_CURSES_H + +/* Define to 1 if you have the `des_read_pw_string' function. */ +#undef HAVE_DES_READ_PW_STRING + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +#undef HAVE_DIRENT_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_DLFCN_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_DMALLOC_H + +/* Define to 1 if you have the `endwin' function. */ +#undef HAVE_ENDWIN + +/* Define to 1 if you have the `err' function. */ +#undef HAVE_ERR + +/* Define to 1 if you have the header file. */ +#undef HAVE_ERRNO_H + +/* Define to 1 if you have the `errx' function. */ +#undef HAVE_ERRX + +/* Define to 1 if you have the header file. */ +#undef HAVE_ERR_H + +/* Define to 1 if you have the `err_set_exit' function. */ +#undef HAVE_ERR_SET_EXIT + +/* Define to 1 if you have the `EVP_read_pw_string' function. */ +#undef HAVE_EVP_READ_PW_STRING + +/* Define to 1 if you have the `EVP_sha256' function. */ +#undef HAVE_EVP_SHA256 + +/* Define to 1 if you have the header file. */ +#undef HAVE_FCNTL_H + +/* Define to 1 if fseeko (and presumably ftello) exists and is declared. */ +#undef HAVE_FSEEKO + +/* Define to 1 if you have the `fstatfs' function. */ +#undef HAVE_FSTATFS + +/* Define to 1 if you have the `ftruncate' function. */ +#undef HAVE_FTRUNCATE + +/* Define to 1 if you have the header file. */ +#undef HAVE_GETOPT_H + +/* Define to 1 if you have the `getprogname' function. */ +#undef HAVE_GETPROGNAME + +/* Define to 1 if you have the `gotorc' function. */ +#undef HAVE_GOTORC + +/* Define to 1 if you have the header file. */ +#undef HAVE_INTTYPES_H + +/* Define to 1 if you have the `isalnum' function. */ +#undef HAVE_ISALNUM + +/* Define to 1 if you have the `isalphanum' function. */ +#undef HAVE_ISALPHANUM + +/* Define to 1 if you have the `isatty' function. */ +#undef HAVE_ISATTY + +/* Define to 1 if you have the `isdigit' function. */ +#undef HAVE_ISDIGIT + +/* Define to 1 if you have the `crypto' library (-lcrypto). */ +#undef HAVE_LIBCRYPTO + +/* Define to 1 if you have the `curl' library (-lcurl). */ +#undef HAVE_LIBCURL + +/* Define to 1 if you have the `ewf' library (-lewf). */ +#undef HAVE_LIBEWF + +/* Define to 1 if you have the header file. */ +#undef HAVE_LIBEWF_H + +/* Define to 1 if you have the `expat' library (-lexpat). */ +#undef HAVE_LIBEXPAT + +/* Define to 1 if you have the `md' library (-lmd). */ +#undef HAVE_LIBMD + +/* Define to 1 if you have the `ncurses' library (-lncurses). */ +#undef HAVE_LIBNCURSES + +/* Define to 1 if you have the `readline' library (-lreadline). */ +#undef HAVE_LIBREADLINE + +/* Define to 1 if you have the `ssl' library (-lssl). */ +#undef HAVE_LIBSSL + +/* Define to 1 if you have the `z' library (-lz). */ +#undef HAVE_LIBZ + +/* Define to 1 if you have the header file. */ +#undef HAVE_LINUX_FS_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_MALLOC_H + +/* Define to 1 if you have the `MD5' function. */ +#undef HAVE_MD5 + +/* Define to 1 if you have the header file. */ +#undef HAVE_MEMORY_H + +/* Define to 1 if you have the `memset' function. */ +#undef HAVE_MEMSET + +/* Define to 1 if you have the `mkdir' function. */ +#undef HAVE_MKDIR + +/* Define to 1 if you have the header file. */ +#undef HAVE_NCURSES_TERM_H + +/* Define to 1 if you have the header file, and it defines `DIR'. */ +#undef HAVE_NDIR_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_NETINET_IN_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_OPENSSL_AES_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_OPENSSL_BIO_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_OPENSSL_EVP_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_OPENSSL_FIPS_SHA_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_OPENSSL_HMAC_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_OPENSSL_MD5_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_OPENSSL_PEM_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_OPENSSL_RAND_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_OPENSSL_RSA_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_OPENSSL_SHA_H + +/* Define to 1 if you have the `PEM_read_bio_RSA_PUBKEY' function. */ +#undef HAVE_PEM_READ_BIO_RSA_PUBKEY + +/* Define to 1 if you have the `popen' function. */ +#undef HAVE_POPEN + +/* Define to 1 if you have the `printw' function. */ +#undef HAVE_PRINTW + +/* Define to 1 if you have the `putenv' function. */ +#undef HAVE_PUTENV + +/* Define to 1 if you have the `RAND_pseudo_bytes' function. */ +#undef HAVE_RAND_PSEUDO_BYTES + +/* Define to 1 if you have the header file. */ +#undef HAVE_READLINE_READLINE_H + +/* Define to 1 if you have the `regcomp' function. */ +#undef HAVE_REGCOMP + +/* Define to 1 if you have the `setupterm' function. */ +#undef HAVE_SETUPTERM + +/* Define to 1 if you have the header file. */ +#undef HAVE_SHA256_H + +/* Define to 1 if you have the `SHA256_Init' function. */ +#undef HAVE_SHA256_INIT + +/* Define to 1 if you have the header file. */ +#undef HAVE_SIGNAL_H + +/* Do we have sockaddr.sin_len? */ +#undef HAVE_SOCKADDR_SIN_LEN + +/* Define to 1 if you have the `srandom' function. */ +#undef HAVE_SRANDOM + +/* Define to 1 if you have the `srandomdev' function. */ +#undef HAVE_SRANDOMDEV + +/* Define to 1 if stdbool.h conforms to C99. */ +#undef HAVE_STDBOOL_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_STDINT_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_STDIO_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_STDLIB_H + +/* Define to 1 if you have the `strcasecmp' function. */ +#undef HAVE_STRCASECMP + +/* Define to 1 if you have the `strchr' function. */ +#undef HAVE_STRCHR + +/* Define to 1 if you have the `strdup' function. */ +#undef HAVE_STRDUP + +/* Define to 1 if you have the `strerror' function. */ +#undef HAVE_STRERROR + +/* Define to 1 if you have the header file. */ +#undef HAVE_STRINGS_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_STRING_H + +/* Define to 1 if you have the `strlcat' function. */ +#undef HAVE_STRLCAT + +/* Define to 1 if you have the `strlcpy' function. */ +#undef HAVE_STRLCPY + +/* Define to 1 if you have the `strrchr' function. */ +#undef HAVE_STRRCHR + +/* Define to 1 if `st_blocks' is member of `struct stat'. */ +#undef HAVE_STRUCT_STAT_ST_BLOCKS + +/* Define to 1 if `st_rdev' is member of `struct stat'. */ +#undef HAVE_STRUCT_STAT_ST_RDEV + +/* Define to 1 if your `struct stat' has `st_blocks'. Deprecated, use + `HAVE_STRUCT_STAT_ST_BLOCKS' instead. */ +#undef HAVE_ST_BLOCKS + +/* Define to 1 if your `struct stat' has `st_rdev'. Deprecated, use + `HAVE_STRUCT_STAT_ST_RDEV' instead. */ +#undef HAVE_ST_RDEV + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYSEXITS_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_CDEFS_H + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +#undef HAVE_SYS_DIR_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_DISK_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_IOCTL_H + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +#undef HAVE_SYS_NDIR_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_PARAM_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_SOCKET_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_STAT_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_TIME_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_TYPES_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_VFS_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_TERMCAP_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_TERM_H + +/* Define to 1 if you have the `tgetnum' function. */ +#undef HAVE_TGETNUM + +/* Define to 1 if you have the `tgetstr' function. */ +#undef HAVE_TGETSTR + +/* Define to 1 if you have the `tgoto' function. */ +#undef HAVE_TGOTO + +/* Define to 1 if you have the header file. */ +#undef HAVE_TIME_H + +/* Define to 1 if you have the `tputs' function. */ +#undef HAVE_TPUTS + +/* Define to 1 if you have the header file. */ +#undef HAVE_UNISTD_H + +/* Define to 1 if you have the `valloc' function. */ +#undef HAVE_VALLOC + +/* Define to 1 if you have the `warn' function. */ +#undef HAVE_WARN + +/* Define to 1 if you have the `warnx' function. */ +#undef HAVE_WARNX + +/* Define to 1 if you have the header file. */ +#undef HAVE_ZLIB_H + +/* Define to 1 if the system has the type `_Bool'. */ +#undef HAVE__BOOL + +/* Define to 1 if your C compiler doesn't accept -c and -o together. */ +#undef NO_MINUS_C_MINUS_O + +/* Name of package */ +#undef PACKAGE + +/* Define to the address where bug reports for this package should be sent. */ +#undef PACKAGE_BUGREPORT + +/* Define to the full name of this package. */ +#undef PACKAGE_NAME + +/* Define to the full name and version of this package. */ +#undef PACKAGE_STRING + +/* Define to the one symbol short name of this package. */ +#undef PACKAGE_TARNAME + +/* Define to the version of this package. */ +#undef PACKAGE_VERSION + +/* If using the C implementation of alloca, define if you know the + direction of stack growth for your system; otherwise it will be + automatically deduced at runtime. + STACK_DIRECTION > 0 => grows toward higher addresses + STACK_DIRECTION < 0 => grows toward lower addresses + STACK_DIRECTION = 0 => direction of growth unknown */ +#undef STACK_DIRECTION + +/* Define to 1 if you have the ANSI C header files. */ +#undef STDC_HEADERS + +/* Define to 1 if your declares `struct tm'. */ +#undef TM_IN_SYS_TIME + +/* Use FUSE to mount AFF images */ +#undef USE_FUSE + +/* Use libewf to read EnCase files */ +#undef USE_LIBEWF + +/* Enable support for Amazon S3 */ +#undef USE_S3 + +/* Version number of package */ +#undef VERSION + +/* Number of bits in a file offset, on hosts where this is settable. */ +#undef _FILE_OFFSET_BITS + +/* Define to 1 to make fseeko visible on some hosts (e.g. glibc 2.2). */ +#undef _LARGEFILE_SOURCE + +/* Define for large files, on AIX-style hosts. */ +#undef _LARGE_FILES + +/* Define to empty if `const' does not conform to ANSI C. */ +#undef const + +/* Define to `__inline__' or `__inline' if that's what the C compiler + calls it, or to nothing if 'inline' is not supported under any name. */ +#ifndef __cplusplus +#undef inline +#endif + +/* Define to `int' if does not define. */ +#undef mode_t + +/* Define to `long int' if does not define. */ +#undef off_t + +/* Define to `unsigned int' if does not define. */ +#undef size_t diff --git a/configure.ac b/configure.ac new file mode 100644 index 0000000..e48c3de --- /dev/null +++ b/configure.ac @@ -0,0 +1,316 @@ + -*- Autoconf -*- +# Process this file with autoconf to produce a configure script. +# Order is largely irrevellant, although it must start with AC_INIT and end with AC_OUTPUT +# See http://autotoolset.sourceforge.net/tutorial.html +# and http://www.openismus.com/documents/linux/automake/automake.shtml + +AC_INIT([AFFLIB],[3.7.0],[bugs@afflib.org]) +AM_INIT_AUTOMAKE +AM_MAINTAINER_MODE + +AC_CONFIG_FILES([Makefile lib/Makefile tools/Makefile lzma443/Makefile + tests/Makefile doc/Makefile pyaff/Makefile man/Makefile lib/version.h]) +AC_CONFIG_FILES([afflib.spec]) +AM_CONFIG_HEADER([affconfig.h]) + +# Where we get installed +AC_PREFIX_PROGRAM + +# Programs that we will be using +AC_PROG_CC +AC_PROG_CXX +AC_PROG_INSTALL + +# Turn off shared libraries during beta-testing, since they +# make the build process take too long. +AC_PROG_LIBTOOL +AC_CONFIG_MACRO_DIR([m4]) + +# We are now threading +m4_include([m4/acx_pthread.m4]) +m4_include([m4/acinclude.m4]) +ACX_PTHREAD() + +if test x"${cross_compiling}" = "xno" ; then + # Bring additional directories where things might be found into our + # search path. I don't know why autoconf doesn't do this by default + for spfx in /usr/local /opt/local /sw ${prefix} ; do + AC_MSG_NOTICE([checking ${spfx}/include]) + if test -d ${spfx}/include; then + CPPFLAGS="-I${spfx}/include $CPPFLAGS" + LDFLAGS="-L${spfx}/lib $LDFLAGS" + AC_MSG_NOTICE([ *** ADDING ${spfx}/include to CPPFLAGS *** ]) + AC_MSG_NOTICE([ *** ADDING ${spfx}/lib to LDFLAGS *** ]) + fi + done + AC_MSG_NOTICE([ CPPFLAGS = ${CPPFLAGS} ]) + AC_MSG_NOTICE([ LDFLAGS = ${LDFLAGS} ]) +else + AC_MSG_NOTICE([Cross Compiling --- will not update CPPFALGS or LDFLAGS with /usr/local, /opt/local or /sw]) + LIBS="$LIBS -lws2_32 -lgdi32" +fi + +if test -r /bin/uname.exe ; then + if test `uname -o` == "Msys" ; then + AC_MSG_NOTICE([Compiling with Msys. Setting flags appropriately.]) + enable_qemu="no" + #CPPFLAGS="$CPPFLAGS -D__USE_MINGW_ANSI_STDIO" + LIBS="$LIBS -lws2_32 -lgdi32" + #LDFLAGS="$LDFLAGS -Wl,--enable-auto-import" + fi +fi + +# Special features that I use +AC_FUNC_ALLOCA +AC_FUNC_FSEEKO +AC_HEADER_DIRENT +AC_HEADER_STDC +AC_SYS_LARGEFILE + + +# Checks for typedefs, structures, and compiler characteristics. +AC_HEADER_STDBOOL +AC_C_CONST +AC_C_INLINE +AC_TYPE_MODE_T +AC_TYPE_OFF_T +AC_TYPE_SIZE_T +AC_STRUCT_TM +AC_STRUCT_ST_BLOCKS +AC_STRUCT_ST_RDEV + + +# Specific headers that I plan to use +AC_CHECK_HEADERS([arpa/inet.h assert.h ctype.h dmalloc.h err.h errno.h fcntl.h getopt.h inttypes.h linux/fs.h malloc.h ncurses/term.h netinet/in.h regex.h signal.h stdint.h stdio.h stdlib.h string.h sys/cdefs.h sys/disk.h sys/file.h sys/ioctl.h sys/ioctl.h sys/param.h sys/param.h sys/socket.h sys/signal.h sys/stat.h sys/time.h sys/types.h sys/vfs.h sysexits.h term.h time.h unistd.h zlib.h _mingw.h]) + +AC_CHECK_LIB([regex],[regcomp]) # see if we need -lregex + +AC_CHECK_MEMBER([struct sockaddr_in.sin_len], + [ AC_DEFINE(HAVE_SOCKADDR_SIN_LEN, 1, [Do we have sockaddr.sin_len?]) ], + [], + [ +#include +#include +#include +]) + + +# Specific functions that we want to know about +AC_CHECK_FUNCS([getprogname strlcpy strlcat err_set_exit srandom srandomdev flock fstatfs valloc isdigit isalnum isalphanum isatty popen ftruncate memset mkdir putenv regcomp srandomdev strcasecmp strchr strdup strerror strrchr err errx warn warnx utimes unsetenv]) + +# Special features that can be enabled or disabled +AC_ARG_WITH([noopt], AC_HELP_STRING([--with-noopt],[Drop -O C flags])) + +# C++ headers +AC_PROG_CXX +AC_LANG_PUSH([C++]) +AC_CHECK_HEADERS([cstring]) +AC_CHECK_HEADERS([string], + AC_DEFINE([HAVE_STL],1,[STL Present]) + AC_MSG_NOTICE([STL Present])) +AC_LANG_POP([C++]) + +################################################################ +# For AFF tools +AC_CHECK_HEADERS([readline/readline.h]) +AC_CHECK_HEADERS([curses.h termcap.h]) +AC_CHECK_LIB([readline],[readline],, AC_MSG_RESULT([readline not installed])) +AC_CHECK_LIB([ncurses],[initscr],, AC_MSG_RESULT([ncurses not installed])) +AC_CHECK_LIB([z],[uncompress],, AC_MSG_ERROR([zlib not installed; cannot continue. Try adding zlib-dev or zlib1g-dev.])) +AC_CHECK_LIB([rt],[aio_error64]) +AC_SEARCH_LIBS(tgetent, termlib termcap tinfo curses ncurses) +AC_CHECK_FUNCS(putp tputs tgoto tgetstr tgetnum gotorc beep endwin setupterm printw) + +################################################################ +## Expat +## Required for S3 and Digital Signatures +## +AC_ARG_WITH(expat, + AS_HELP_STRING([--with-expat=PATH], [where libexpat is compiled (if it isn't installed); required for S3 and Digital Signatures]), + [LDFLAGS="-L${with_expat} $LDFLAGS" ; + CPPFLAGS="-I${with_expat}/lib $CPPFLAGS"]) + +have_expat=yes +AC_CHECK_HEADER([expat.h]) +AC_CHECK_LIB([expat],[XML_ParserCreate],,[have_expat="no ";AC_MSG_WARN([expat not found; S3 and Digital Signatures not enabled])]) + + +################################################################ +## Amazon S3 +## S3 requires curl and expat; otherwise we don't need them +AC_ARG_ENABLE(s3, + AC_HELP_STRING([--enable-s3=yes], + [Support for Amazon's S3 service. Requires CURL and Expat.]), + [enable_s3=$enableval], [enable_s3=no]) + +if test "x${enable_s3}" = "xyes" ; then + AC_MSG_NOTICE([S3 support requested. Looking for curl and expat...]) + AC_PATH_PROG(CURL_CONFIG,curl-config) + AC_ARG_WITH(curl, + AC_HELP_STRING([--with-curl=PATH], [where libcurl is installed; required for S3]), + [CURL_CONFIG="${with_curl}/bin/curl-config"]) + AC_CHECK_HEADER([curl/curl.h],, + AC_MSG_WARN([curl/curl.h not found; Disabling S3 Support.]) + enable_s3=no) + AC_CHECK_LIB([curl],[curl_global_init],, + AC_MSG_WARN([Curl library corrupt; Disabling S3 Support.]) + enable_s3=no) +fi + +S3_BIN= +if test "x${enable_s3}" = "xyes" ; then + AC_DEFINE([USE_S3],1,[Enable support for Amazon S3]) + S3_BIN='s3$(EXEEXT)' +fi +AC_SUBST(S3_BIN) +AM_CONDITIONAL([MAYBE_S3],[test "x${enable_s3}" = "xyes"]) # used by automake + +################################################################ +### QEMU +################################################################ + +AC_ARG_ENABLE([qemu]) + +echo "disable_qemu: " ${disable_qemu} +echo "enable_qemu: " ${enable_qemu} + +if test x"${cross_compiling}" = "xyes" ; then + AC_MSG_NOTICE([cross-compiling: Disabling QEMU]) + enable_qemu="no"; +fi + + +if test "x${enable_qemu}" = "xno" ; then + enable_qemu="no" + AC_DEFINE([DISABLE_QEMU],1,[User has disabled QEMU support]) +else + AC_DEFINE([USE_QEMU],1,[Use QEMU image drivers]) + enable_qemu="yes" +fi +AM_CONDITIONAL([MAYBE_QEMU],[test "x${enable_qemu}" = "xyes"]) # used by automake + + +################################################################ +### PyAFF +################################################################ +# python checks +# (requires autoconf 1.5+ and the macros in acinclude.m4) +AC_ARG_ENABLE([python], + AC_HELP_STRING([--enable-python=no], [Build python bindings (pyaff)]), + [enable_python=$enableval], [enable_python=no]) +if test "${enable_python}" = "yes" ; then + AM_PATH_PYTHON([2.5]) + AC_PYTHON_DEVEL() + AC_DEFINE([HAVE_PYTHON],1,[Build Python bindings]) +fi + +AM_CONDITIONAL(HAVE_PYTHON, test "$enable_python" = yes) + + + +## +################################################################ +################################################################ +## Crypto (must follow S3) +## Note: -lssl is needed on Linux +## -lcrypto is needed on MacOS +## -lmd is needed on some older systems +## Always check for headers, then libs, then functions +### + +AC_CHECK_HEADERS([openssl/aes.h openssl/bio.h openssl/evp.h openssl/hmac.h openssl/md5.h openssl/rand.h openssl/rsa.h openssl/sha.h openssl/pem.h openssl/x509.h]) + + +AC_CHECK_LIB([ssl],[SSL_library_init],, + AC_MSG_ERROR([OpenSSL developer library 'libssl-dev' or 'openssl-devel' not installed; cannot continue.])) + +AC_CHECK_LIB([crypto],[EVP_get_digestbyname]) +AC_CHECK_LIB([md],[MD5]) # if libmd is available, get it + +AC_CHECK_FUNCS([MD5 SHA1 AES_encrypt RAND_pseudo_bytes des_read_pw_string EVP_read_pw_string EVP_MD_size]) +AC_CHECK_FUNCS([PEM_read_bio_RSA_PUBKEY]) + +################################################################ +## FUSE: Filesystem in Userspace +AC_ARG_ENABLE(fuse, + AC_HELP_STRING([--enable-fuse=yes], + [Support for FUSE, Filesystem in Userspace. (default yes)]), + [enable_fuse=$enableval], [enable_fuse=yes]) +if test "x${enable_fuse}" = "xyes" ; then + AC_MSG_NOTICE([FUSE requested]) + CPPFLAGS="-D_FILE_OFFSET_BITS=64 -DFUSE_USE_VERSION=26 $CPPFLAGS" + if test `uname -s` = Darwin ; then + AC_MSG_NOTICE([FUSE IS NOT SUPPORTED ON MACOS]) + enable_fuse=no + fi + AC_CHECK_HEADER([fuse.h],, + AC_MSG_NOTICE([fuse.h not found; Disabling FUSE support.]) + enable_fuse=no) + AC_SUBST(FUSE_LIBS) + AC_SUBST(FUSE_CFLAGS) +fi +AFFUSE_BIN= +if test "${enable_fuse}" = "yes"; then + AC_DEFINE([USE_FUSE],1,[Use FUSE to mount AFF images]) + AFFUSE_BIN='affuse$(EXEEXT)' + FUSE_LIBS=-lfuse +fi +AC_SUBST(AFFUSE_BIN) +AM_PROG_CC_C_O dnl for affuse +## +################################################################ + +############## drop optimization flags if requeted ################ + +if test x"${AFF_NOOPT}" != "x" ; then + with_noopt="yes"; +fi + +if test "${with_noopt}" = "yes" ; then + CFLAGS=`echo "$CFLAGS" | sed s/-O[[0-9]]//` # note the double quoting! + CXXFLAGS=`echo "$CXXFLAGS" | sed s/-O[[0-9]]//` +fi + +# I am a glutten for punishment and this is security-critical software +CFLAGS="$CFLAGS -D_FORTIFY_SOURCE=2 -Wall -g" +CXXFLAGS="$CXXFLAGS -D_FORTIFY_SOURCE=2 -Wall -g" + + +################ drop threading if requested ################ +AC_ARG_ENABLE(threading, + [ --disable-threading builds library without threading + --enable-threading use threading if available]) +if test "x$disable_threading" != xno; then + PTHREAD_CFLAGS="" + PTHREAD_CXXFLAGS="" + PTHREAD_LDFLAGS="" + AC_DEFINE(DISABLE_PTHREAD,1,[User has disabled PTHREADING]) +fi +AM_CONDITIONAL([THREADING], [test "x$enable_threading" != xno]) + +###### Tell user what flags we have ####### +# see http://pre.plt-scheme.org/plt/src/mzscheme/configure.ac +# +AC_MSG_NOTICE([]) +AC_MSG_NOTICE([*****************************************]) +AC_MSG_NOTICE([AFFLIB ${PACKAGE_VERSION} configuration]) +AC_MSG_NOTICE([Amazon S3 Support: ${enable_s3}]) +AC_MSG_NOTICE([LZMA Compression: yes]) +AC_MSG_NOTICE([PYTHON Bindings: ${enable_python}]) +AC_MSG_NOTICE([QEMU Image Drivers: ${enable_qemu}]) +AC_MSG_NOTICE([FUSE: ${enable_fuse}]) +AC_MSG_NOTICE([LIBEXPAT: ${have_expat} (needed for AFF signatures)]) +AC_MSG_NOTICE([]) +AC_MSG_NOTICE([CFLAGS: ${CFLAGS}]) +AC_MSG_NOTICE([CPPFLAGS: ${CPPFLAGS}]) +AC_MSG_NOTICE([CXXFLAGS: ${CXXFLAGS}]) +AC_MSG_NOTICE([LIBS: ${LIBS}]) +AC_MSG_NOTICE([LDFLAGS: ${LDFLAGS}]) +AC_MSG_NOTICE([*****************************************]) +AC_MSG_NOTICE([]) + +# AC_PROG_RANLIB not needed if you are using AC_PROG_LIBTOOL +# AC_PROG_RANLIB +AC_OUTPUT([afflib.pc]) + + diff --git a/depcomp b/depcomp new file mode 100755 index 0000000..04701da --- /dev/null +++ b/depcomp @@ -0,0 +1,530 @@ +#! /bin/sh +# depcomp - compile a program generating dependencies as side-effects + +scriptversion=2005-07-09.11 + +# Copyright (C) 1999, 2000, 2003, 2004, 2005 Free Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + +# Originally written by Alexandre Oliva . + +case $1 in + '') + echo "$0: No command. Try \`$0 --help' for more information." 1>&2 + exit 1; + ;; + -h | --h*) + cat <<\EOF +Usage: depcomp [--help] [--version] PROGRAM [ARGS] + +Run PROGRAMS ARGS to compile a file, generating dependencies +as side-effects. + +Environment variables: + depmode Dependency tracking mode. + source Source file read by `PROGRAMS ARGS'. + object Object file output by `PROGRAMS ARGS'. + DEPDIR directory where to store dependencies. + depfile Dependency file to output. + tmpdepfile Temporary file to use when outputing dependencies. + libtool Whether libtool is used (yes/no). + +Report bugs to . +EOF + exit $? + ;; + -v | --v*) + echo "depcomp $scriptversion" + exit $? + ;; +esac + +if test -z "$depmode" || test -z "$source" || test -z "$object"; then + echo "depcomp: Variables source, object and depmode must be set" 1>&2 + exit 1 +fi + +# Dependencies for sub/bar.o or sub/bar.obj go into sub/.deps/bar.Po. +depfile=${depfile-`echo "$object" | + sed 's|[^\\/]*$|'${DEPDIR-.deps}'/&|;s|\.\([^.]*\)$|.P\1|;s|Pobj$|Po|'`} +tmpdepfile=${tmpdepfile-`echo "$depfile" | sed 's/\.\([^.]*\)$/.T\1/'`} + +rm -f "$tmpdepfile" + +# Some modes work just like other modes, but use different flags. We +# parameterize here, but still list the modes in the big case below, +# to make depend.m4 easier to write. Note that we *cannot* use a case +# here, because this file can only contain one case statement. +if test "$depmode" = hp; then + # HP compiler uses -M and no extra arg. + gccflag=-M + depmode=gcc +fi + +if test "$depmode" = dashXmstdout; then + # This is just like dashmstdout with a different argument. + dashmflag=-xM + depmode=dashmstdout +fi + +case "$depmode" in +gcc3) +## gcc 3 implements dependency tracking that does exactly what +## we want. Yay! Note: for some reason libtool 1.4 doesn't like +## it if -MD -MP comes after the -MF stuff. Hmm. + "$@" -MT "$object" -MD -MP -MF "$tmpdepfile" + stat=$? + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile" + exit $stat + fi + mv "$tmpdepfile" "$depfile" + ;; + +gcc) +## There are various ways to get dependency output from gcc. Here's +## why we pick this rather obscure method: +## - Don't want to use -MD because we'd like the dependencies to end +## up in a subdir. Having to rename by hand is ugly. +## (We might end up doing this anyway to support other compilers.) +## - The DEPENDENCIES_OUTPUT environment variable makes gcc act like +## -MM, not -M (despite what the docs say). +## - Using -M directly means running the compiler twice (even worse +## than renaming). + if test -z "$gccflag"; then + gccflag=-MD, + fi + "$@" -Wp,"$gccflag$tmpdepfile" + stat=$? + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile" + exit $stat + fi + rm -f "$depfile" + echo "$object : \\" > "$depfile" + alpha=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz +## The second -e expression handles DOS-style file names with drive letters. + sed -e 's/^[^:]*: / /' \ + -e 's/^['$alpha']:\/[^:]*: / /' < "$tmpdepfile" >> "$depfile" +## This next piece of magic avoids the `deleted header file' problem. +## The problem is that when a header file which appears in a .P file +## is deleted, the dependency causes make to die (because there is +## typically no way to rebuild the header). We avoid this by adding +## dummy dependencies for each header file. Too bad gcc doesn't do +## this for us directly. + tr ' ' ' +' < "$tmpdepfile" | +## Some versions of gcc put a space before the `:'. On the theory +## that the space means something, we add a space to the output as +## well. +## Some versions of the HPUX 10.20 sed can't process this invocation +## correctly. Breaking it into two sed invocations is a workaround. + sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile" + rm -f "$tmpdepfile" + ;; + +hp) + # This case exists only to let depend.m4 do its work. It works by + # looking at the text of this script. This case will never be run, + # since it is checked for above. + exit 1 + ;; + +sgi) + if test "$libtool" = yes; then + "$@" "-Wp,-MDupdate,$tmpdepfile" + else + "$@" -MDupdate "$tmpdepfile" + fi + stat=$? + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile" + exit $stat + fi + rm -f "$depfile" + + if test -f "$tmpdepfile"; then # yes, the sourcefile depend on other files + echo "$object : \\" > "$depfile" + + # Clip off the initial element (the dependent). Don't try to be + # clever and replace this with sed code, as IRIX sed won't handle + # lines with more than a fixed number of characters (4096 in + # IRIX 6.2 sed, 8192 in IRIX 6.5). We also remove comment lines; + # the IRIX cc adds comments like `#:fec' to the end of the + # dependency line. + tr ' ' ' +' < "$tmpdepfile" \ + | sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' | \ + tr ' +' ' ' >> $depfile + echo >> $depfile + + # The second pass generates a dummy entry for each header file. + tr ' ' ' +' < "$tmpdepfile" \ + | sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' -e 's/$/:/' \ + >> $depfile + else + # The sourcefile does not contain any dependencies, so just + # store a dummy comment line, to avoid errors with the Makefile + # "include basename.Plo" scheme. + echo "#dummy" > "$depfile" + fi + rm -f "$tmpdepfile" + ;; + +aix) + # The C for AIX Compiler uses -M and outputs the dependencies + # in a .u file. In older versions, this file always lives in the + # current directory. Also, the AIX compiler puts `$object:' at the + # start of each line; $object doesn't have directory information. + # Version 6 uses the directory in both cases. + stripped=`echo "$object" | sed 's/\(.*\)\..*$/\1/'` + tmpdepfile="$stripped.u" + if test "$libtool" = yes; then + "$@" -Wc,-M + else + "$@" -M + fi + stat=$? + + if test -f "$tmpdepfile"; then : + else + stripped=`echo "$stripped" | sed 's,^.*/,,'` + tmpdepfile="$stripped.u" + fi + + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile" + exit $stat + fi + + if test -f "$tmpdepfile"; then + outname="$stripped.o" + # Each line is of the form `foo.o: dependent.h'. + # Do two passes, one to just change these to + # `$object: dependent.h' and one to simply `dependent.h:'. + sed -e "s,^$outname:,$object :," < "$tmpdepfile" > "$depfile" + sed -e "s,^$outname: \(.*\)$,\1:," < "$tmpdepfile" >> "$depfile" + else + # The sourcefile does not contain any dependencies, so just + # store a dummy comment line, to avoid errors with the Makefile + # "include basename.Plo" scheme. + echo "#dummy" > "$depfile" + fi + rm -f "$tmpdepfile" + ;; + +icc) + # Intel's C compiler understands `-MD -MF file'. However on + # icc -MD -MF foo.d -c -o sub/foo.o sub/foo.c + # ICC 7.0 will fill foo.d with something like + # foo.o: sub/foo.c + # foo.o: sub/foo.h + # which is wrong. We want: + # sub/foo.o: sub/foo.c + # sub/foo.o: sub/foo.h + # sub/foo.c: + # sub/foo.h: + # ICC 7.1 will output + # foo.o: sub/foo.c sub/foo.h + # and will wrap long lines using \ : + # foo.o: sub/foo.c ... \ + # sub/foo.h ... \ + # ... + + "$@" -MD -MF "$tmpdepfile" + stat=$? + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile" + exit $stat + fi + rm -f "$depfile" + # Each line is of the form `foo.o: dependent.h', + # or `foo.o: dep1.h dep2.h \', or ` dep3.h dep4.h \'. + # Do two passes, one to just change these to + # `$object: dependent.h' and one to simply `dependent.h:'. + sed "s,^[^:]*:,$object :," < "$tmpdepfile" > "$depfile" + # Some versions of the HPUX 10.20 sed can't process this invocation + # correctly. Breaking it into two sed invocations is a workaround. + sed 's,^[^:]*: \(.*\)$,\1,;s/^\\$//;/^$/d;/:$/d' < "$tmpdepfile" | + sed -e 's/$/ :/' >> "$depfile" + rm -f "$tmpdepfile" + ;; + +tru64) + # The Tru64 compiler uses -MD to generate dependencies as a side + # effect. `cc -MD -o foo.o ...' puts the dependencies into `foo.o.d'. + # At least on Alpha/Redhat 6.1, Compaq CCC V6.2-504 seems to put + # dependencies in `foo.d' instead, so we check for that too. + # Subdirectories are respected. + dir=`echo "$object" | sed -e 's|/[^/]*$|/|'` + test "x$dir" = "x$object" && dir= + base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'` + + if test "$libtool" = yes; then + # With Tru64 cc, shared objects can also be used to make a + # static library. This mecanism is used in libtool 1.4 series to + # handle both shared and static libraries in a single compilation. + # With libtool 1.4, dependencies were output in $dir.libs/$base.lo.d. + # + # With libtool 1.5 this exception was removed, and libtool now + # generates 2 separate objects for the 2 libraries. These two + # compilations output dependencies in in $dir.libs/$base.o.d and + # in $dir$base.o.d. We have to check for both files, because + # one of the two compilations can be disabled. We should prefer + # $dir$base.o.d over $dir.libs/$base.o.d because the latter is + # automatically cleaned when .libs/ is deleted, while ignoring + # the former would cause a distcleancheck panic. + tmpdepfile1=$dir.libs/$base.lo.d # libtool 1.4 + tmpdepfile2=$dir$base.o.d # libtool 1.5 + tmpdepfile3=$dir.libs/$base.o.d # libtool 1.5 + tmpdepfile4=$dir.libs/$base.d # Compaq CCC V6.2-504 + "$@" -Wc,-MD + else + tmpdepfile1=$dir$base.o.d + tmpdepfile2=$dir$base.d + tmpdepfile3=$dir$base.d + tmpdepfile4=$dir$base.d + "$@" -MD + fi + + stat=$? + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" "$tmpdepfile4" + exit $stat + fi + + for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" "$tmpdepfile4" + do + test -f "$tmpdepfile" && break + done + if test -f "$tmpdepfile"; then + sed -e "s,^.*\.[a-z]*:,$object:," < "$tmpdepfile" > "$depfile" + # That's a tab and a space in the []. + sed -e 's,^.*\.[a-z]*:[ ]*,,' -e 's,$,:,' < "$tmpdepfile" >> "$depfile" + else + echo "#dummy" > "$depfile" + fi + rm -f "$tmpdepfile" + ;; + +#nosideeffect) + # This comment above is used by automake to tell side-effect + # dependency tracking mechanisms from slower ones. + +dashmstdout) + # Important note: in order to support this mode, a compiler *must* + # always write the preprocessed file to stdout, regardless of -o. + "$@" || exit $? + + # Remove the call to Libtool. + if test "$libtool" = yes; then + while test $1 != '--mode=compile'; do + shift + done + shift + fi + + # Remove `-o $object'. + IFS=" " + for arg + do + case $arg in + -o) + shift + ;; + $object) + shift + ;; + *) + set fnord "$@" "$arg" + shift # fnord + shift # $arg + ;; + esac + done + + test -z "$dashmflag" && dashmflag=-M + # Require at least two characters before searching for `:' + # in the target name. This is to cope with DOS-style filenames: + # a dependency such as `c:/foo/bar' could be seen as target `c' otherwise. + "$@" $dashmflag | + sed 's:^[ ]*[^: ][^:][^:]*\:[ ]*:'"$object"'\: :' > "$tmpdepfile" + rm -f "$depfile" + cat < "$tmpdepfile" > "$depfile" + tr ' ' ' +' < "$tmpdepfile" | \ +## Some versions of the HPUX 10.20 sed can't process this invocation +## correctly. Breaking it into two sed invocations is a workaround. + sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile" + rm -f "$tmpdepfile" + ;; + +dashXmstdout) + # This case only exists to satisfy depend.m4. It is never actually + # run, as this mode is specially recognized in the preamble. + exit 1 + ;; + +makedepend) + "$@" || exit $? + # Remove any Libtool call + if test "$libtool" = yes; then + while test $1 != '--mode=compile'; do + shift + done + shift + fi + # X makedepend + shift + cleared=no + for arg in "$@"; do + case $cleared in + no) + set ""; shift + cleared=yes ;; + esac + case "$arg" in + -D*|-I*) + set fnord "$@" "$arg"; shift ;; + # Strip any option that makedepend may not understand. Remove + # the object too, otherwise makedepend will parse it as a source file. + -*|$object) + ;; + *) + set fnord "$@" "$arg"; shift ;; + esac + done + obj_suffix="`echo $object | sed 's/^.*\././'`" + touch "$tmpdepfile" + ${MAKEDEPEND-makedepend} -o"$obj_suffix" -f"$tmpdepfile" "$@" + rm -f "$depfile" + cat < "$tmpdepfile" > "$depfile" + sed '1,2d' "$tmpdepfile" | tr ' ' ' +' | \ +## Some versions of the HPUX 10.20 sed can't process this invocation +## correctly. Breaking it into two sed invocations is a workaround. + sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile" + rm -f "$tmpdepfile" "$tmpdepfile".bak + ;; + +cpp) + # Important note: in order to support this mode, a compiler *must* + # always write the preprocessed file to stdout. + "$@" || exit $? + + # Remove the call to Libtool. + if test "$libtool" = yes; then + while test $1 != '--mode=compile'; do + shift + done + shift + fi + + # Remove `-o $object'. + IFS=" " + for arg + do + case $arg in + -o) + shift + ;; + $object) + shift + ;; + *) + set fnord "$@" "$arg" + shift # fnord + shift # $arg + ;; + esac + done + + "$@" -E | + sed -n -e '/^# [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' \ + -e '/^#line [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' | + sed '$ s: \\$::' > "$tmpdepfile" + rm -f "$depfile" + echo "$object : \\" > "$depfile" + cat < "$tmpdepfile" >> "$depfile" + sed < "$tmpdepfile" '/^$/d;s/^ //;s/ \\$//;s/$/ :/' >> "$depfile" + rm -f "$tmpdepfile" + ;; + +msvisualcpp) + # Important note: in order to support this mode, a compiler *must* + # always write the preprocessed file to stdout, regardless of -o, + # because we must use -o when running libtool. + "$@" || exit $? + IFS=" " + for arg + do + case "$arg" in + "-Gm"|"/Gm"|"-Gi"|"/Gi"|"-ZI"|"/ZI") + set fnord "$@" + shift + shift + ;; + *) + set fnord "$@" "$arg" + shift + shift + ;; + esac + done + "$@" -E | + sed -n '/^#line [0-9][0-9]* "\([^"]*\)"/ s::echo "`cygpath -u \\"\1\\"`":p' | sort | uniq > "$tmpdepfile" + rm -f "$depfile" + echo "$object : \\" > "$depfile" + . "$tmpdepfile" | sed 's% %\\ %g' | sed -n '/^\(.*\)$/ s:: \1 \\:p' >> "$depfile" + echo " " >> "$depfile" + . "$tmpdepfile" | sed 's% %\\ %g' | sed -n '/^\(.*\)$/ s::\1\::p' >> "$depfile" + rm -f "$tmpdepfile" + ;; + +none) + exec "$@" + ;; + +*) + echo "Unknown depmode $depmode" 1>&2 + exit 1 + ;; +esac + +exit 0 + +# Local Variables: +# mode: shell-script +# sh-indentation: 2 +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-end: "$" +# End: diff --git a/doc/FAQ.txt b/doc/FAQ.txt new file mode 100644 index 0000000..407095c --- /dev/null +++ b/doc/FAQ.txt @@ -0,0 +1,100 @@ +# This file is distributed under the Berkeley 4-part license + +Q: Why a new file format? What's wrong with block-by-block? + +A: Raw image files take up a lot of space. In many cases this space + can be dramatically reduced by using compression. + Unfortunately, if you just use "gzip" or "bzip2" for compression, + you need to uncompress the entire file in order to use it with a + forensics program. That's because there is no easy way to "seek" + within a compressed file. + + The proprietary EnCase file format supports seeking within a + compressed file, but the specification for this file format is not + publicly available. Also, EnCase does not allow the storage of + arbitrary name/value pairs. + + +Q: Why not put meta information into log files? + +A: In many cases it is advantageous to store meta information (such as + case numbers, acquisition times, the name of the investigator, + etc.) directly in the image file. For example, storing this + information in a single file with the image makes it very unlikely + that they will become separated, and perhaps the wrong log file + being used with an image. + +Q: Will AFF support hashes other than MD5 and SHA-1? + +A: Yes. The MD5 hash is stored in a segment named "md5". The SHA-1 + hash is stored in a segment named "sha1". As support for other hash + functions are added to the OpenSSL library, the "aconvert" and + "aimage" programs will be updated to automatically calculate and + store the other hashes in the AFF files. + + +Q: Are the images directly mountable so that they can be used with + today's forensic tools, or must they be uncompressed for host-based tools to + work with them? + +A: If you have source code for a scanner, you can modify it to use + af_open() and af_read() instead of fopen() and fread(). You can + then read the AFF files directly. If you don't have source code, + but have a scanner that can read from standard input, you can + use the "affcat" program to copy the contents of an AFF file to + standard output. + + Eventually, we plan to have a version of samba that is modified to + transparently mount and serve an AFF file. This will allow + off-the-shelf Windows executables to be used with AFF archives. + + +Q: How long do you think it will be before EnCase, + ProDiscover, FTK, and the open source tools are able + to process files in this format? + +A: I'm currently modifying some of the Open Source tools to handle + AFF and hope to have an announcement regarding this relatively + soon. It's actually quite easy to modify an Open Source tool to work + with AFF: you simply replace the fopen() call with af_open(), fseek() + with af_seek(), and fclose() with af_close(). + + I've had no contact with the authors of EnCase, ProDiscover, and + FTK. I imagine that if AFF becomes popular they will modify their + tools to handle the format. But even if they don't, we plan to + support those tools through the use of a samba loopback + filesystem. + + +Q: Will it be possible to mount an AFF image as a "virtual file + system" the way you can with EnCase. + +A: Yes, it is quite possible to create a device driver that would + perform the necessary transformation. That is how the AFFLIB + af_read() and af_seek() function calls are implemented. + + + +Q: Wouldn't it be more efficient to have an index segment, rather than + having to read through all of the individual AF Headers for each + segment? + +A: We thought so as well! However, our initial experiments indicated + that the overhead for doing a seek for every 16MB segment and + reading a few bytes was quite minimal. The advantage of not having + to maintain the index is significant. However, if the overhead + becomes substantial, we can easily add an index segment type. The + design of AFF allows for an index segment to be added to an + existing AFF file without changing the contents of the segments + that contain forensic information. + + +Q: Its very important for us to have a format which can be written + into a pipe because that makes acquisition over the network much + easier. + +A: The "aimage" acquisition program currently under development allows + for acquisition either from an ATA/USB/Firewire device or over a + network. It allows for discontinuous segments of the disk to be + acquired at different times and for data to be inserted into a + single AFF file. diff --git a/doc/Makefile.am b/doc/Makefile.am new file mode 100644 index 0000000..917eff2 --- /dev/null +++ b/doc/Makefile.am @@ -0,0 +1,13 @@ +# This file is a work of a US government employee and as such is in the Public domain. +# Simson L. Garfinkel, March 12, 2012 + + +EXTRA_DIST = FAQ.txt crypto_api.txt crypto_design.txt crypto_doc.txt filetypes.txt\ + threading.txt \ + annotations.txt \ + announce_1.0.txt \ + announce_1.7.txt \ + announce_1.8.txt \ + announce_2.2.txt \ + announce_3.0.txt \ + announce_3.3.txt diff --git a/doc/aff1_announce.doc b/doc/aff1_announce.doc new file mode 100644 index 0000000..2cfbd59 Binary files /dev/null and b/doc/aff1_announce.doc differ diff --git a/doc/aff4_crypto.txt b/doc/aff4_crypto.txt new file mode 100644 index 0000000..112ab16 --- /dev/null +++ b/doc/aff4_crypto.txt @@ -0,0 +1,192 @@ +# This file is a work of a US government employee and as such is in the Public domain. +# Simson L. Garfinkel, March 12, 2012 + +Hi Guys, + As discussed I finally finished implementing the AFF4 encryption +scheme based on the RDF dataType. A Cipher is an RDF dataType which +takes care of encrypting blocks. Each implementation is responsible +for serialising itself and parsing itself from the RDF serialization. + +This is how the API works: + +## Make the volume +volume = oracle.create(pyaff4.AFF4_ZIP_VOLUME) +volume.set(pyaff4.AFF4_STORED, url) +volume = volume.finish() +volume_urn = volume.urn +volume.cache_return() + +## Make the image +image = oracle.create(pyaff4.AFF4_IMAGE) +image.set(pyaff4.AFF4_STORED, volume_urn) +image = image.finish() +image_urn = image.urn +image.cache_return() + +# Make the encrypted stream +encrypted = oracle.create(pyaff4.AFF4_ENCRYTED) +encrypted.urn.set(volume_urn.value) +encrypted.urn.add(SOURCE) + +encrypted.set(pyaff4.AFF4_STORED, volume_urn) +encrypted.set(pyaff4.AFF4_TARGET, image_urn) + +## Set the certificate: +cipher = oracle.new_rdfvalue(pyaff4.AFF4_AES256_X509) +cert_urn = pyaff4.RDFURN() +cert_urn.set(CERT_LOCATION) +cipher.set_authority(cert_urn) + +## Add the cert cipher to the encrypted stream +encrypted.add(pyaff4.AFF4_CIPHER, cipher) + +## Now just for fun we also want a password cipher: +cipher = oracle.new_rdfvalue(pyaff4.AFF4_AES256_PASSWORD) +encrypted.add(pyaff4.AFF4_CIPHER, cipher) + +## Ok done with the encrypted stream +encrypted = encrypted.finish() +encrypted_urn = encrypted.urn + +## Copy the source to the encrypted stream. +infd = open(SOURCE) +while 1: + data = infd.read(2**24) + if not data: break + + encrypted.write(data) + +## Close everything down now +encrypted.close() + +image = oracle.open(image_urn, "w") +image.close() + +volume = oracle.open(volume_urn, 'w') +volume.close() + + +Basically the general pattern for creating any object entails the +general pattern: + +1) Create a new object via oracle.create(TYPE_OF_OBJECT) +2) Set various attributes on the object via object.set() or object.add() +3) Finish the creation of the object via object.finish(). +4) Use the object however you want +5) Return the object to the cache via object.cache_return() + +For most objects we dont need to use them directly but we need to have +their URNs which we keep around in order to set other attributes on +other objects. Once we returned the object to the cache we do not own +it any more and can not touch it. So to close the objects we need to +call oracle.open() to receive it by URN. (there is a locking semantic +here in which the object is locked to our thread between the +create/open call and the cache_return() - thats why we can not touch +it outside these calls). + +The resulting RDF from the above is: +@base . +@prefix rdf: . +@prefix aff4: . +@prefix xsd: . + + + aff4:index "0,32784,65568,"^^aff4:integer_array_inline . + + + aff4:cipher +"9XBDFCIRlIhE/HL9xHvIEbixsbUr6CbVkfnFUKhErnlS6h41yTh8qY7Fk+7Oq8zn4kkSnKyTE5eIx4xTYl1yGg=="^^aff4:aes256-password, +"file:///home/mic/projects/aff4/python2.6/tests/sign.key#9XBDFCIRlIhE/HL9xHvIEZ0JM9Hjn6c+EV1OeFa9ZclztWwtdN1Bd191N2kAA4ksOl/sLm2b/r/1GaPAQZNty+QI/mqg61r8OCWkGelhU2OuWImVD6moGZQuVmvkJUHbDdawSsaSh8f7gn5FYPwBbkR665FhklzWEXkaYhLEs1ugUqRk4sKS20FZtrt0yEOPFgfZylf/6hB6SlIPyVoVGBSdHRSg82AcTN48HJU1Q8KYElBaMT5S54Nm7JOCijAqeDsI4NmB2QYy/aaAOWgXKA=="^^aff4:aes256-x509 +; + aff4:size 96216 ; + aff4:stored <> ; + aff4:target ; + aff4:timestamp "2010-03-12T12:11:23.000000+00:00"^^xsd:dateTime ; + a aff4:encrypted . + + + aff4:chunk_size 32768 ; + aff4:chunks_in_segment 500 ; + aff4:compression 8 ; + aff4:size 98304 ; + aff4:stored <> ; + aff4:timestamp "2010-03-12T12:11:23.000000+00:00"^^xsd:dateTime ; + a aff4:image . + +Note how there are 2 cipher objects now - a password and an x509 +cipher. We just need to find one that works and it will unlock the +master key. + +AFF4 doesnt care about key management or cert management or anything +of the sort - this is not our problem and we dont really care about +it. The old AFFLIB does a lot of key management approaches: + +1) It uses environment variables +2) It can accept passwords or certs via the API + +These approaches are clearly inadequate for a large scale library +implementation because the library may be part of a large application +which can not provide passwords or certs in advance. Imagine a GUI +application with the user selecting an AFF4 stream to open - the +application will need to know in advance if the object we need to open +is encrypted so it can prompt the user. Further the application will +need to know how its encrypted so it can prompt for password or a +cert or whatever other way. At least in the AFF4 universe its pretty +much impossible to know in advance if encryption will be required +because you might be opening a map which dereferences another map +which finally dereferences a stream which happens to be stored in an +encrypted volume. + +Another problem is that applications generally have different ways to +manage certs and private keys. For example you can imaging a windows +based application might want to use the OS secure storage facility for +private keys, or ldap for certificates. We can not predict how they +want to use our library. + +For these reasons the AFF4 library has a concept of a +SecurityProvider. This is an object which is registered by the user +with various methods. There is a default object which just grabs stuff +from the environment. The user can install a new security manager +implementation to implement whatever crazy key management protocols +they want. We dont care about the sepecifics: + +class SecurityProvider: + """ This is a demonstration security provider object which will be + called by the AFF4 library to get keying material for different + streams. + """ + def passphrase(self, cipher, subject): + print "Setting passphrase for subject %s" % subject.value + return "Hello" + + def x509_private_key(self, cert_name, subject): + """ Returns the private key (in pem format) for the +certificate name provided. """ + print "Certificate for %s" % cert_name + return open(CERT_LOCATION).read() + +## This registers the security provider +oracle.register_security_provider(pyaff4.ProxiedSecurityProvider(SecurityProvider())) + +Now we get the following behaviour: +1) Whenever the library hits an encrypted stream, it will try to +decode the cipher RDF dataType. Depending on the type of the +aff4:cipher property, calls will be made to the security provider to +retrieve the passphrase or key or whaterver for the subject. +2) The application can then do whatever it needs to do to satisfy the +call. For example it might look it up on ldap or pop a gui or do a db +query whatever is needed. +3) If the required passphrase is not provided, we try to decode the +next cipher attributes. So the user might get asked for a password, +then a certificate private key etc. + +The nice thing about it is that it only happens when its needed - the +user doesnt need prior knowledge about the encryption status of the +volumes and pre-seed the keys before they start. This facilitates the +library's use in a more complex application rather than a single shot +app. + +I will now add this encryption to the tools. I am now looking at +signatures which I think can be made using the same approach. + +Michael. diff --git a/doc/affdoc.doc b/doc/affdoc.doc new file mode 100644 index 0000000..3960cbc Binary files /dev/null and b/doc/affdoc.doc differ diff --git a/doc/announce_1.0.txt b/doc/announce_1.0.txt new file mode 100644 index 0000000..444fae0 --- /dev/null +++ b/doc/announce_1.0.txt @@ -0,0 +1,307 @@ + The Advanced Forensic Format + Library and Tools + + Simson L. Garfinkel + and + Basis Technology, Inc. + (C) 2005-2006 + + + +INTRODUCTION + +The Advanced Forensic Format --- AFF(tm) --- is an extensible open +format for the storage of disk images and related forensic +information. + +Features of AFF include: + +* Open format, free from any patent or license restriction. + Can be used with both open-source and proprietary forensic tools. + +* Extensible. Any amount of metadata can be encoded in AFF files in the + format of name/value pairs. + +* Efficient. AFF supports both compression and seeking within + compressed files. + +* Open Source C/C++ Implementation. A freely redistributable C/C++ + implementation including the AFF Library and basic conversion tools + is available for download. AFFLIB(tm) is being distributed under the BSD + license, allowing it to be incorporated in free and proprietary + programs without the need to pay license fees. + +* Byte-order independent. AFFLib has been tested on both Intel and + PowerPC-based systems. Images created on one platform can be read on + another. + +* Automatic calculation and storage of MD5 and SHA-1 hash codes, + allowing AFF files to be automatically validated after they are + copied to check for accidentally corruption. + +* Explicit identification of sectors that could not be read from the + original disk. + +* Planned support for digital signatures and direct authoring of AFF + files using dd_recover. + +AFF is distributed as a subroutine library. The library implements a +FILE-like abstraction that supports the full range of POSIX-like file +routines, including af_open(), af_read(), af_seek(), af_close(). The +af_open() routine checks to see if a file is an AFF file and, if not, +can automatically fall-back into raw mode. Thus, most existing +forensic tools can be trivially modified to work with AFF-formatted +files. + + +The AFF library can be downloaded from: + +http://www.simson.net/afflib/ + + +================================================================ + HOW AFF WORKS + +AFF is a segmented archive file specification. Each AFF file consists +of an AFF File Header followed by one or more AFF Segments. + + +AFF SEGMENTS: + +AFF Segments are used to store all information inside the AFF +File. This includes the image itself and image metadata. + +Segments can be between 32 bytes and 2^32-1 bytes long. When used to +store the contents of a disk image, the image is broken up into a +number of equal-sized Image Segments. These Image Segments are then optionally +compressed and stored sequentially in the AFF file. + +Each AFF Segment has a header, a name, a 32-bit argument, an optional data +payload, and finally a tail. The header and tail make it possible to +seek rapidly through the AFF file, skipping much of the image data. + +The segment size of the image file is determined when the file is +converted from a RAW file to an AFF file. Once a file is converted, it +can be opened using the af_open() function call and read using +af_read() and af_seek(). The AFF library automatically handles the +locating, reading, and optional decompressing of each segment as needed. + +Other segments can be used to hold information such as the time that +the disk was imaged, a case number, the forensic examiner, and the MD5 +or SHA-1 of the original unconverted image file. Utility programs are +included in the AFF Library to display this information and validate +the contents of an AFF file against the stored hashes. + + +AFF uses OpenSSL for computing hash functions and ZLIB for compressing +image segments. + +================================================================ + AFF UTILITY PROGRAMS + +The AFF Library comes with the following utility programs: + +afcat - copies from the contents of an AFFILE to stdout. +afcompare - compares two AFF files or an AFF file and a raw file +afconvert - converts AFF->raw, raw->AFF, or AFF->AFF (or even raw->raw, if you want) + optionally recompressed files. +affix - Reports errors with AFF files and optioanlly fixes them. +afinfo - prints info about an AFF file from an examination of the segments +afstats - prints statistics about one or more AFF files +aftest - regression testing for AFF library +afxml - outputs an AFF file's metadata as XML + +aimage - Image a hard drive into AFF or raw format + + +================================================================ + AFF DETAILS + +AFF SEGMENT NAMES: + +The following AFF Segment Names have been defined in the initial +release: + +segsize - The size of each image segment, stored as a 32-bit value. + +imagesize - The total number of bytes in the file, stored as a 64-bit + value. + +md5 - The MD5 of the uncompressed image file, stored as + 128-bit value. + +sha1 - The SHA-1 of the uncompressed image file, stored as a + 160-bit value. + +badflag - A 512-bit value that is stored in the file to denote a bad + sector. This value typically consists of the string + "BAD SECTOR\000" followed by a timestamp and a block of + random data. + + +badsector - The total number of bad sectors in the image, stored as a + 32-bit number. + +seg0 - The contents of the first segment in the image file. + A flag of '1' stored in the segment argument indicates + that the segment was compressed with zlib. + +seg1 - The contents of the second segment in the image file + +segNNN - The contents of the NNNth segment of the image file. + +a.manufacturer - The manufacturer of the disk drive, stored as a UTF-8 + string. + +a.model - The model number of the disk drive, stored as a UTF-8 + string + +a.property - Any arbitrary "property" of the disk drive, stored as a + UTF-8 string. + +xxx - This segment should be ignored. (Space may be left for + future use.) + +"" - Segments with 0-length name are to be ignored and can + be garbage collected. + + +THE AFF SEGMENT FORMAT: + +Each AFF Segment contains the following information: + + - The Segment Header + - The Segment Data Payload + - The Segment Footer + +The Segment Header consists of the following + - A 4-byte Segment Header Flag ("AFF\000") + - The Length of the segment name (as an unsigned 4-byte value) + - The Length of the segment data payload + - The "argument", a 32-bit unsigned value + - The data segment name (stored as a Unicode UTF-8 string) + +The Segment Footer consists of: + - The 4-byte Segment Footer Flag ("ATT\000") + - The length of the entire segment, as a 32-bit unsigned value + +Because the segment length can be determined by reading both the +Header or the Footer, the AFF library can seek forwards or backwards +in the AFF file, similar to the way that a tape drive seeks forwards +and backwards through a tape drive. + +All 4-byte binary values are stored in network byte order to provide +for byte order independence. These values are automatically written +with the htonl() macro and read with the ntohl() macro by the AFF +Library. + + +AFF OPTIMIZATIONS: + +Although the AFF file format is quite simple, the library and +conversion routines implement a variety of optimizations to speed +conversion and reading. Among these optimizations are: + +* Image segments are only compressed in the AFF file if compression + would decrease the amount of data required by 5%. Otherwise no + compression is performed. As a result, images containing + uncompressable data are not compressed. This saves CPU time. + +* When an image is converted, space is left at the beginning of the + AFF file for the image hash and other metadata. As a result, this + information can be rapidly read when a new AFF image is opened. + +* AFF's af_read() routine caches the current image segment being read, + allowing for rapid seeking within the segment. And because all image + segments represent the same number of bytes in the original image + file, the library routine can rapidly locate the image segment that + corresponds to any byte offset within the original raw image, load + that image segment into memory, and return the sectors that are + requested. + + +LICENSE + +The AFF Library is distributed with a modified BSD license that allows +the use of AFF in any program, free or commercial, provided that the +copyright statement is included in both the source and binary file, +and in any advertisements for products based on the Garfinkel/Basis +AFF implementation. + +================================================================ +ACKNOWLEDGEMENTS + +Brian Carrier provided useful feedback on the initial AFF design. + +Jean-Francois Beckers has provided many suggestions for functionality +and has found many bugs in his extensive testing. + +================================================================ +BUILDING AFF + + export AFF_NOOPT=1 to disable compiler optimizations +================================================================ +AFF DIRECTORIES (AFD) + +Many file systems limit the maximum file size to 2^32-1 or +4,294,967,295 bytes. Unfortunately, even compressed images can easily +exceed this size. + +An AFF Directory (.affd) is a directory of AFF files that the AFF +Library knits together to give the appearance of a single AFF +file. No rules are imposed on the names of these files or the order of +the segments that they contain. If a segment with the same name exists +in more than one AFF file it is undefined which file's segment will be +returned if that segment is read. + +The AFFDIR is actually implemented using the AFF VNODE abstraction +(which allows arbitrary file system interfaces) and a special driver +that recursively opens multiple AFF sub-files. The current system +opens all of the files in the directory and keeps them all open. + + +The "aimage" analyzes the disk being imaged and the file system and +determines if the image can be stored in an AFF file or in an +AFFDIR. If it creates a dir, it automatically creates new AFF files +when the files being imaged exceeds 500MB. + +================================== +PyAFF - Python bindings for AFFLIB +================================== + +These bindings currently support a read-only file-like interface to AFFLIB and +basic metadata accessor functions. The binding is not currently complete. + +-------- +BUILDING +-------- + +./configure --enable-python + +----- +USAGE +----- + +Basic usage example: + + #/usr/bin/python + import pyaff + + fd = pyaff("diskimage.aff") + data = fd.read(1000) + fd.seek(0, SEEK_SET) + + print fd.get_seg_names() + print fd.get_seg("afflib_version") + + + +================ +Note: AFF and AFFLIB are trademarks of Simson L. Garfinkel and Basis +Technology, Inc. + + +# Local Variables: +# mode: auto-fill +# mode: flyspell +# End: diff --git a/doc/announce_1.7.txt b/doc/announce_1.7.txt new file mode 100644 index 0000000..5f7aa7e --- /dev/null +++ b/doc/announce_1.7.txt @@ -0,0 +1,126 @@ + ANNOUNCING AFFLIB 1.7 + +I'm happy to announce the release of AFFLIB 1.7. You can download it +from the standard location at http://www.afflib.org/ + +Version 1.7 represents a major advance over previous versions. +Upgrading is recommended for all users. + +Key improvements in Version 1.7 include: + + * Dramatically smaller image size in almost all cases. + + * Significantly improved performance + + * Improved tools! + + +DRAMATICALLY SMALLER IMAGES +--------------------------- + +AFFLIB has always supported pluggable compression algorithms, but previous +versions featured just a single compression algorithm --- zlib (the +same algorithm used in gzip). + +Version 1.7 introduces two new compressors: + + LZMA - This is the compressor developed by Igor Pavlov and + popularized by the 7-Zip compression system. LZMA uses + a very large dictionary to achive compression that is + 30-70% better, on average, than zlib. + + NULL - This compressor checks to see if a page is 16MB of NULLs. If it is, + this fact is simply noted. The result is considerable savings on many + drives, since many drives have large regions that are all NULLs. + + +The result of these two compression algorithms together is +significant. My library of images from 1000 disks purchased on the +secondary market now takes roughly 1/2 the space that it prevously +did. And it is faster to work with these highly-compressed images than +with raw images or impages compressed with zlib, since decompression +is I/O bound, not CPU-bound. + + + +SIGNIFICANTLY IMPROVED PERFORMANCE +---------------------------------- + +Forensic programs like The Sleuth Kit will run significantly faster +with Version 1.7 than with previous versions. Instead of a single page +cache, Version 1.7 can cache multiple pages. The number of pages +cached is determined by the environment variable AFFLIB_CACHE_PAGES; +the default value is 2 which gives good performance with TSK --- one +page for the inode table, one page for the data. + +Remember, each page is typically 16MB, so you can quickly overwhelm +your memory if you set the value too large. Don't page the cache out +to the hard drive --- it's faster to throw away the pages, re-read the +originally compressed page and decompress it, than it is to write out +the uncompressed page and read it back. + +One of the problems with using compressed images is that you can't +memory-map the file directly into the memory space of a 64-bit +processor. (You can't do it with a 32-bit processor, of course, if the +image is larger than 1GB or so.) Setting AFFLIB_CACHE_PAGES to a large +value gives you even better performance than memory-mapping a raw +file. + + + +IMPROVED TOOLS +-------------- + +The AFFLIB Tools have been expanded and refined. The toolset now +consists of 10 programs to assist in working with libraries of AFF +images. + +Current tools include: + +afcat - Outputs the contents of an AFF file stdout. + +afcompare - Compares two AFF files, or an AFF file and a non-AFF file. + Can also report on the success of "preening" + +afconvert - Converts a non-AFF file to an AFF file. + +afcopy - Makes a verified copy of an AFF file to another location. + Accepts multiple destinations and minimizes reads (ie: + reads each segment once and then does the copies.) Future + version might do writes simultaneously if they are to + different devices. + + -p option causes files to be recompressed with LZMA when + they are copied. This can be done without invalidating + digital signatures. + +affix - If an AFF file ends with an incomplete segment, this + program removes it. + +afinfo - Reports information about an AFF file. + +afsegment - Allows an individual segment to be read or + written. Typically used to allow shell-script access to metadata. + +afstats - Prints statistics from a single AFF file or multiple + files. Prints reports. + +afxml - Outputs the AFF medata in XML, for all of you XML fans. + + +and, of course: + +aimage - The advanced disk imager. Now images to LZMA (but it's slow...) + + + +WHAT'S NEXT? +------------ + +And here is what's on the agenda for the next major release of AFF: + + + Support for Amazon's S3. + + Labeling of "bad" and redacted sectors in the metadata (rather than + relying on high-entropy tokens in the data.) + + Support for NSRL bloom filters. + diff --git a/doc/announce_1.8.txt b/doc/announce_1.8.txt new file mode 100644 index 0000000..84798b8 --- /dev/null +++ b/doc/announce_1.8.txt @@ -0,0 +1,128 @@ + ANNOUNCING AFFLIB 1.8 + +I'm happy to announce the release of AFFLIB 1.8. You can download it +from the standard location at http://www.afflib.org/ + +Version 1.8 introduces support for Amazon's Simple Storage Service +(S3). Using S3 you can store an unlimited amount of information on S3 +and then process it using Amazon's Elastic Computing Cloud (EC2). +This makes it possible for researchers to work with extremely large +forensic data sets in an efficient and cost-effective manner. + +Amazon's S3 system allows you to store data for just 15 cents per +gigabyte per month. That's great; for $150 per month I can store a +terrabyte. It's true that I can buy a terrabyte hard drive for $700 +(or two 500 gigabyte hard drives for $300). The key difference between +Amazon's storage and my storage is that Amazon's is backed-up, +replicated, and available at high-speed from multiple locations around +the world. + +The problem with S3 is that it costs 20 cents per gigabyte to transfer +data in-or-out of Amazon. That's more than storing it! This turns out +to be not so much of a problem, though, because Amazon also rents CPUs +for 10 cents per hour on EC2 and there is no bandwidth charge to move +data between EC2 and S3. + +As a result, my typical workload looks like this: + + * Image disks to S3. + * Bring up a few dozen computers on EC2 to analyze the disks. + * Wipe the disks or store them for long-term. + + + +USING S3 FOR COMPUTER FORENSICS +================== + +All objects in Amazon S3 must fit in the same global namespace. The +namespace consists of a "bucket name" which must be unique, and an +"object name" which is pretty much anything you want. ("foo" is okay +here.) These get turns into URLs of the form +http://s3.amazonws.com/bucket/object-name using the REST API. (There +is also a CORBA API, but you don't want to use that because it +requires that every object be repeatedly encoded into Base64 and +decoded from Base64.) + +Buckets come on a first-come, first-serve basis. You can't +have "foo" because it's already been taken, but you could have a +bucket with the base64 coding of the MD5 of your social security +number, if you wanted to do that. It's even reasonably secure, because +bucket names are private --- nobody can list your bucket names. +It's possible to infer that your bucket name exists by doing an +exhaustive search, which is a problem for dictionary words but not for +base64 encodings of MD5 hashes. + +You can only have 100 buckets per account, so don't go crazy with +them. + +Access control in based on buckets. You can make your bucket readable +by up to 100 Amazon AWS IDs and read-write for another 100 IDs. If you +need more than that, deploy a server on EC2 and implement your own +access control policy using a database of your choosing. + +The easiest way to use S3 is to some something like MonkeyDrive, which +makes an S3 bucket look like a remote file system. Unfortunately, this won't +work for computer forensics for two reasons: + + 1. S3 likes to read and write entire objects at once. + 2. S3 has a maximum object size of 5GB, which MonkeyDrive lowers + to 2GB due to a bug in Amazon's load balancers. + +The AFFLIB S3 implementation gets around this by storing each AFF +segment inside its own object. Recall that AFF breaks a single disk +image into data "pages" and metadata, where each data page is +16MB. Because these pages are then compressed with zlib or LZMA, they +can be quite small. That's good when you are paying 15 cents per +gigabyte per month for storage. + +USING S3 with AFFLIB + +Using AFFLIB with S3 is really easy. AFF files stored on S3 are given +URLs where the service is "s3" and the hostname is actually the name +of the bucket in which the AFF file is stored. For example, if you +have an AFF file called myfile.aff stored in a bucket called +subjectfiles, the filename would be: + + s3://subjectfiles/myfile.aff + +(Behind the scenes the S3 implementation is mapping this to a whole +bunch of objects. For example, the segment "page1" maps to +http://s3.amazonaws.com/subjectfiles/myfile.aff/page1 . But that level +of detail shouldn't matter to most users of this system.) + +To actually use S3 and EC2 you need to sign up for an Amazon Web +Service's account, which you can do with a credit-card. There are no +start-up fees. + +1. Sign up for Amazon AWS and add S3 as a service. +2. Set the following environment variables: + +setenv AWS_ACCESS_KEY_ID " id" +setenv AWS_SECRET_ACCESS_KEY " + +3. You'll need a bucket to store your files in. There's a new AFF +utility called "s3" that controls the Amazon S3 system. Go ahead and +make a bucket: + + s3 mkdir + +for example, we could make a bucket called subjectfiles: + + s3 mkdir subjectfiles + +4. You can now use the afcopy program to copy an AFF file to this + bucket: + + afcopy myfile.aff s3://subjectfiles/myfile.aff + + +5. You can set the environment variable S3_DEFAULT_BUCKET If you don't + want to type "subjectfiles" all the time: + + setenv S3_DEFAULT_BUCKET subjectfiles + + Then you can use the URL: + + s3:///myfile.aff + + diff --git a/doc/announce_2.2.txt b/doc/announce_2.2.txt new file mode 100644 index 0000000..c5c651c --- /dev/null +++ b/doc/announce_2.2.txt @@ -0,0 +1,16 @@ + ANNOUNCING AFFLIB 2.2 + +I'm happy to announce the release of AFFLIB 2.2. You can download it +from the standard location at http://www.afflib.org/ + +Version 2.2 brings several important new features to AFFLIB: + + * AIMAGE now computes SHA256 when an entire image is acquired, + in addition to also calculating MD5 and SHA1. + + * Support for AFFLIB running under Windows has been greatly + expanded. Now AFFLIB compiles on both Cygwin and Microsoft VC++ + (version 2005). Extensive documentation is included in the Win32 + directory for both downloading and installing the (free) Microsoft + build tools, and for compiling AFFLIB. + diff --git a/doc/announce_3.0.txt b/doc/announce_3.0.txt new file mode 100644 index 0000000..a20cf61 --- /dev/null +++ b/doc/announce_3.0.txt @@ -0,0 +1,181 @@ + ANNOUNCING AFFLIB 3.0 + +I'm happy to announce the release of AFFLIB 3.0. You can download it +from the standard location at http://www.afflib.org/ + +Version 3.0 is a significant upgrade to AFFLIB which introduces the +following features: + + * STRONG ENCRYPTION FOR AFF FILES. + * STRONG DIGITAL SIGNATURES WITH X.509 CERTIFICATES + * SIGNED BILL-OF-MATERIALS AND CHAIN OF CUSTODY + * SIGNED ISO FILES + * PARITY PAGES ALLOW RECONSTRUCTION OF DAMAGED DISK IMAGES + +* STRONG ENCRYPTION FOR AFF FILES. + + With Version 3.0 we are introducing the ability to encrypt AFF + evidence files with the AES-256 algorithm, the strongest encryption + algorithm available today. + + Each AFF 3.0 file can be encrypted with a unique AES-256 key. This key is + can then itself be encrypted using a passphrase provided by the + user, or using an X.509 public key. Because of this two-step + process, the passphraseor public key can be changed in just a few + seconds without having to decrypt and re-encrypt the entire disk + image. + + Whereas some other forensic programs provide the ability to put a + "password" on an evidence file, those passwords can be disregarded + by non-conformant programs. (For example, GetData claims that it's + MountImage Pro program can "open EnCase password protected image + files without the password.) AFF 3.0 uses true encryption: if you + do not know the correct decryption key, the only way to access the + evidence is to brute-force the encryption passphrase (if there is + one). THERE IS NO BACK DOOR. + + +* STRONG DIGITAL SIGNATURES WITH X.509 CERTIFICATES + + Version 3.0 introduces strong digital signatures (SHA-256) signed + with X.509 certificates. + + Digital signatures represents a significant improvement for evidence + integrity over today's standard practice of recording the MD5 or + SHA-1 of an imaged disk in an investigator's notebook. + + AFF Digital Signatures, signatures are written for the entire disk + image, all of the disk's metadata, and every 16-megabyte AFF "page." + + Because digital signatures are written after each "page" is + acquired, the integrity of these pages can be established in court + even if the entire disk cannot be images (for example, because the + device is fault, or because there is insufficient time). + + AFF Digital Signatures complement existing integrity + measures. Because the signature is stored in its own metadata + segment, the signature does not change the content of the acquired + disk image. + + Signatures can be written with either self-signed certificates or + with X.509 certificates that are issued as part of an organization's + PKI. Using X.509 certificates means that AFF can support RSA or DSA + algorithms with 1024, 2048 or larger keys. + + +* SIGNED BILL-OF-MATERIALS AND CHAIN OF CUSTODY + + Version 3.0 introduces a special XML structure that contains a list + of every AFF segment in the file, a signature for each segment, a + set of "notes," and a public key. This structure is called an "AFF + Bill Of Materials" (AFFBOM). + + When an AFF image is created with AIMAGE, the AFFBOM is created and + signed with the private key belonging to the person who did the + acquisiton. Thereafter, each time a signed AFF file is copied, a new + AFFBOM can be created which includes a new AFFBOM which covers all + of the original segments and all of the previous AFFBOMs. In this + manner the sequence of signed bill-of-materials becomes a custody + chain, showing who has copied the image and verifying that no + evidentuary segments have been added, deleted, or modified. + + +* SIGNED ISO FILES + + AFF's "AFM" format allows a disk image to be stored in an uncompressed + raw file (eg "file.iso") and the associated metadata to be stored in a + ".afm" file. The AFM format can also handle raw data stored as a + series of "split" raw files (eg "file.001", "file.002", "file.003" + etc.) + + Beacuse AFF tools operating on named segments that are independent + of the underlying storage container, the AFM format allows any + ISO-file to be signed using the "afsign" command. When filename.iso + is signed, the afsign create a new file called filename.afm which + contains the signatures, the signed bill of materials, and other + metadata. + + Although it is also possible sign ISO files using existing tools + such as PGP with detached signatures, afsign has several advantages: + + - afsign will sign every 16-megabytes chunk of the ISO file. In this + way, if the file is corrupted, you will be able to pinpoint what + data is invalid and what data is still good. + + - Unlike PGP, afsign allows you to add arbitrary metadata and + maintain chain-of-custody information. + + - You can sign with X.509 certificates + + +* PARITY PAGES ALLOW RECONSTRUCTION OF DAMAGED DISK IMAGES + + Because every 16-megabyte chunk of an AFF or AFM file is signed, + it is easy to detect when a page has been modified or accidently + corrupted. The BoM allows missing pages to be detected. + + Similar to RAID5 on hard drives, an AFF parity page makes possible + to reconstruct damaged or missing AFF data segments. Once repaired + or reconstructed, the signature (which is stored in a differnet + location) can be used to determine if the reconstruction is correct. + + Partiy Pages are automatically created when an image is signed with + afsign. The rewritten aimage that will be part of AFFLIB 3.1 will + create parity pages as the drive is imaged. + +================================================================ +AVAILABILITY + +AFFLIB 3.0.0 is available now. + + +================================================================ +NEW AND MODIFIED TOOLS IN AFF 3.0: + +The following tools have been aded for AFF 3.0: + +* afsign - signs an AFF file. + +* afverify - verifies the signature and chain-of-custody segments of + an AFF file. + +* afcrypto - manipulates the cryptographic properties of an AFF file. + - Can change the passphrase + +The following tools have been modified: + +* afcopy - If you provide a signing key, a signed bill-of-materials + will be added to extend the chain-of-custody. + + +Other changes in AFF3.0: + +* A few bugs have been fixed which caused difficulties in testing. (No + users reported problems with them.) + +================================================================ +COMING IN AFFLIB 3.1: + +I've pushed out Version 3.0 so that people can start to experiment +with it now. Meanwhile, I'm now working on the following features +which, I'm hoping, will make it into Version 3.1: + +* Public key encryption (so agents in the field can encrypt to a + public key, and the images can only be decrypted in the lab.) + +* Dramatically improved performance when opening AFF files with signed + bill-of-materials. (The BOM will be used as a table-of-contents so + that large AFF files do not need to be scanned from end-to-end.) + +* A rewrite of aimage: + + - The ability to image raw and AFF files at the same time will be + removed (since AFF can now write raw files directly). + + - page-at-a-time imaging, resulting in more compact AFF files (less + wasted space) and easier implementation of novel data recovery + algorithms. + + - Calculation of parity pages while the image is written, rather + than afterwards. + diff --git a/doc/announce_3.3.5.txt b/doc/announce_3.3.5.txt new file mode 100644 index 0000000..e6c19a7 --- /dev/null +++ b/doc/announce_3.3.5.txt @@ -0,0 +1,38 @@ +Announcing AFFLIB version 3.5.5. + +AFFLIB 3.5.5 is a robust forensic file format for storing computer forensic information. AFFLIB offers: + * Transparent access to AFF, EnCase (E01), Raw, and split-raw disk images. + * Transparent integration to Amazon's S3 storage system. + * True encryption of disk images, using either pass phrases or PKI. + * True digital signatures of disk images. + * Support for MD5, SHA-1, and SHA-256 + * Support for recovery through the use of parity files. + +AFFLIB 3 is in maintenance phase while AFFLIB 4.0 is developed and integrated into our forensic tools. This means that AFFLIB will not have any new features introduced, but we will continue to fix bugs and improve performance. + +AFFLIB 3.5.5 has been tested on: + * MacOS 10.6 + * Debian Linux + * Fedora Core 12 Linux + +A windows version is also available. + +AFFLIB 3.5.5 can be downloaded from http://afflib.org/ + +=========== +We have also created a new release of the following tools that you may find interesting: + +* bulk_extractor, a tool that will find email addresses and URLs in disk images. This program is now written in both Java and C++ (two versions, twice the fun) + +* frag_find, a tool that will take a set of TARGET files and a disk image and tell you if the files, or pieces of the files, are on the disk. It does this by comparing every sector of every file to every sector of the disk. Ideal for finding exfiltrated corporate documents. + +* fiwalk, a tool that turns the metadata information on a disk image into a block of XML, and a python module that makes it easy to do disk processing. See http://simson.net/xml_forensics.pdf for details. + +* ATA Raw, a system for constructing ATA commands at the user level and sending them to an ATA device. Ideal for testing write blockers. + +Details on all of this and more can be found at our new website: + https://domex.nps.edu/deep/ + + + + diff --git a/doc/announce_3.3.txt b/doc/announce_3.3.txt new file mode 100644 index 0000000..c400792 --- /dev/null +++ b/doc/announce_3.3.txt @@ -0,0 +1,53 @@ + ANNOUNCING AFFLIB 3.3 + +I'm happy to announce the release of AFFLIB 3.3. You can download it +from the standard location at http://www.afflib.org/ + +Version 3.0 is a significant upgrade to AFFLIB which introduces the +following features: + + * SUPPORT FOR VMWARE, PARALLELS, APPLE DMG DISK IMAGES + + * SIGNIFICANTLY IMPROVED PERFORMANCE + +================================================================ +SUPPORT FOR VMWARE DISK IMAGES + +AFFLIB 3.3 incorporates the disk image subsystem from the open-source +QEMU processor virtualization project. Although most of QEMU is +distributed under the GPL license, the disk subsystem is distributed +under a less restrictive license that allows any use. + +As a result, forensic programs linked with AFFLIB can now +transparently access disk images stored in any of the following +formats; currently the format is specified with the indicated extension. + +So far we have only tested with VMWare .vmdk images: + + * VMWare VMDK (.vmdk) (tested) + +We also have the ability to add additional file types, including: + * Bochs Virtual HD Image + * cloop + * cow + * DMG + * qcow + * qcow2 + * VFAT + * Parallels + * Connectix Virtual PC + +Support for these will be enabled if requested. +================================================================ +SIGNIFICANTLY IMPROVED PERFORMANCE + +Now that the feature set for AFFLIB is largely complete, we are +beginning to pay attention to performance issues. In particular, we +found a significant problem in versions 3.0 through 3.2 that would +significantly degrade performance of disk images larger than 1GB. This +has now been addressed. + +================================================================ + + + diff --git a/doc/announce_3.7.txt b/doc/announce_3.7.txt new file mode 100644 index 0000000..34d6e63 --- /dev/null +++ b/doc/announce_3.7.txt @@ -0,0 +1,67 @@ + ANNOUNCING AFFLIB 3.7 + +I'm happy to announce the release of AFFLIB 3.7. Significant +highlights of the release include the following: + +- Copyright Clarification. AFFLIB was originally a collaboration + between Simson L. Garfinkel and Basis Technology and under the terms + of that collaboration the copyright was jointly held. The original + library was licensed under the 4-part Berkeley license. + + Work done on AFFLIB by Simson Garfinkel after December 26, 2006 was + done within the context of my employment with the US Government and, + as such, is not subject to copyright. + + All of the files in the current release have been examined to + determine whether they are licensed under the four-part license or + if they are in the public domain. + +- Move to github.org. As I am now largely working on other projects, + I am moving AFFLIB from afflib.org to github.org. This will make it + easier for others to perform maintenance on this release as + necessary for the eventual transition to AFFv4. + +There are no other significant changes in this release from release +3.6.15. + + + +RELEASE ENGINEERING + +Although it is possible to do releases on github using the git tag[1] +interface, I will still be making digitally signed releases and +putting them in the github downloads[2] section. + +[1] http://gitref.org/branching/#tag +[2] https://github.com/blog/742-new-uploader-downloads-screen + + + +USING GIT + +The move to git makes it easier for potential AFFLIB developers and +users to directly access the AFFLIB development tree. For those of you +who are new to git, here are some simple commands that you can use: + +You can check out a copy of the source code with: + + $ git clone git://github.com/simsong/AFFLIBv3.git + +You can then keep your copy of AFFLIB up-to-date with: + + $ get pull + +If you are working in the repository, you will need to make your own +configure script. Do that with these commands: + + $ sh bootstrap.sh + $ ./configure + $ make + + +Other kit commands that may be useful: + + $ git add [-f] filename (adds a file) + $ git commit (commits to local repo) + $ git push (push back to clone) + diff --git a/doc/crypto_api.txt b/doc/crypto_api.txt new file mode 100644 index 0000000..a9634de --- /dev/null +++ b/doc/crypto_api.txt @@ -0,0 +1,59 @@ +# This file is a work of a US government employee and as such is in the Public domain. +# Simson L. Garfinkel, March 12, 2012 + +IMPLEMENTATION +============== + +AFFLIB encryption will continue to use the cryptographic primitives +provided by the OpenSSL library. + +The AFFILE Structure will be modified to include these additional fields: + AES_KEY ekey - The OpenSSL AES256 encryption key + AES_KEY dkey - The OpenSSL AES256 decryption key + + +Reading: + +Getting pages is currently implemented with a chain of functions: + + af_get_page() - gets the page and decompresses it if necessary. + af_get_page_raw() - gets raw pages (without compression) + af_get_seg() - gets the actual segment + + +Proposed modification: + + If af_get_seg(s1) fails AND if a symmetric encryption key has been + set, the function will then look for s1/aes. If this is found the + segment will be decrypted and returned. + +Writing: + + +Currently pages are written with these functions: + + af_update_page(af,pagenum,data,datalen) + af_update_seg() + +Procedure for writing encrypted pages: + + - Modify af_update_page() to call a new function, + af_update_page_raw(), which does the encryption. + + +Other work that needs to be done: + + - Make sure that pages are only written with this function. In + particular, check out afconvert, aimage, and atest + + +================================================================ +Performance Notes: + +When reading encrypted AFF files, specify read buffers that are at +least 16 bytes larger than you expect. This gives the internal +routines space to do the decryption in place. Otherwise additional +memory needs to be allocated and data needs to be copied. + + +================================================================ diff --git a/doc/crypto_design.txt b/doc/crypto_design.txt new file mode 100644 index 0000000..74a4502 --- /dev/null +++ b/doc/crypto_design.txt @@ -0,0 +1,282 @@ +# This file is a work of a US government employee and as such is in the Public domain. +# Simson L. Garfinkel, March 12, 2012 + +Design for the encryption system: + +Encryption on AFF will be implemented by AFF Base Encryption +Services. On top of the Base Encryption may be layered either +Passphrase Encryption or Public Key Encryption. + +AFF Base Encryption: +--------------------- +Currently we'll be doing this with AES-256, but the system can be +evolved to accommodate other encryption schemes as needed. + +Today AFF data pages are stored in segments named page%d --- page0, page1, etc. +The flag indicates if compression is used or not. + +Encrypted pages will be stored in segments named page%d/aes --- ie, +page0/aes, page1/aes, etc. + +Restrictions: + +* A single "affkey" is used to encrypt every page. +* The AES-256 key cannot be changed. + +Encryption will be done with AES256 in CBC mode. +The IV is the name of the sector, padded with NULs. + +AES256 requires that all buffers be padded to the AES block size, +which is 16 bytes. For performance we don't want to add padding if +the page is already a multiple of the bock size, so here is the +algorithm: + +* If len%16==0, do not pad +* If len%16!=0, let + extra = len%16 + pad = 16-extra + Append pad NUL bytes + Encrypt + Append extra NUL bytes. + Write + +Now, when segment is read: + extra = len%16 + pad = 16-extra + +* extra==0, it wasn't padded +* Otherwise + Remove extra NUL bytes + Decrypt + Remove pad NUL bytes + +In this way, the length does not need to be explicitly coded. + +On decryption, the key can be "validated" by attempting to decrypt +page0/aes and seeing if page0_md5 matches (because that's the MD5 +for the unencrypted, uncompressed page.) A new API call will be +created for this purpose. + +If a key is set, then pages that are written are automatically encrypted first. + +If both an encrypted page and an unencrypted page are present in the +file, the unencrypted page is returned (because the software never +looks for the encrypted page.) + +If an unencrypted page is updated and encryption is turned on, the +encrypted page is first written, then the unencrypted page is deleted. + +It is an error to change the affkey encryption key once it has been set. + + + +Advantages: +* Simple to implement & test. +* It's real encryption, not a "password" like E01 format uses. +* Works transparently with S3 implementation. +* Allows an unencrypted file to be encrypted in-place. +* We can push this down into a lower layer to provide for encryption + of all metadata, although that won't be done in the initial + implementation. + +Disadvantages: +* Only encrypts the page data, not the metadata, in the initial implementation. +* Only way to change the key is to copy to a new AFF file. +* Encryption key is cached in memory in the AF structure. + + +Proposed API: + af_set_aes_key(af,key,keysize) - sets the key; use alg=0 to turn off encryption. + - key is unsigned char. + - keysize is in bits. + af_validate_key(af) - returns 0 if the key that was set can be used + to validate a page + af_validate_key_page(af,pagenum) - Specifically checks to see if pagenum + can be validated with the key that was set. + returns 0 - validates, -1 = does't validate; -2 = page doesn't + exist; -3 = page md5 doesn't exist. + + +AFF Passphrase Encryption +-------------------------- +This approach builds upon the Base Encryption, but allows the user to +store a passphrase. Instead of using SHA256 to generate the encryption +key directly, the encryption key is a random 256 bit string. This +string is then encrypted with the passphrase and stored in the AFF +file. + +The scheme could easily support multiple passphrases on each file, +although that may not be useful. + +The encrypted encryption key is stored in a new segment: affkey-aes256 + +The contents of affkey_aes256 a 68 byte structure: + bytes 0-3 - Version number. This is version 1. Stored in network byte order. + bytes 4-67 - The affkey, encrypted with AES in codebook mode + using SHA-256 of the passphrase as the encryption key. + bytes 68-131 - the SHA-256 of the affkey (so you know when you got it). + +With this scheme the passphrase can be changed without requiring the +entire disk image to be re-encrypted---just rewrite affkey-aes256 +with a new password. + +Advantages: +* Easy to change the key +* The passphrase is not cached in memory. + +Disadvantages: +* If you can encrypt, you can decrypt (it's a passphrase). + + +Proposed API: +af_use_passphrase(af,char *phrase) + - Tries to use an existing passphrase from an AES-encrypted AFFILE + - errors if there is no AES-encrypted data to decrypt of if passphrase is wrong. + +af_establish_passphrase(af,char *phrase) + - If no encryption has been used yet, makes a random key and + stores it encrypted with the passphrase. + - fails if encryption has been used + +af_establish_passphrase_key(af,char *passphrase,char *key,int keylen) + - Verifies that the key is good (by decrypting existing encrypted data) + +af_change_passphrase(af,char *oldphrase,char *newphrase) + - Validates that oldphrase is correct, then changes it to new phrase. + + +Signing AFF files with X.509 certificates +-------------------------- +This approach is similar to AFF Passphrase Encryption, except that the +instead of encrypting the affkey with a passphrase, we encrypt it a +an X.509 certificate and its matching private key. + +The easiest way to get a private key and a corresponding X.509 +certificate is to make a self-signed certificate using the openssl command: + +It can also use self-signed certificates: + openssl req -x509 -newkey rsa:1024 -keyout sign.key -out sign.key -nodes + +This command will ask you a bunch of questions; the results are stored +in the file sign.crt. When you create a signed AFF file the +certificate will be stored in the file, so be careful what you +say. Alternatively, you can create an RSA private/public key pair, +create a certificate request (CSR), send the CSR to a certificate +authority, and use the certificate that the authority sends you back. + +Note that this puts both the key and the self-signed certificate in +the same file. That's fine for our purposes. + +You can view the contents the certificate with this openssl command: + + openssl x509 -text -in sign.key + +Each segment is signed with the X509 private key by the AFF library +when the segment is written. Two signature modes are support: + +Mode 0 : RAW SEGMENT SIGNATURE. + The signature is computed by calculating + the SHA256 hash of the segment name, a NULL byte, the segment argument (a 32-bit + number) in network byte order, and the segment data. + +Mode 1 : DATA PAGE SIGNATURE + The signature is computed by calculating + the SHA256 hash of the segment name, five NULL bytes, and the + page data. + +Mode 1 is used for signing user data acquired from the hard drive; it +intentionally signs uncompressed data. Mode 0 is used for signing all +other data, including metadata returned from the drive, examiner +notes, and so on. + +The signatures are written into segments themselves, with the segment +name being "name/sha256" where "name" is the original segment +name. The argument of the segment is the signature mode. + +Notice that AFF signatures are independent of the underlying storage +system. The signatures can be stored in one file and the data in +another file (as in an AFM file), or in multiple AFF files (as in an +AFD directory). They can even be stored in a network-based object +storage system (like S3). + +If not private key exists and a data segment is written AFFLIB will +automatically compute the MD5 of the uncompressed data page and write +it to the file. This isn't done if a private key exist. + +Right now the primary limitation is that the signature does not +indicate which private key was used to create it. This isn't so much a +problem, though, because we only support a single signing key. It is +an issue for chain of custody blocks + +AFF Chain of Custody Block. + +When an AFF file is created or copied, an AFF Chain of Custody Block +can be added. This block can be thought of as a signed table of +contents, although it isn't strictly a TOC because the CCB doesn't +indicate the position of each signed segment within the file. + +The AFF Chain of Custody Block is an XML block. Right now the block is +just written into a segment and that signment is signed using the +standard segment signing appraoch. Eventually the segment may be +signed using XML signatures. + +XML elements: + + + - Date that this is being written + mycert - base64 encoding of X.509 certificate + + + - segments that are present in AFF file at time it was received + signature in base64 coding + ... + + + +Right now this is signed with a base-64 signature following the +chain. Eventually we may move to XML signatures. + +================================================================ + +Encrypting AFF files with X.509 certificates +------------------------------------------- + +The public key is specified when the file is created. After the file +is created, it can only be accessed using the corresponding private +key. This includes all access--both reading and writing. + +For encrypting the private key is stored in the +environment variable AFFLIB_ENCRYPTING_PUBLIC_KEY or in a filename +referenced by the variable AFFLIB_ENCRYPTING_PUBLIC_KEYFILE + +For decrypting the private key is stored in the +environment variable AFFLIB_DECRYPTING_PRIVATE_KEY or in a filename +referenced by the variable AFFLIB_DECRYPTING_PRIVATE_KEYFILE + +Public key encryption is implemented by taking the affkey and storing +it in a segment called "affkey-rsannn" where nnn runs from 0 to whatever. +Padding is with PKCS1. + +Advantages: + * Easy to implement with existing cryptographic tools. + * Can encrypt to multiple keys + +================================================================ + +AFF Public Key Signatures: +-------------------------- +This approach uses a private key to sign each segment when it is written. +In this case, a private key is used for signing and the public key is +used for verifying signatures. The public key will be stored in the +image itself; its fingerprint can be recorded elsewhere. + +Once again, filenames are specified in environment variables: + + AFFLIB_SIGNING_PRIVATE_KEY + AFFLIB_SIGNING_PUBLIC_KEY + + +================================================================ + + +================================================================ diff --git a/doc/crypto_doc.txt b/doc/crypto_doc.txt new file mode 100644 index 0000000..055683a --- /dev/null +++ b/doc/crypto_doc.txt @@ -0,0 +1,168 @@ +# This file is a work of a US government employee and as such is in the Public domain. +# Simson L. Garfinkel, March 12, 2012 + +AFF Encryption +============= + +Release 2.4 of AFFLIB implements AFF pass-phrase encryption. + +Encryption is based on a 256-bit randomly-generated AES key (called +the AFF key). This key is itself encrypted with an AFF passphrase and +stored in its own segment. This strategy allows an AFF image +encryption passphrase to be changed without re-encrypting the entire +disk image. + +AFF PASSPHRASE ENCRYPTION +========================= + +The AFF passphrase may be specified either as part of the filename or, +in some cases, as an optional argument for some of the AFF commands. +AFF uses RFC 1630 URI syntax to specify encryption +passphrases. Specifically, RFC 1630 allows the file myfile.aff to be +specified as a URI: + + afinfo file:///myfile.aff + +The passphrase 'mypassphrase' can be added to this URL: + + afinfo file://:mypassphrase@/myfile.aff + +If you wish to refer to myfile.aff in the root directory, use this +syntax: + + afinfo file:////myfile.aff + +Because windows interperts the forward and back slashes in the same +manner, this will refer to the file c:\myfile.aff + + afinfo file:///c:/myfile.aff + +You can also save the passphrase in an environment variable called AFFPASSPHRASE: + + setenv AFFLIB_PASSPHRASE "mypassphrase" (csh) + export AFFLIB_PASSPHRASE="mypassphrase" (bash) + set AFFLIB_PASSPHRASE="mypassphrase" (windows) + afinfo myfile.aff + +You can store the passphrase in a file and specify that file with the AFFLIB_PASSPHRASE_FILE variable. + setenv AFFLIB_PASSPHRASE_FILE "/tmp/myfile" + echo "mypassphrase" > /tmp/myfile + afinfo myfile.aff + + +A passphrase can also be read from a file descriptor by putting the file descriptor number in +the environment variable AFFLIB_PASSPHRASE_FD: + setenv AFFLIB_PASSPHRASE_FD "5" + echo "mypassphrase" > /tmp/myfile + afinfo myfile.aff 5maxsize (also ap->sr->maxsize). + + This value is determined by multiplying the AF_PAGESIZE (which defaults to + AFF_DEFAULT_PAGESIZ, currently 1024*1024*16) by AF_PAGES_PER_RAW_IMAGE_FILE. + + By default, AF_PAGES_PER_RAW_IMAGE_FILE is not set. This means that, by default, + AFM files are not split but consist of a single raw file and a single AFM file + (which holds the metadata for the raw file). + + You can set AF_PAGES_PER_RAW image like this: + + int64_t pages_per_file = 128; + af_update_segq (af, AF_PAGES_PER_RAW_IMAGE_FILE, pages_per_file); + + AFM files that are created by aimage do have this value set, + however: it is set to 2^31. + + + diff --git a/doc/threading.txt b/doc/threading.txt new file mode 100644 index 0000000..2ea6f7f --- /dev/null +++ b/doc/threading.txt @@ -0,0 +1,23 @@ +# This file is a work of a US government employee and as such is in the Public domain. +# Simson L. Garfinkel, March 12, 2012 + +Plans for making AFFLIB multi-threaded: + +Low hanging fruit: +* Compress in background +* Calculate hashes in background + +Easy implementation strategy: +* mutext for: + - the entire cache + - each page of the cache + - the TOC (should this be re-implemented as a C++ vector?) + +Both compressing and hashing needs access to the page cache in another thread: + - Needs to lock the page cache so no other process will write to it. + - Or needs to make a copy of it. + - Then needs to write it out. + + +Test program: + - Can we write a method that computes hash in another thread? diff --git a/doc/upload.txt b/doc/upload.txt new file mode 100644 index 0000000..84df50e --- /dev/null +++ b/doc/upload.txt @@ -0,0 +1,77 @@ +# This file is a work of a US government employee and as such is in the Public domain. +# Simson L. Garfinkel, March 12, 2012 + +Upload protocol. + +Goals: +1. Anybody who is approved can upload. +2. Omipodent. +3. Stateless server. +4. Automatic integration into the library. + + +Database change: + * Add nonce to drives table. + * Add a new table for uploaders + uploaders + +uploaders: + * ID - my ID (8 digits hex) + * Country - my ISO Country Code + * Sequence - Sequence number for this uploader + +RPC command ping: + * args: a dictionary with + * uid:uploaderID + * Returns OK if this user is authorized + * Raises exception if user is not. + +RPC check command: + * args: a dictionary with + * image_gid_hex:image_gid + * uid:uploaderID + * seg_name: segment name being uploaded + * Reports if the segment has been uploaded or not. + +RPC upload command: + * args: check dictionary with additions: + * seg_arg: argument + * seg_len: length + * seg_md5: md5 of data + * seg_data: the data + * Calls check. If we can upload, does upload + +RPC command: seglist + * args: dictionary with + * runs ping + +Upload returns: + * OK + * HAVE (already have) + * FAIL + * Permission Denied (GID already exists with different random number) + +Algorithm on upload: + * Check uploader id; if it is not valid, return + * Get country & sequence numbers from ID + * + * If image_gid is not in the database + - add it to the database + - add the nonce to the database + - allocate a new AFF filename given the appropriate country code. + + * If image_gid is in the database + - If nonce does not matche the nonce in the database, generate an error + + * If command is seglist: + - return a list of the segments in the file + + * If command is check: + - return TRUE if the segent is in the file, otherwise return FALSE + + * If the command is upload: + - add the segment to the file + + * server maps image_gid to a filename. Server remembers random number. + * If random number is different, no more upload is provided. + diff --git a/lib/Makefile.am b/lib/Makefile.am new file mode 100644 index 0000000..84d7233 --- /dev/null +++ b/lib/Makefile.am @@ -0,0 +1,115 @@ +EXTRA_PROGRAMS = aftest +TESTS = aftest + +CLEANFILES = aftest + +aftest_SOURCES = aftest.cpp +aftest_LDADD = libafflib.la +aftest_LDFLAGS = $(PTHREAD_CFLAGS) -static # easier debugging + +AFFLIB_SOURCES = aff_db.cpp aff_db.h aff_toc.cpp \ + afflib.cpp afflib.h afflib_i.h afflib_os.cpp \ + afflib_pages.cpp afflib_stream.cpp afflib_util.cpp \ + crypto.cpp \ + base64.cpp base64.h lzma_glue.cpp s3_glue.h \ + vnode_aff.cpp vnode_aff.h \ + vnode_afd.cpp vnode_afd.h \ + vnode_afm.cpp vnode_afm.h \ + vnode_raw.cpp vnode_raw.h \ + vnode_split_raw.cpp vnode_split_raw.h \ + aftimer.h \ + utils.cpp utils.h display.cpp + +libafflib_la_SOURCES = $(AFFLIB_SOURCES) + +QEMU_SOURCES = \ + vnode_qemu.cpp \ + vnode_qemu.h \ + qemu/aes.h \ + qemu/aff-block-vmdk.h \ + qemu/block-bochs.c \ + qemu/block-cloop.c \ + qemu/block-cow.c \ + qemu/block-dmg.c \ + qemu/block-parallels.c \ + qemu/block-qcow.c \ + qemu/block-qcow2.c \ + qemu/block-raw-posix.c \ + qemu/block-vmdk.c \ + qemu/block-vpc.c \ + qemu/block-vvfat.c \ + qemu/block.c \ + qemu/block.h \ + qemu/block_int.h \ + qemu/bswap.h \ + qemu/config-host.h \ + qemu/console.h \ + qemu/exec-all.h \ + qemu/osdep.h \ + qemu/qemu-common.h \ + qemu/qemu-timer.h \ + qemu/qemu_glue.c + +if MAYBE_QEMU + libafflib_la_SOURCES += $(QEMU_SOURCES) + libafflib_la_CFLAGS = -DQEMU_IMG + aftest_CFLAGS = -DQEMU_IMG +endif + +if MAYBE_S3 + EXTRA_PROGRAMS += s3 + libafflib_la_SOURCES += vnode_s3.cpp vnode_s3.h s3_glue.cpp + s3_SOURCES = s3.cpp + s3_LDADD = libafflib.la + s3_LDFLAGS = $(PTHREAD_CFLAGS) +endif + + +install-exec-hook: + @if [ -r $(DESTDIR)/etc/ld.so.conf ] ; then \ + echo "*************************************************" ;\ + echo "*** Checking shared library config for $(libdir) " ;\ + if grep ^$(libdir) $(DESTDIR)/etc/ld.so.conf >/dev/null ; \ + then echo $(libdir) already installed ; \ + else echo installing $(libdir) in $(DESTDIR)/etc/ld.so.conf ; \ + echo $(libdir) >> $(DESTDIR)/etc/ld.so.conf ; \ + PATH=$(PATH):/sbin; \ + ldconfig; \ + fi ; \ + echo "*************************************************" ;\ + fi + +EXTRA_DIST = + +INCLUDES = \ + -I@top_srcdir@/lzma443/C \ + -I@top_srcdir@/lzma443/C/7zip/Compress/LZMA_Alone + +LZMA_SOURCES = \ + @top_srcdir@/lzma443/C/7zip/Compress/LZMA_Alone/LzmaBench.cpp \ + @top_srcdir@/lzma443/C/7zip/Compress/LZMA_Alone/LzmaRam.cpp \ + @top_srcdir@/lzma443/C/7zip/Compress/LZMA_Alone/LzmaRamDecode.c \ + @top_srcdir@/lzma443/C/7zip/Compress/LZMA_C/LzmaDecode.c \ + @top_srcdir@/lzma443/C/7zip/Compress/Branch/BranchX86.c \ + @top_srcdir@/lzma443/C/7zip/Compress/LZMA/LZMADecoder.cpp \ + @top_srcdir@/lzma443/C/7zip/Compress/LZMA/LZMAEncoder.cpp \ + @top_srcdir@/lzma443/C/7zip/Compress/LZ/LZInWindow.cpp \ + @top_srcdir@/lzma443/C/7zip/Compress/LZ/LZOutWindow.cpp \ + @top_srcdir@/lzma443/C/7zip/Compress/RangeCoder/RangeCoderBit.cpp \ + @top_srcdir@/lzma443/C/7zip/Common/InBuffer.cpp \ + @top_srcdir@/lzma443/C/7zip/Common/OutBuffer.cpp \ + @top_srcdir@/lzma443/C/7zip/Common/StreamUtils.cpp \ + @top_srcdir@/lzma443/C/Common/Alloc.cpp \ + @top_srcdir@/lzma443/C/Common/CommandLineParser.cpp \ + @top_srcdir@/lzma443/C/Common/CRC.cpp \ + @top_srcdir@/lzma443/C/Common/String.cpp \ + @top_srcdir@/lzma443/C/Common/StringConvert.cpp \ + @top_srcdir@/lzma443/C/Common/StringToInt.cpp \ + @top_srcdir@/lzma443/C/Common/Vector.cpp + + +# specify lib_ to force tools to be dynamically linked against the installed library. +# specify noinst_LTLIBRARIES to have the tools be statically linked +# Unfortunately, when we specify lib_, things don't build. I can't figure it out. +lib_LTLIBRARIES = libafflib.la +libafflib_la_SOURCES += $(LZMA_SOURCES) diff --git a/lib/aff_db.cpp b/lib/aff_db.cpp new file mode 100644 index 0000000..0279b0e --- /dev/null +++ b/lib/aff_db.cpp @@ -0,0 +1,186 @@ +/* Distributed under the 4-part Berkeley License */ + +/* + * afflib_db.cpp: + * + * Functions for the manipulation of the AFF database. + */ + + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" +#include "aff_db.h" + + +/**************************************************************** + *** Low-level functions + ****************************************************************/ + +/**************************************************************** + *** Probe Functions + ****************************************************************/ + +int af_probe_next_seg(AFFILE *af, + char *segname, + size_t segname_len, + uint32_t *arg_, // optional arg + size_t *datasize_, // optional get datasize + size_t *segsize_, // optional get size of entire segment + int do_rewind) // optional rewind af->aseg, otherwise leave at start of segment data +{ + if(!af->aseg)(*af->error_reporter)("af_probe_next_segment only works with aff files"); + + struct af_segment_head segh; + memset(&segh,0,sizeof(segh)); + + uint64_t start = ftello(af->aseg); +#ifdef __BORLANDC__ + fseeko(af->aseg, start, SEEK_SET) ; // Windows is dumb :-( +#endif + + if(fread(&segh,sizeof(segh),1,af->aseg)!=1){ + return AF_ERROR_EOF; + } + if(strcmp(segh.magic,AF_SEGHEAD)!=0){ + snprintf(af->error_str,sizeof(af->error_str),"afflib: segh is corrupt at %" PRIu64,start); + return AF_ERROR_SEGH; + } + + uint32_t name_len = ntohl(segh.name_len); + uint32_t datasize = ntohl(segh.data_len); + if(name_len>AF_MAX_NAME_LEN){ + snprintf(af->error_str,sizeof(af->error_str),"afflib: name_len=%"PRIu32" (an outrageous value)",name_len); + return AF_ERROR_NAME; + } + + if(name_len+1 > segname_len){ + fseeko(af->aseg,start,SEEK_SET); // rewind to start + return -2; + } + + if(fread(segname,1,name_len,af->aseg)!=name_len){ + fseeko(af->aseg,start,SEEK_SET); // rewind + return -1; + } + segname[name_len] = 0; + + if(do_rewind) fseeko(af->aseg,start,SEEK_SET); // rewind + + uint32_t segsize = sizeof(struct af_segment_head) + + sizeof(struct af_segment_tail) + + name_len + datasize; + + if(arg_) *arg_ = ntohl(segh.flag); + if(datasize_) *datasize_ = datasize; + if(segsize_) *segsize_ = segsize; + +#ifdef DEBUG + fprintf(stderr,"af_probe_next_seg(segname=%s datasize=%d segsize=%d) do_rewind=%d\n", + segname,datasize,segsize,do_rewind); +#endif + return 0; +} + +/* af_backspace: + * moves back one segment in the aff file. + * Returns 0 if success, -1 if we can't back up further. + */ +int af_backspace(AFFILE *af) +{ + struct af_segment_tail segt; + + uint64_t start = ftello(af->aseg); + + if(start==sizeof(AF_HEADER) || startaseg,pos_tail,SEEK_SET); + if(fread(&segt,sizeof(segt),1,af->aseg)!=1){ + fseeko(af->aseg,start,SEEK_SET); // put it back + return -1; // can't read segt? + } + /* Verify that this is a segment tail. if it isn't, put the file pointer back and return */ + if(memcmp(segt.magic,AF_SEGTAIL,sizeof(AF_SEGTAIL))!=0){ + fseeko(af->aseg,start,SEEK_SET); + return -1; + } + + /* Now I know how long the segment was. Compute where it started */ + uint64_t seg_start = start - ntohl(segt.segment_len); + fseeko(af->aseg,seg_start,SEEK_SET); + return 0; +} + + + +/* find the given segment and return 0 if found. + * Leave the file pointer positioned at the start of the segment. + * Return -1 if segment is not found, and leave pointer at the end + */ +int aff_find_seg(AFFILE *af,const char *segname, + uint32_t *arg, + size_t *datasize, + size_t *segsize) +{ + char next_segment_name[AF_MAX_NAME_LEN]; + size_t next_segsize = 0; + size_t next_datasize = 0; + uint32_t next_arg; + + /* Try to use the TOC to find the segment in question */ + struct aff_toc_mem *adm = aff_toc(af,segname); + if(adm){ + if(datasize==0 && segsize==0 && arg==0){ + /* User was just probing to see if it was present. And it is! */ + return 0; + } + fseeko(af->aseg,adm->offset,SEEK_SET); + } + else { + af_rewind_seg(af); + } + while(af_probe_next_seg(af,next_segment_name,sizeof(next_segment_name), + &next_arg,&next_datasize,&next_segsize,1)==0){ + if(strcmp(next_segment_name,segname)==0){ // found the segment! + if(datasize) *datasize = next_datasize; + if(segsize) *segsize = next_segsize; + if(arg) *arg = next_arg; + return 0; // return the info + } + fseeko(af->aseg,next_segsize,SEEK_CUR); // skip the segment + } + return -1; // couldn't find segment +} + +int af_get_segq(AFFILE *af,const char *name,int64_t *aff_quad) +{ + unsigned char buf[8]; + size_t bufsize = sizeof(buf); + + if(af_get_seg(af,name,0,(unsigned char *)&buf,&bufsize)){ + return -1; // couldn't get it... + } + if(bufsize!=sizeof(struct aff_quad)){ // make sure size is good. + return -1; + } + + *aff_quad = af_decode_q(buf); + return 0; +} + + +/* af_update_segq: + * Update the named aff_quad-byte value. + */ + + +int af_update_segq(AFFILE *af, const char *name, int64_t value) +{ + struct aff_quad q; + q.low = htonl((uint32_t)(value & 0xffffffff)); + q.high = htonl((uint32_t)(value >> 32)); + return af_update_seg(af,name,AF_SEG_QUADWORD,(const u_char *)&q,8); +} + + diff --git a/lib/aff_db.h b/lib/aff_db.h new file mode 100644 index 0000000..b05fad6 --- /dev/null +++ b/lib/aff_db.h @@ -0,0 +1,10 @@ +/* Distributed under the 4-part Berkeley License */ + +#ifdef __cplusplus +extern "C" { +#endif + int aff_find_seg(AFFILE *af,const char *segname, uint32_t *arg,size_t *datasize, size_t *segsize); +#ifdef __cplusplus +} +#endif + diff --git a/lib/aff_toc.cpp b/lib/aff_toc.cpp new file mode 100644 index 0000000..5bd067b --- /dev/null +++ b/lib/aff_toc.cpp @@ -0,0 +1,139 @@ +/* Distributed under the 4-part Berkeley License */ + +/* + * afflib_dir.cpp: + * + * Functions for the manipulation of the AFF directories + */ + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" + +int aff_toc_free(AFFILE *af) +{ + if(af->toc){ + for(int i=0;itoc_count;i++){ + if(af->toc[i].name) free(af->toc[i].name); + } + free(af->toc); + af->toc = 0; + af->toc_count = 0; + } + return 0; +} + + +void aff_toc_print(AFFILE *af) +{ + printf("AF DIRECTORY:\n"); + for(int i=0;itoc_count;i++){ + if(af->toc[i].name){ + printf("%-32s @%"I64u" len: %"I64u" \n",af->toc[i].name, af->toc[i].offset,af->toc[i].segment_len); + } + } +} + +#if 0 +static int toc_sort(const void *a_,const void *b_) +{ + const aff_toc_mem *a = (const aff_toc_mem *)a_; + const aff_toc_mem *b = (const aff_toc_mem *)b_; + if(a->offset < b->offset) return -1; + if(a->offset > b->offset) return 1; + return 0; +} +#endif + +static int aff_toc_append(AFFILE *af,const char *segname,uint64_t offset,uint64_t datalen) +{ + af->toc = (aff_toc_mem *)realloc(af->toc,sizeof(*af->toc)*(af->toc_count+1)); + if(af->toc==0){ + (*af->error_reporter)("realloc() failed in aff_toc_append. toc_count=%d\n",af->toc_count); + return -1; + } + af->toc[af->toc_count].offset = offset; + af->toc[af->toc_count].name = strdup(segname); // make a copy of the string + af->toc[af->toc_count].segment_len = aff_segment_overhead(segname)+datalen; + af->toc_count++; + return 0; +} + +/* Find an empty slot in the TOC in which to put the TOC. + * Otherwise add it to the end. + */ +void aff_toc_update(AFFILE *af,const char *segname,uint64_t offset,uint64_t datalen) +{ + for(int i=0;itoc_count;i++){ + if(af->toc[i].name==0 || strcmp(af->toc[i].name,segname)==0){ + if(af->toc[i].name==0){ // if name was empty, copy it over + af->toc[i].name = strdup(segname); + } + af->toc[i].offset = offset; + af->toc[i].segment_len = aff_segment_overhead(segname)+datalen; + return; + } + } + aff_toc_append(af,segname,offset,datalen); /* Need to append it to the directory */ +} + +/* + * aff_toc_build: + * Build the directory by reading the existing file. + * Notice that we know that we can simply append. + */ +int aff_toc_build(AFFILE *af) // build the dir if we couldn't find it +{ + aff_toc_free(af); // clear the old one + af_rewind_seg(af); // start at the beginning + + // note: was malloc(0), but that causes problems under Borland + af->toc = (aff_toc_mem *)malloc(sizeof(aff_toc_mem)); + while(1){ + char segname[AF_MAX_NAME_LEN]; + size_t segname_len=sizeof(segname); + uint64_t pos = ftello(af->aseg); + size_t datalen=0; + + int r = af_get_next_seg(af,segname,segname_len,0,0,&datalen); + switch(r){ + case AF_ERROR_NO_ERROR: + if(aff_toc_append(af,segname,pos,datalen)){ + return -1; // malloc error? + } + break; + case AF_ERROR_EOF: + return 0; // end of file; no errors + default: /* unknown error */ + fseeko(af->aseg,pos,SEEK_SET); // go back + return r; // send up the error code + } + } + return AF_ERROR_NO_ERROR; +} + +/* + * return the named entry in the directory + */ + +struct aff_toc_mem *aff_toc(AFFILE *af,const char *segname) +{ + for(int i=0;itoc_count;i++){ + if(af->toc[i].name && strcmp(af->toc[i].name,segname)==0) return &af->toc[i]; + } + return 0; +} + +/* Delete something from the directory, but don't bother reallocating.*/ +int aff_toc_del(AFFILE *af,const char *segname) +{ + for(int i=0;itoc_count;i++){ + if(af->toc[i].name && strcmp(af->toc[i].name,segname)==0){ + free(af->toc[i].name); + af->toc[i].name=0; + return 0; // should only be in TOC once + } + } + return -1; +} + diff --git a/lib/afflib.cpp b/lib/afflib.cpp new file mode 100644 index 0000000..45eda59 --- /dev/null +++ b/lib/afflib.cpp @@ -0,0 +1,979 @@ +/* + * afflib.cpp + * Distributed under the Berkeley 4-part license + */ + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" + +#ifdef HAVE_OPENSSL_PEM_H +#include +#include +#endif + +#include "vnode_raw.h" +#include "vnode_split_raw.h" +#include "vnode_afm.h" +#include "vnode_aff.h" +#include "vnode_afd.h" + +#ifdef USE_QEMU +#include "vnode_qemu.h" +#endif + +#ifdef USE_S3 +#include "vnode_s3.h" +#endif + +#ifdef HAVE_FCNTL_H +#include +#endif + +#include +#include + + +// vnode implementations. +// order matters +struct af_vnode *af_vnode_array[] = { +#ifdef USE_S3 + &vnode_s3, // must be first for s3:// interpertation +#endif + &vnode_afd, + &vnode_afm, // must be before aff + &vnode_aff, +#ifdef USE_QEMU + &vnode_vmdk, + &vnode_dmg, +#endif +#ifdef USE_SPARSEIMAGE + &vnode_sparseimage, +#endif + &vnode_split_raw, // must be before raw + &vnode_raw, // greedy; must be last + 0}; + + + +/**************************************************************** + *** Support functions that don't use the "af" + ****************************************************************/ + +static int aff_initialized = 0; +int af_cache_debug = 0; +FILE *af_trace = 0; + +void af_initialize() +{ + if(aff_initialized) return; + + /* make sure things were compiled properly */ + assert(sizeof(struct af_head)==8); + assert(sizeof(struct af_segment_head)==16); + assert(sizeof(struct af_segment_tail)==8); + assert(sizeof(struct affkey)==AFFKEY_SIZE); + + /* Make sure OpenSSL is working */ + OpenSSL_add_all_algorithms(); + + const char *val = getenv(AFFLIB_CACHE_DEBUG); + if(val) af_cache_debug = atoi(val); + + val = getenv(AFFLIB_TRACEFILE); + if(val){ + af_trace = fopen(val,"wa"); + fprintf(af_trace,"============================\n"); + fprintf(af_trace,"AFFLIB trace started\n"); + setvbuf(af_trace,0,_IOLBF,0); + } + aff_initialized = 1; +} + + +/**************************************************************** + *** Other functions that don't use the af + ****************************************************************/ + + +const char *af_version(void) +{ + return PACKAGE_VERSION; +} + + +/* Return 1 if a file is probably an AFF file + * 0 if it is not. + * -1 if failure. + */ + +int af_identify_file_type(const char *filename,int exists) +{ + for(int i = 0; af_vnode_array[i]; i++){ + if( (*af_vnode_array[i]->identify)(filename,exists)==1 ){ + return (af_vnode_array[i]->type); + } + } + return exists ? AF_IDENTIFY_NOEXIST : AF_IDENTIFY_ERR; +} + +const char *af_identify_file_name(const char *filename,int exists) +{ + for(int i = 0; af_vnode_array[i]; i++){ + if( (*af_vnode_array[i]->identify)(filename,exists)==1 ){ + return (af_vnode_array[i]->name); + } + } + return 0; +} + +char af_error_str[64]; // in case af_perror is called +void af_perror(const char *str) +{ +#ifdef HAVE_GETPROGNAME + fprintf(stderr,"%s: %s: %s\n",getprogname(),str,af_error_str); +#else + fprintf(stderr,"%s: %s\n",str,af_error_str); +#endif +} + + +/* Return the 'extension' of str. + * af_str("filename.aff") = ".aff" + */ +const char *af_ext(const char *str) +{ + int len = strlen(str); + if(len==0) return str; // no extension + for(int i=len-1;i>0;i--){ + if(str[i]=='.') return &str[i+1]; + } + return str; +} + +int af_ext_is(const char *filename,const char *ext) +{ + return strcasecmp(af_ext(filename),ext)==0; +} + +static int ends_with(const char *buf,const char *with) +{ + if(buf && with){ + size_t buflen = strlen(buf); + size_t withlen = strlen(with); + if(buflen>withlen && strcmp(buf+buflen-withlen,with)==0) return 1; + } + return 0; +} + +/**************************************************************** + *** GET FUNCTIONS + ****************************************************************/ + +/**************************************************************** + *** Probe the next segment: + *** Return its name and argument, but don't advance the pointer.... + *** Returns 0 on success, -1 on end of file or other error. + ****************************************************************/ + +/**************************************************************** + *** AFF creation functions + ****************************************************************/ + +static void af_deallocate(AFFILE *af) +{ + /* Clear out the cache */ + if(af->pbcache){ + for(int i=0;inum_pbufs;i++){ + struct aff_pagebuf *p = &af->pbcache[i]; + if(p->pagebuf){ + memset(p->pagebuf,0,af->image_pagesize); // clean object reuse + free(p->pagebuf); + } + } + free(af->pbcache); + } +#ifdef HAVE_PTHREAD + AF_UNLOCK(af); + pthread_rwlock_destroy(&af->rwlock); +#endif + if(af->protocol) free(af->protocol); + if(af->fname) free(af->fname); + if(af->username) free(af->username); + if(af->password) free(af->password); + if(af->hostname) free(af->hostname); + if(af->badflag) free(af->badflag); + if(af->toc) free(af->toc); + if(af->crypto) af_crypto_deallocate(af); + if(af->vni_cache) free(af->vni_cache); + memset(af,0,sizeof(*af)); // clean object reuse + free(af); +} + +static void af_sanitize_password(AFFILE *af) +{ + for(char *cc = af->password;*cc;cc++){ + *cc = 'X'; + } + free(af->password); + af->password = 0; +} + + +/* af_open_with is the real open routine. + * It opens a particular file with a particular vnode implementation. + */ +AFFILE *af_open_with(const char *url,int flags,int mode, struct af_vnode *v) +{ + /* Alloate the space for the AFFILE structure */ + AFFILE *af = (AFFILE *)calloc(sizeof(AFFILE),1); + af_crypto_allocate(af); +#ifdef HAVE_PTHREAD + pthread_rwlock_init(&af->rwlock); + AF_WRLOCK(af); +#endif + af->v = v; + af->version = 2; + af->openflags = flags | O_BINARY; // make sure that we ask for binray + af->openmode = mode; + af->image_sectorsize = 512; // default size + af->error_reporter = warnx; + af->badflag = (unsigned char *)malloc(af->image_sectorsize); + + /* Decode URL */ + af_parse_url(url,&af->protocol,&af->hostname,&af->username,&af->password, + &af->port,&af->fname); + + /* A null passphrase is the same as no passphrase*/ + if(af->password && af->password[0]==0){ + free(af->password); + af->password=0; + } + /* If no password was set and the AFFLIB_PASSPHRASE environment variable is set, use that */ + if(af->password==0 && getenv(AFFLIB_PASSPHRASE)){ + af->password = strdup(getenv(AFFLIB_PASSPHRASE)); + } + /* If no password is set and its in a file, get it there */ + if(af->password==0 && getenv(AFFLIB_PASSPHRASE_FILE)){ + int fd = open(AFFLIB_PASSPHRASE_FILE,O_RDONLY,0); + if(fd>0){ + struct stat sb; + if(fstat(fd,&sb)==0){ + af->password = (char *)malloc(sb.st_size); + int r = read(fd,af->password,sb.st_size); + if(r!=sb.st_size){ + free(af->password); + af->password=0; // couldn't read it + } + close(fd); + } + } + } + /* If no password is set and its in a file, get it there */ + if(af->password==0 && getenv(AFFLIB_PASSPHRASE_FD)){ + int fd = atoi(AFFLIB_PASSPHRASE_FD); + af->password = (char *)malloc(1); + int buflen = 0; + int rlen = 0; + char mybuf[1024]; + + while((rlen=read(fd,mybuf,sizeof(mybuf)))>0){ + af->password = (char *)realloc(af->password,buflen+rlen+1); + memcpy(af->password+buflen,mybuf,rlen); + buflen += rlen; + af->password[buflen] = '\000'; + } + } + + /* TK: If no password was set and the AFFLIB_ASK_PASS is set, ask for a passphrase */ + + /* Note things for hard files */ + af->exists = (access(af->fname,R_OK) == 0); // does the file exist? + + /* Right now just set up the cache by hand */ + const char *cache_pages = getenv(AFFLIB_CACHE_PAGES); + if(cache_pages) af->num_pbufs = atoi(cache_pages); + if(af->num_pbufs<1) af->num_pbufs = AFFLIB_CACHE_PAGES_DEFAULT; // default valuen + + af->pbcache = (struct aff_pagebuf *)calloc(af->num_pbufs,sizeof(struct aff_pagebuf)); + if(af->pbcache==0){ // if can't allocate the full amount + af->num_pbufs = 2; // try a significantly smaller cache + af->pbcache = (struct aff_pagebuf *)calloc(af->num_pbufs,sizeof(struct aff_pagebuf)); + } + + if(flags & AF_HALF_OPEN) return af; // for low-level tools + + /* Try opening it! */ + if((*af->v->open)(af)){ + strlcpy(af_error_str,af->error_str,sizeof(af_error_str)); // make a copy of the error string + af_deallocate(af); + return 0; + } + + /* If there is no AFFKEY and the file is read-only, don't use a password */ + if(af->password && (af_get_seg(af,AF_AFFKEY,0,0,0)!=0) && ((af->openflags & O_ACCMODE)==O_RDONLY)){ + af_sanitize_password(af); + } + + /* Set up the encryption if requested and if this support metadata */ + if(AF_SEALING_VNODE(af) && ((flags & AF_NO_CRYPTO)==0)){ + bool can_decrypt = false; + if(af->password){ + struct af_vnode_info vni; + memset(&vni,0,sizeof(vni)); + if((*af->v->vstat)(af,&vni)==0 && vni.supports_metadata){ + int r = 0; + if(af_get_seg(af,AF_AFFKEY,0,0,0)!=0){ // it does not have a password + r = af_establish_aes_passphrase(af,af->password); + } + if(r==0){ + r = af_use_aes_passphrase(af,af->password); + if(r==0) { + can_decrypt = true; + } else { + (*af->error_reporter)("af_open: invalid passphrase: '%s'",af->password); + } + } + af_sanitize_password(af); + } + } + + /* Try public key... */ + if(can_decrypt==false){ + const char *kf = getenv(AFFLIB_DECRYPTING_PRIVATE_KEYFILE); + if(kf){ + af_set_unseal_keyfile(af,kf); + } + } + } + + af_read_sizes(af); // set up the metadata + if(af_trace) fprintf(af_trace,"af_open_with(%s,%o,%o,%s)\n",url,flags,mode,v->name); + return af; +} + + + +AFFILE *af_open(const char *filename,int flags,int mode) +{ + if(!aff_initialized) af_initialize(); + if(ends_with(filename,".E01") || ends_with(filename,".e01")){ + return 0; + } + + if(flags & O_WRONLY){ + errno = EINVAL; + return 0; // this flag not supported + } + int exists = (flags & O_CREAT) ? 0 : 1; // file must exist if O_CREAT not specified + + + /* Figure out it's format, then hand off to the correct subsystem. */ + for(int i = 0; af_vnode_array[i]; i++){ + /* Check to see if the implementation identifies the file */ + if( (*af_vnode_array[i]->identify)(filename,exists)==1 ){ + AFFILE *af = af_open_with(filename,flags,mode,af_vnode_array[i]); + return af; + } + } + errno = EINVAL; + if(exists) errno = ENOENT; + return 0; // can't figure it out; must be an invalid extension +} + +/** Set an option and return the previous value */ +int af_set_option(AFFILE *af,int option,int value) +{ + int prev = 0; + switch(option){ + case AF_OPTION_AUTO_ENCRYPT: + prev = af->crypto->auto_encrypt; + af->crypto->auto_encrypt = value; + return prev; + case AF_OPTION_AUTO_DECRYPT: + prev = af->crypto->auto_decrypt; + af->crypto->auto_decrypt = value; + return prev; + } + return -1; +} + +/* Open a regular file as an affile. + * Can only be a raw file... + */ +AFFILE *af_freopen(FILE *file) +{ + if(!aff_initialized) af_initialize(); + + AFFILE *af = (AFFILE *)calloc(sizeof(AFFILE),1); + af->v = &vnode_raw; + af->image_sectorsize = 512; // default + raw_freopen(af,file); + return af; +} + +#ifdef UNIX +/* Open a regular file as an affile */ +AFFILE *af_popen(const char *command,const char *type) +{ + if(!aff_initialized) af_initialize(); + AFFILE *af = (AFFILE *)calloc(sizeof(AFFILE),1); + af->v = &vnode_raw; + raw_popen(af,command,type); + af->image_sectorsize = 512; // default + af->openflags = O_RDONLY; + af->fname = strdup(command); + return af; +} +#endif + + +/* Close the image and unallocate fields */ +int af_close(AFFILE *af) +{ + int ret = 0; + + AF_WRLOCK(af); + af_cache_flush(af); // flush the cache (if writing) + + if(af->image_size != af->image_size_in_file){ + af_update_segq(af,AF_IMAGESIZE,(int64_t)af->image_size); + af->image_size_in_file = af->image_size; + } + if(getenv(AFFLIB_CACHE_STATS)){ + fputc('\n',stderr); + af_stats(af,stderr); + } + + (*af->v->close)(af); + af_deallocate(af); + return ret; +} + + +/* Seek in the virtual file */ +uint64_t af_seek(AFFILE *af,int64_t pos,int whence) +{ + AF_WRLOCK(af); + if(af_trace) fprintf(af_trace,"af_seek(%p,%"I64d",%d)\n",af,pos,whence); + uint64_t new_pos=0; + switch(whence){ + case SEEK_SET: + new_pos = pos; + break; + case SEEK_CUR: + if(pos<0 && ((uint64_t)(-pos)) > af->pos) new_pos=0; + else new_pos = af->pos + pos; + break; + case SEEK_END: + if((uint64_t)pos > af->image_size) new_pos=0; + else new_pos = af->image_size - pos; + break; + } + + /* Note if the direction has changed */ + int direction = (new_pos > af->pos) ? 1 : ((new_pos < af->pos) ? -1 : 0); + if(af->last_direction != direction) af->direction_changes++; + if(af->direction_changes > 5 && af->random_access==0){ + af->random_access=1; + } + af->last_direction = direction; + /*****************************************************************/ + + /* Finally update the direction */ + af->pos = new_pos; // set the new position + AF_UNLOCK(af); + return af->pos; +} + +/* Returns the name and offset of the last segment */ +int af_last_seg(AFFILE *af,char *last_segname,int last_segname_len,int64_t *pos) +{ + AF_WRLOCK(af); + /* Find the name of the last segment */ + fseeko(af->aseg,0,SEEK_END); + af_backspace(af); // back up one segment + *pos = ftello(af->aseg); // remember where it is + last_segname[0] = 0; + int ret = af_probe_next_seg(af,last_segname,last_segname_len,0,0,0,0); + AF_UNLOCK(af); + return ret; +} + + +uint64_t af_tell(AFFILE *af) +{ + AF_READLOCK(af); + uint64_t ret = af->pos; + AF_UNLOCK(af); + return ret; +} + +/* Return if we are at the end of the file */ +int af_eof(AFFILE *af) +{ + AF_READLOCK(af); + af_vnode_info vni; + + if(af_vstat(af,&vni)) return -1; // this is bad; we need vstat... + if(vni.use_eof) return vni.at_eof; // if implementation wants to use it, use it + if(af->pos<0){ // pos shouldn't be <0 + errno = EINVAL; + return -1; // this is bad + } + int ret = (int64_t)af->pos >= (int64_t)vni.imagesize; + AF_UNLOCK(af); + return ret; +} + +void af_set_callback(AFFILE *af,void (*wcb)(struct affcallback_info *)) +{ + AF_WRLOCK(af); + af->w_callback = wcb; + AF_UNLOCK(af); +} + + +void af_enable_compression(AFFILE *af,int type,int level) +{ + AF_WRLOCK(af); + af->compression_type = type; + af->compression_level = level; + AF_UNLOCK(af); +} + +int af_compression_type(AFFILE *af) +{ + AF_READLOCK(af); + int ret = af->compression_type; + AF_UNLOCK(af); + return ret; +} + + +/* Doesn't need locking because it won't change */ +const char *af_filename(AFFILE *af) +{ + return af->fname; +} + +/* Doesn't need locking because it won't change */ +int af_identify(AFFILE *af) +{ + return af->v->type; +} + +/* af_get_imagesize: + * Return the byte # of last mapped byte in image, or size of device; + * No locking is needed because individual elements of the af structure are not accessed. + */ +int64_t af_get_imagesize(AFFILE *af) +{ + int64_t ret = -1; + struct af_vnode_info vni; + memset(&vni,0,sizeof(vni)); + if(af_vstat(af,&vni)==0){ + /* If vni.imagesize is 0 and if there are encrypted segments and if there + * is no imagesize segment but there is an encrypted one, then we can't read this encrypted file... + */ + if(vni.imagesize<=0 && vni.segment_count_encrypted>0){ + if(af_get_seg(af,AF_IMAGESIZE,0,0,0)!=0){ + errno = EPERM; + goto done; + } + } + ret = vni.imagesize; + } + done:; + return ret; +} + +/* + * af_make_badflag: + * Create a randomized bag flag and + * leave an empty segment of how many badsectors there are + * in the image... + */ +int af_make_badflag(AFFILE *af) +{ +#ifdef HAVE_RAND_pseudo_bytes + /* Use a good random number generator if we have it */ + RAND_pseudo_bytes(af->badflag,af->image_sectorsize); + strcpy((char *)af->badflag,"BAD SECTOR"); +#else + /* Otherwise use a bad one */ + for(uint32_t i=0;iimage_sectorsize;i++){ + af->badflag[i] = rand() & 0xff; + } +#endif + + AF_WRLOCK(af); + af->badflag_set = 1; + if(af_update_seg(af,AF_BADFLAG,0,af->badflag,af->image_sectorsize)){ + AF_UNLOCK(af); + return -1; + } + if(af_update_segq(af,AF_BADSECTORS,0)){ + AF_UNLOCK(af); + return -1; + } + AF_UNLOCK(af); + return 0; +} + + +/* + * make the IMAGE_GID segment if it doesn't exist + * Returns -1 if an error, 0 if the GID exists, and 1 if one is made. + */ +int af_make_gid(AFFILE *af) +{ + int ret = 0; + AF_WRLOCK(af); + if(af_get_seg(af,AF_IMAGE_GID,0,0,0)!=0){ + unsigned char bit128[16]; + RAND_pseudo_bytes(bit128,sizeof(bit128)); + int r = af_update_seg(af,AF_IMAGE_GID,0,bit128,sizeof(bit128)); + if(r<0) ret = -1; + else ret = 1; + } + AF_UNLOCK(af); + return ret; +} + + + +/* Decrypt data and perform unblocking if necessary. + * This could eliminate a memory copy by doing the decryption for everything + * but the last block in place, and just doing a copy for the last block. + */ +void af_aes_decrypt(AFFILE *af,const char *segname,unsigned char *data,size_t *datalen) +{ +#ifdef HAVE_AES_ENCRYPT + if(datalen==0) return; // can't decrypt; no clue how long it is + + /* An encrypted segment was retrieved; decrypt and trunc the length as necessary */ + uint32_t extra = (*datalen) % AES_BLOCK_SIZE; + uint32_t pad = (AES_BLOCK_SIZE - extra) % AES_BLOCK_SIZE; + + if(data==0){ // just wants to find out new length + if(extra>0){ + *datalen -= AES_BLOCK_SIZE; + } + return; + } + + if(extra!=0 && *datalen < AES_BLOCK_SIZE){ + *datalen = 0; // something is wrong + return; + } + + if(data==0){ // just report the new size + if(extra!=0) *datalen -= AES_BLOCK_SIZE; // a block was added + return; + } + + *datalen -= extra; // *datalen is now a multiple of AES_BLOCK_SIZE + /* Create an IV */ + unsigned char iv[AES_BLOCK_SIZE]; + memset(iv,0,sizeof(iv)); + strlcpy((char *)iv,segname,sizeof(iv)); + + /* Decrypt! */ + AF_READLOCK(af); + AES_cbc_encrypt(data,data,*datalen,&af->crypto->dkey,iv,AES_DECRYPT); + AF_UNLOCK(af); + *datalen -= pad; // remove the padding +#endif +} + + +int af_get_seg(AFFILE *af,const char *segname,uint32_t *arg,unsigned char *data,size_t *datalen) +{ + AF_READLOCK(af); + if(af->v->get_seg==0){ + errno = ENOTSUP; + return -1; // not supported by this file system + } +#ifdef HAVE_AES_ENCRYPT + /* If we have encryption and it is turned on, check for encrypted segment first */ + if(AF_SEALING_VNODE(af) && af->crypto->auto_decrypt){ + size_t datalen_orig = datalen ? *datalen : 0; + char aesname[AF_MAX_NAME_LEN]; + strlcpy(aesname,segname,sizeof(aesname)); + strlcat(aesname,AF_AES256_SUFFIX,sizeof(aesname)); + int r = (*af->v->get_seg)(af,aesname,arg,data,datalen); + if(r==0){ + af_aes_decrypt(af,segname,data,datalen); + return 0; + } + if(r==AF_ERROR_DATASMALL && datalen && (*datalen % AES_BLOCK_SIZE != 0)){ + /* Not enough space was provided to decrypt, probably because this was blocked out */ + size_t bigger_data_len = datalen_orig + AES_BLOCK_SIZE; + unsigned char *bigger_data = (unsigned char *)malloc(bigger_data_len); + if(!bigger_data) return -1; // Malloc failed + r = (*af->v->get_seg)(af,aesname,arg,bigger_data,&bigger_data_len); + if(r!=0){ + free(bigger_data); + return -1; // something deeper is wrong + } + af_aes_decrypt(af,segname,bigger_data,&bigger_data_len); + if(bigger_data_len > datalen_orig){ + free(bigger_data); + return -1; // it's still too big + } + memcpy(data,bigger_data,bigger_data_len); + *datalen = bigger_data_len; + free(bigger_data); + return 0; // finally it fits + } + } +#endif + /* Try for the unencrypted segment */ + int ret = (*af->v->get_seg)(af,segname,arg,data,datalen); + AF_UNLOCK(af); + return ret; +} + +int af_get_next_seg(AFFILE *af,char *segname,size_t segname_len,uint32_t *arg, + unsigned char *data,size_t *datalen) +{ + size_t datalen_orig = datalen ? *datalen : 0; + AF_READLOCK(af); + if(af->v->get_next_seg==0){ + errno = ENOTSUP; + AF_UNLOCK(af); + return -1; + } + int r = (*af->v->get_next_seg)(af,segname,segname_len,arg,data,datalen); +#ifdef HAVE_AES_ENCRYPT + if(AF_SEALING_VNODE(af) + && ends_with(segname,AF_AES256_SUFFIX) + && af->crypto->auto_decrypt){ + segname[strlen(segname)-strlen(AF_AES256_SUFFIX)] = 0; + /* An encrypted segment was retrieved. + * If it fit, decrypt and return. + * If it doesn't fit, try to get it again (which will use our adaptive blocksize) + * + * Normaly it will fit becuase the caller doesn't know how long the 'next' segment is, + * so the caller normally leaves enough space. + */ + if(r==0){ + af_aes_decrypt(af,segname,data,datalen); + AF_UNLOCK(af); + return 0; + } + if(r==AF_ERROR_DATASMALL && datalen && (*datalen % AES_BLOCK_SIZE !=0)){ + *datalen = datalen_orig; + AF_UNLOCK(af); + return af_get_seg(af,segname,arg,data,datalen); + } + AF_UNLOCK(af); + return r; // not sure why we got this error + } +#endif + AF_UNLOCK(af); + return r; +} + +int af_rewind_seg(AFFILE *af) +{ + if(af_trace) fprintf(af_trace,"af_rewind_seg(%p)\n",af); + AF_READLOCK(af); + if(af->v->rewind_seg==0){ + errno = ENOTSUP; + AF_UNLOCK(af); + return -1; + } + int ret = (*af->v->rewind_seg)(af); + AF_UNLOCK(af); + return ret; +} + +/** Main routine for writing segments + */ + +int af_update_segf(AFFILE *af, const char *segname, + uint32_t arg,const u_char *data,uint32_t datalen,uint32_t flag) +{ + if(af_trace) fprintf(af_trace,"af_update_segf(%p,segname=%s,arg=%"PRIu32",datalen=%d)\n", + af,segname,arg,datalen); + AF_WRLOCK(af); + if(af->v->update_seg==0){ + errno = ENOTSUP; + AF_UNLOCK(af); + return -1; // not supported by this file system + } + + af_invalidate_vni_cache(af); + + /* See if we need to encrypt. New memory might need to be allocated. + * This isn't a big deal, because encryption requires copying memory + * in any event; it's either an in-place copy or a copy to another location. + */ +#ifdef HAVE_AES_ENCRYPT + const char *oldname = 0; + unsigned char *newdata = 0; + if(AF_SEALING(af) && ((flag & AF_SIGFLAG_NOSEAL)==0) && af->crypto->auto_encrypt){ + /* Create an IV */ + unsigned char iv[AES_BLOCK_SIZE]; + memset(iv,0,sizeof(iv)); + strlcpy((char *)iv,segname,sizeof(iv)); + + /* Figure out the real segment name */ + char aesname[AF_MAX_NAME_LEN]; + strlcpy(aesname,segname,sizeof(aesname)); + strlcat(aesname,AF_AES256_SUFFIX,sizeof(aesname)); + oldname = segname; + segname = aesname; + + /* Figure out if we need to padd out for encryption. Allocate space and + */ + uint32_t extra = (datalen) % AES_BLOCK_SIZE; + uint32_t pad = (AES_BLOCK_SIZE - extra) % AES_BLOCK_SIZE; + newdata = (unsigned char *)malloc(datalen+pad+extra); + memset(newdata+datalen,pad+extra,pad); // PKCS7 uses 01 for one pad byte, 02 02 for two, etc. + /* Encrypt */ + AES_cbc_encrypt((const unsigned char *)data, + newdata, + datalen+pad,&af->crypto->ekey,iv,AES_ENCRYPT); + data = newdata; // we will write this out + datalen += pad + extra; + } +#endif + int r = (*af->v->update_seg)(af,segname,arg,data,datalen); // actually update the segment + if(r==0) af->bytes_written += datalen; +#ifdef HAVE_AES_ENCRYPT + /* if we encrypted, make sure the unencrypted segment is deleted */ + if(r==0 && oldname) (*af->v->del_seg)(af,oldname); + if(newdata){ + free(newdata); // free any allocated data + newdata = 0; + } +#endif + /* If we wrote out an unencrypted segment, make sure that the corresopnding encrypted + * segment is deleted. + */ + char encrypted_name[AF_MAX_NAME_LEN]; + strlcpy(encrypted_name,segname,sizeof(encrypted_name)); + strlcat(encrypted_name,AF_AES256_SUFFIX,sizeof(encrypted_name)); + if(*af->v->del_seg) (*af->v->del_seg)(af,encrypted_name); // no need to check error return + + + /* Sign the segment if: + * - there is a signing private key + * - the data structure and flag not set + * - This is not a signature segment + */ +#ifdef USE_AFFSIGS + const u_char *signdata = data; // remember the original data location + if(AF_SEALING(af) + && (r==0) + && af->crypto->sign_privkey + && ((flag & AF_SIGFLAG_NOSIG)==0) + && !ends_with(segname,AF_SIG256_SUFFIX)){ + // return code doesn't matter; it's either signed or not. + af_sign_seg3(af,segname,arg,signdata,datalen,AF_SIGNATURE_MODE0); + } +#endif + AF_UNLOCK(af); + return r; +} + +/* Requires no locking because locking is done in af_update_segf */ +int af_update_seg(AFFILE *af, const char *segname, + uint32_t arg,const u_char *data,uint32_t datalen) +{ + return af_update_segf(af,segname,arg,data,datalen,0); +} + +#ifdef HAVE_OPENSSL_BIO_H +/* Requires no locking */ +int af_update_seg_frombio(AFFILE *af,const char *segname,uint32_t arg,BIO *bio) +{ + /* Get the buffer to write out */ + u_char *buf=0; + size_t buflen = BIO_get_mem_data(bio,&buf); + return af_update_seg(af,segname,0,buf,buflen); +} +#endif + +int af_del_seg(AFFILE *af,const char *segname) +{ + AF_WRLOCK(af); + if(af->v->del_seg==0){ + errno = ENOTSUP; + AF_UNLOCK(af); + return -1; // not supported + } +#ifdef HAVE_AES_ENCRYPT + if(AF_SEALING(af)){ + /* Delete the encrypted segment if it exists */ + char aesname[AF_MAX_NAME_LEN]; + strlcpy(aesname,segname,sizeof(aesname)); + strlcat(aesname,AF_AES256_SUFFIX,sizeof(aesname)); + (*af->v->del_seg)(af,aesname); + } +#endif + /* Delete the unencrypted segment */ + int ret = (*af->v->del_seg)(af,segname); + AF_UNLOCK(af); + return ret; +} + +void af_invalidate_vni_cache(AFFILE *af) +{ + if(af->vni_cache){ + free(af->vni_cache); + af->vni_cache = 0; + } +} + +int af_vstat(AFFILE *af,struct af_vnode_info *vni) +{ + AF_READLOCK(af); + if(af->v->vstat==0){ + errno = ENOTSUP; + AF_UNLOCK(af); + return -1; // not supported + } + int ret = 0; + if(af->vni_cache==0){ // no cached copy? + af->vni_cache = (struct af_vnode_info *)calloc(1,sizeof(struct af_vnode_info)); // allocate a space + ret = (*af->v->vstat)(af,af->vni_cache); + } + if(ret==0) memcpy(vni,af->vni_cache,sizeof(*vni)); + AF_UNLOCK(af); + return ret; +} + +/* Requires no locking */ +int af_has_pages(AFFILE *af) +{ + struct af_vnode_info vni; + if(af_vstat(af,&vni)) return -1; // can't figure it out + return vni.has_pages; // will return 0 or 1 +} + + +void af_stats(AFFILE *af,FILE *f) +{ + AF_READLOCK(af); + fprintf(f,"AFSTATS for %s\n",af_filename(af)); + fprintf(f,"Pages read: %"I64u"\n",af->pages_read); + fprintf(f,"Pages written: %"I64u"\n",af->pages_written); + fprintf(f,"Pages compressed: %"I64u"\n",af->pages_compressed); + fprintf(f,"Pages decompressed: %"I64u"\n",af->pages_decompressed); + fprintf(f,"Cache hits: %"I64u"\n",af->cache_hits); + fprintf(f,"Cache misses: %"I64u"\n",af->cache_misses); + fprintf(f,"Bytes copied: %"I64u"\n",af->bytes_memcpy); + AF_UNLOCK(af); +} + + +int af_set_acquisition_date(AFFILE *af,time_t t) +{ + char timebuf[64]; + strftime(timebuf,sizeof(timebuf),"%Y-%m-%d %H:%M:%S\n",localtime(&t)); + return af_update_seg(af,AF_ACQUISITION_DATE,0,(const u_char *)timebuf,strlen(timebuf)); +} diff --git a/lib/afflib.h b/lib/afflib.h new file mode 100644 index 0000000..8f57280 --- /dev/null +++ b/lib/afflib.h @@ -0,0 +1,550 @@ +#ifndef _AFFLIB_H_ +#define _AFFLIB_H_ + +/* + * afflib.h: + * + * This file describes the public AFFLIB interface. + * The interface to reading AFF files and Raw files. + * + * Copyright (c) 2005-2006 + * Simson L. Garfinkel and Basis Technology, Inc. + * All rights reserved. + * + * This code is derrived from software contributed by + * Simson L. Garfinkel + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by Simson L. Garfinkel + * and Basis Technology Corp. + * 4. Neither the name of Simson Garfinkel, Basis Technology, or other + * contributors to this program may be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY SIMSON GARFINKEL, BASIS TECHNOLOGY, + * AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL SIMSON GARFINKEL, BAIS TECHNOLOGy, + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* Figure out what kind of OS we are running on */ + +/* These are both needed; no need to bother with affconfig.h #defines */ +#include +#ifdef HAVE_STDINT_H +#include +#endif +#include + +#ifdef HAVE_SYS_CDEFS_H +#include +#endif + +#ifdef linux +/* Horrible lossage stuff for largefile support under Linux */ +#define _LARGEFILE_SOURCE 1 +#define _FILE_OFFSET_BITS 64 +#endif + +#ifdef HAVE_INTTYPES_H +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif +#include +#endif + +/** WIN32 is defined by the NMAKE makefile for Visual C++ under Windows and by mingw **/ +#ifdef WIN32 +#include +#include // gets isatty + +/* These aren't needed for mingw */ +#if !defined(__MINGW_H) +#ifndef _UINT64_T_DECLARED +typedef unsigned __int64 uint64_t; /* 64-bit types Types */ +#define _UINT64_T_DECLARED +#endif + +#ifndef _INT64_T_DECLARED +typedef __int64 int64_t; +#define _INT64_T_DECLARED +#endif +#endif + +#ifndef PRId64 +#define PRId64 "I64d" +#endif + +#ifndef PRIi64 +#define PRIi64 "I64i" +#endif + +#ifndef PRIu64 +#define PRIu64 "I64u" +#endif + +#endif +/** END OF WIN32 DEFINES **/ + +#define I64d PRIi64 +#define I64u PRIu64 + +/* If our types still aren't defined, give some kind of error + */ +#define USE_LZMA +struct affcallback_info; +struct aff_pagebuf { + int64_t pagenum; // -1 means no page loaded + unsigned char *pagebuf; // where the data is; size is image_pagesize + size_t pagebuf_bytes; // number of bytes in the pagebuf that are valid. + uint32_t pagenum_valid:1; // buffer contains data + uint32_t pagebuf_valid:1; // buffer contains data + uint32_t pagebuf_dirty:1; // data was modified + int last; // when the page was last visited +}; + +struct af_vnode_info { + uint64_t imagesize; // size of this image + int pagesize; // what is the natural page size? + uint32_t supports_compression:1; // supports writing compressed segments + uint32_t has_pages:1; // does system support page segments? + uint32_t supports_metadata:1; // does it support metadata? + uint32_t is_raw:1; // file is raw + uint32_t use_eof:1; // should we use the EOF flag? + uint32_t at_eof:1; // are we at the EOF? + uint32_t changable_pagesize:1; // pagesize can be changed at any time + uint32_t changable_sectorsize:1; // sectorsize can be changed at any time + uint32_t cannot_decrypt:1; // encrypted pages cannot be decrypted becuase passphrase is invalid + uint32_t segment_count_total; + uint32_t page_count_total; + uint32_t segment_count_signed; + uint32_t segment_count_encrypted; + uint32_t page_count_encrypted; +}; // + + +/* All of this stuff should be hidden inside a single private structure... */ +typedef struct _AFFILE AFFILE; + +/* The information that is provided in the aff callback */ +struct affcallback_info { + int info_version; // version number for this segment + AFFILE *af; // v1: the AFFILE responsibile for the callback + int phase; // v1: 1 = before compress; 2 = after compressing; + // 3 = before writing; 4 = after writing + int64_t pagenum; // v1: page number being written + int bytes_to_write; // v1: >0 if we are going to write bytes + int bytes_written; // v1: >0 if bytes were written + int compressed; // v1: >0 if bytes were/will be compressed + int compression_alg; // v1: compression algorithm + int compression_level; // v1: compression level +}; + +/* Utility Functions */ + +#ifdef __cplusplus +extern "C" { +#endif +#ifdef __never_defined__ +} +#endif + +/**************************************************************** + *** + *** Intended user AFF interface + *** + ****************************************************************/ + +const char * af_version(void); // returns AFF Version Number + +/* af_file stream functions */ +AFFILE *af_open(const char *filename,int flags,int mode); +AFFILE *af_freopen(FILE *file); // reopen a raw file as an AFFILE +AFFILE *af_popen(const char *command,const char *type); // no need to use pclose(); af_close() is fine +int af_close(AFFILE *af); +void af_set_error_reporter(AFFILE *af,void (*reporter)(const char *fmt,...)); +void af_stats(AFFILE *af,FILE *f); // print stats to f +void af_set_cachesize(AFFILE *af,int max); // how much memory can the cache use? +int af_vstat(AFFILE *af,struct af_vnode_info *vni); // does the stat +void af_perror(const char *fname); // print the error string to stderr +void af_err(int code,const char *fname,...); // like err(), but will also print AFF-specific errors + + + +/* Generic set/get option routines; this replaces individual options in previous implementations. + * af==0 to set global options. Return the previous value. + */ +int af_set_option(AFFILE *af,int option,int value); + +#define AF_OPTION_AUTO_ENCRYPT 1 // 1 = auto-encrypt +#define AF_OPTION_AUTO_DECRYPT 2 // 1 = auto-decrypt +// The following are not implemented yet +#define AF_OPTION_PIECEWISE_MD5 3 // 1 = automatically write pagen_md5 segments +#define AF_OPTION_PIECEWISE_SHA1 4 // 1 = automatically write pagen_md5 segments +#define AF_OPTION_PIECEWISE_SHA256 5 // 1 = automatically write pagen_md5 segments +#define AF_OPTION_DISABLE_RDLOCK 6 // 1 = do not read lock, but report that it should have locked. + + +/* Special AFOPEN flags for af_open_with */ +#define AF_OPEN_PRIMITIVE (1<<31) // only open primtive, not compound files +#define AF_BADBLOCK_FILL (1<<30) // fill unallocated (sparse) with BADBLOCK flag +#define AF_HALF_OPEN (1<<29) // return af before calling af->v->open; +#define AF_NO_CRYPTO (1<<28) // disable encryption layer + +/* navigating within the data segments as if they were a single file */ +#ifdef _WIN32 +SSIZE_T af_read(AFFILE *af,unsigned char *buf,SSIZE_T count); +#else +ssize_t af_read(AFFILE *af,unsigned char *buf,ssize_t count); +#endif +uint64_t af_seek(AFFILE *af,int64_t pos,int whence); // returns new position +uint64_t af_tell(AFFILE *af); +int af_eof(AFFILE *af); // is the virtual file at the end? + +/* Additional routines for writing */ +void af_set_callback(AFFILE *af, void (*cb)(struct affcallback_info *acbi)); +void af_enable_compression(AFFILE *af,int type,int level); // set/gunset compression for writing +int af_compression_type(AFFILE *af); +int af_write(AFFILE *af,unsigned char *buf,size_t count); +const unsigned char *af_badflag(AFFILE *af); // return the pattern used to identify bad sectors +int af_is_badsector(AFFILE *af,const unsigned char *buf); // 0 if not, 1 if it is, -1 if error + + +/* Misc. Functions */ +const char *af_ext(const char *filename); // return the extension of str including the dot +int af_ext_is(const char *filename,const char *ext); +const char *af_filename(AFFILE *af); // returns the filename of an open stream. +int af_identify(AFFILE *af); // returns type of AFFILE pointer + +/* Accessor Functions */ +int64_t af_get_imagesize(AFFILE *af); // byte # of last mapped byte in image, or size of device; + // returns -1 if error +int af_get_pagesize(AFFILE *af); // returns page size, or -1 +int af_set_acquisition_date(AFFILE *af,time_t t); // sets AF_ACQUISITION_DATE + +#define af_imagesize(af) af_get_imagesize(af) // backwards compatiability +int af_get_segq(AFFILE *af,const char *name,int64_t *quad);/* Get/set 8-byte values */ +int af_update_segq(AFFILE *af,const char *name,int64_t quad); + + +/**************************************************************** + * Functions for manipulating the AFFILE as if it were a name/value database. + ****************************************************************/ + +/* get functions: + * get the named segment. + * If arg!=0, set *arg to be the segment's flag. + * if data==0, don't return it. + * if datalen && *datalen==0, return the size of the data segment. + *** Returns 0 on success, + *** -1 on end of file. (AF_ERROR_EOF) + *** -2 if *data is not large enough to hold the segment (AF_ERROR_DATASMALL) + *** -3 file is corrupt or other internal error. (AF_ERROR_CORRUPT) + */ + +int af_get_seg(AFFILE *af,const char *name,uint32_t *arg, + unsigned char *data,size_t *datalen); +int af_get_next_seg(AFFILE *af,char *segname,size_t segname_len, + uint32_t *arg, unsigned char *data, size_t *datalen); + +int af_rewind_seg(AFFILE *af); // rewind seg pointer to beginning + +/* + * af_update_seg() should be your primary routine for writing new values. + */ + +/* Writing arbitrary name/value pairs */ +int af_update_seg(AFFILE *af,const char *segname,uint32_t arg, + const unsigned char *value,uint32_t vallen); +#ifdef HAVE_OPENSSL_BIO_H +/* Write a memory bio to a segment */ +#include +int af_update_seg_frombio(AFFILE *af,const char *segname,uint32_t arg,BIO *bio); +#endif + + +/* Delete functions */ + +int af_del_seg(AFFILE *af,const char *name); // complete delete of first name + // returns 0 if success, -1 if seg not found + +/* Segname parse functions. + * af_segname_page_number: + * - Returns page number if segment name is a page #, and -1 if it is not + * af_segname_hash_page_number: + * - Returns page number if segment name is a page hash, sets hash function + * to be the function used. + */ +int64_t af_segname_page_number(const char *name); // return -1 if it is not a page number +int64_t af_segname_hash_page_number(const char *name,char *hash,int hashlen); // return -1 if it is not a hash page # + +int af_display_as_quad(const char *segname); // afflib recommends displaying this segment as an 8-byte quad +int af_display_as_hex(const char *segname); // afflib recommends displaying this segment as a hex-string + +/****************************************************************/ + +/* Crypto */ +/* AFF Base Encryption */ +int af_SHA256(const unsigned char *buf,size_t buflen,unsigned char md[32]); // return 0 if success, -1 if no cipher +int af_set_aes_key(AFFILE *af,const unsigned char *userKey,const int bits); +int af_cannot_decrypt(AFFILE *af); // encrypted pages are present which cannot be decrypted +int af_has_encrypted_segments(AFFILE *af); +int af_is_encrypted_segment(const char *segname); + +/* AFF Passphrase Encryption */ +int af_establish_aes_passphrase(AFFILE *af,const char *passphrase); +int af_change_aes_passphrase(AFFILE *af,const char *oldphrase,const char *newphrase); +int af_use_aes_passphrase(AFFILE *af,const char *passphrase); +int af_save_aes_key_with_passphrase(AFFILE *af,const char *passphrase, const unsigned char affkey[32]); +int af_get_aes_key_from_passphrase(AFFILE *af,const char *passphrase, unsigned char affkey[32]); + + +/* PKI Signing */ +int af_set_sign_files(AFFILE *af,const char *keyfile,const char *certfile); +int af_sign_seg3(AFFILE *af,const char *segname, uint32_t arg, + const unsigned char *data,uint32_t datalen,uint32_t signmode); +int af_sign_seg(AFFILE *af,const char *segname); +int af_sign_all_unsigned_segments(AFFILE *af); // +int af_sig_verify_seg(AFFILE *af,const char *segname); // see below for return codes + +int af_is_signature_segment(const char *segname); + +/* PKI sealing */ +int af_set_seal_certificates(AFFILE *af,const char *certfiles[],int numcertfiles); +int af_seal_affkey_using_certificates(AFFILE *af,const char *certfiles[],int numcertfiles, unsigned char affkey[32]);// +int af_set_unseal_keybuffer(AFFILE *af,const char *key); // take key from a buffer +int af_set_unseal_keyfile(AFFILE *af,const char *keyfile); // take key from a file +int af_get_affkey_using_keyfile(AFFILE *af, const char *private_keyfile,unsigned char affkey[32]); + + + +#ifdef HAVE_OPENSSL_EVP_H +#include +int af_sig_verify_seg2(AFFILE *af,const char *segname,EVP_PKEY *pubkey,unsigned char *sigbuf, + size_t sigbuf_len,int sigmode); +int af_hash_verify_seg2(AFFILE *af,const char *segname,unsigned char *sigbuf_,size_t sigbuf_len_,int sigmode); +#define AF_HASH_VERIFIES 0 + +#endif +#define AF_SIGNATURE_MODE0 0x0000 // signature is for segname, arg, data in segment +#define AF_SIGNATURE_MODE1 0x0001 // signature is for segname, 0 arg, uncompressed data in segment +#define AF_SIGNATURE_DELETE 0xFFFF // signature is invalid; delete segment + +/* Metadata access */ + +/* Compression amounts */ + +#define AF_COMPRESSION_MIN 1 +#define AF_COMPRESSION_DEFAULT -1 +#define AF_COMPRESSION_MAX 9 +#define AF_COMPRESSION_MIN 1 + + +/**************************************************************** + *** AF segment names that you might be interested in... + ****************************************************************/ + +#define AF_IGNORE "" // ignore segments with 0-length name +#define AF_DIRECTORY "dir" // the directory +#define AF_RAW_IMAGE_FILE_EXTENSION "raw_image_file_extension" +#define AF_PAGES_PER_RAW_IMAGE_FILE "pages_per_raw_image_file" + +#define AF_PAGESIZE "pagesize" // page data size, in bytes, stored in arg +#define AF_IMAGESIZE "imagesize" // last logical byte in image, stored as a 64-bit number +#define AF_BADSECTORS "badsectors" // number of bad sectors +#define AF_SECTORSIZE "sectorsize" // in bytes, stored in arg +#define AF_DEVICE_SECTORS "devicesectors"// stored as a 64-bit number +#define AF_BADFLAG "badflag" // data used to mark a bad sector +#define AF_PAGE "page%"I64d // segment flag indicates compression (replaces seg%d) +#define AF_PAGE_MD5 AF_PAGE"_md5" // md5 hash of page +#define AF_PAGE_SHA1 AF_PAGE"_sha1" // sha1 hash of page +#define AF_PAGE_SHA256 AF_PAGE"_sha256"// sha256 hash of page +#define AF_PARITY0 "parity0" // parity page of all bytes +#define AF_BATCH_NAME "batch_name" +#define AF_BATCH_ITEM_NAME "batch_item_name" + +#define AF_BLANKSECTORS "blanksectors" // all NULs; 8-bytes +#define AF_AFF_FILE_TYPE "aff_file_type" // contents should be "AFF", "AFM" or "AFD" + +#define AF_AFFKEY "affkey_aes256" // segment for AES256 session key encrypted with sha of the passphrase +#define AF_AFFKEY_EVP "affkey_evp%d" // segment for encrypted affkey +#define AF_AES256_SUFFIX "/aes256" // suffix for encrypted segments +#define AF_SIG256_SUFFIX "/sha256" // suffix for signature segments +#define AF_SIGN256_CERT "cert-sha256" // segment name for image creator's public key +#define AF_PARITY0_SIG "parity0/sha256" // signature for parity segment + +/* Chain of custody segments */ +#define AF_BOM_SEG "affbom%d" + +/* Deprecated terminology; pages were originally called data segments */ +#define AF_SEG_D "seg%"I64d // segment flag indicates compression (deprecated) +#define AF_SEGSIZE_D "segsize" // segment data size (deprecated) + +/* Bill of Materials */ +#define AF_XML_AFFBOM "affbom" +#define AF_XML_DATE "date" +#define AF_XML_SIGNING_CER "signingcert" +#define AF_XML_SEGMENT_HASH "segmenthash" + + +/* AFF Flags */ +/* Flags for 8-byte segments */ +#define AF_SEG_QUADWORD 0x0002 + +/* Flags for selecting compression algorithm to try */ +#define AF_COMPRESSION_ALG_NONE 0 // don't compress +#define AF_COMPRESSION_ALG_ZLIB 1 // try to compress with zlib +#define AF_COMPRESSION_ALG_LZMA 2 // try to compress with LZMA + +/* Arg Flags for data pages; this is stored in 'flag' of data segment */ +#define AF_PAGE_COMPRESSED 0x0001 +#define AF_PAGE_COMP_MAX 0x0002 // compressed at maximum; nice to know +#define AF_PAGE_COMP_ALG_MASK 0x00F0 // up to 16 compression algorithms may be used +#define AF_PAGE_COMP_ALG_ZLIB 0x0000 +#define AF_PAGE_COMP_ALG_BZIP 0x0010 // not implemented; why bother? +#define AF_PAGE_COMP_ALG_LZMA 0x0020 // high compression but pretty slow +#define AF_PAGE_COMP_ALG_ZERO 0x0030 // Data segment is a 4-byte value of # of NULLs. + +#define AF_MD5 "md5" // stores image md5 +#define AF_SHA1 "sha1" // stores image sha1 +#define AF_SHA256 "sha256" // stores image sha256 + +#define AF_CREATOR "creator" // progname of the program that created the AFF file + +/* segment names: imaging */ +#define AF_CASE_NUM "case_num" // case number +#define AF_IMAGE_GID "image_gid" // 128-bit unique number +#define AF_ACQUISITION_ISO_COUNTRY "acquisition_iso_country" // ISO country code +#define AF_ACQUISITION_COMMAND_LINE "acquisition_commandline" // actual command line used to create the image +#define AF_ACQUISITION_DATE "acquisition_date" // YYYY-MM-DD HH:MM:SS TZT +#define AF_ACQUISITION_NOTES "acquisition_notes" // notes made while imaging +#define AF_ACQUISITION_DEVICE "acquisition_device" // device used to do the imaging +#define AF_ACQUISITION_SECONDS "acquisition_seconds" // stored in arg +#define AF_ACQUISITION_TECHNICIAN "acquisition_tecnician" +#define AF_ACQUISITION_MACADDR "acquisition_macaddr" +#define AF_ACQUISITION_DMESG "acquisition_dmesg" + + +// mac addresses are store in ASCII as a list of lines that end with \n, +// for example, "00:03:93:14:c5:04\n" +// It is all the mac addresses that were on the acquisition system + +// DMESG is the output from the "dmesg" command at the time of acquisition + + +/* segment names: device hardware */ + +#define AF_AFFLIB_VERSION "afflib_version" // version of AFFLIB that made this file +#define AF_DEVICE_MANUFACTURER "device_manufacturer" +#define AF_DEVICE_MODEL "device_model" // string for ident from drive +#define AF_DEVICE_SN "device_sn" // string of drive capabilities +#define AF_DEVICE_FIRMWARE "device_firmware" // string of drive capabilities +#define AF_DEVICE_SOURCE "device_source" // string +#define AF_CYLINDERS "cylinders" // quad with # cylinders +#define AF_HEADS "heads" // quad with # heads +#define AF_SECTORS_PER_TRACK "sectors_per_track"// quad with # sectors/track +#define AF_LBA_SIZE "lbasize" +#define AF_HPA_PRESENT "hpa_present" // flag = 1 or 0 +#define AF_DCO_PRESENT "dco_present" // flag = 1 or 0 +#define AF_LOCATION_IN_COMPUTER "location_in_computer" // text, where it was found +#define AF_DEVICE_CAPABILITIES "device_capabilities" // string; human-readable + +#define AF_MAX_NAME_LEN 64 // segment names should not be larger than this + +/* AFF error codes */ +#define AF_ERROR_NO_ERROR 0 +#define AF_ERROR_EOF -1 +#define AF_ERROR_DATASMALL -2 +#define AF_ERROR_TAIL -3 // no tail, or error reading tail +#define AF_ERROR_SEGH -4 // no head, or error reading head +#define AF_ERROR_NAME -5 // segment name invalid +#define AF_ERROR_INVALID_ARG -6 // argument invalid +#define AF_ERROR_NO_AES -7 // AES support is not compiled in +#define AF_ERROR_AES_TOO_SMALL -8 // and AES-encrypted segment was too small +#define AF_ERROR_KEY_SET -9 // a key was already set +#define AF_ERROR_AFFKEY_EXISTS -10 // a key already exists in file an attempt was made to establish +#define AF_ERROR_AFFKEY_NOT_EXIST -11 // a key does not exist and an attempt was made to use it. +#define AF_ERROR_AFFKEY_WRONG_VERSION -12 +#define AF_ERROR_WRONG_PASSPHRASE -13 +#define AF_ERROR_RNG_FAIL -13 +#define AF_ERROR_HASH_FAIL -14 +#define AF_ERROR_NO_SHA256 -15 + +#define AF_SIG_GOOD 0 +#define AF_ERROR_SIG_BAD -15 +#define AF_ERROR_SIG_NO_CERT -16 +#define AF_ERROR_SIG_CANNOT_READ_PUBLIC_KEY -17 +#define AF_ERROR_SIG_DATAREAD_ERROR -18 +#define AF_ERROR_SIG_MALLOC -19 +#define AF_ERROR_SIG_READ_ERROR -20 +#define AF_ERROR_SIG_SIG_SEG -21 // can't verify the signature on a signature segment +#define AF_ERROR_SIG_NOT_COMPILED -22 // afflib compiled without signature support + + +/* AFF environment variables */ +#define AFFLIB_CACHE_STATS "AFFLIB_CACHE_STATS" // make non-zero to dump stats to STDERR at end +#define AFFLIB_CACHE_DEBUG "AFFLIB_CACHE_DEBUG" // make "1" to dump a trace of cache events to stderr +#define AFFLIB_CACHE_PAGES "AFFLIB_CACHE_PAGES" // Size of the page cache +#define AFFLIB_CACHE_PAGES_DEFAULT 32 // default number of cache pages +#define AFFLIB_BIGTMP "AFFLIB_BIGTMP" // default directory to put very big files for test programs +#define AFFLIB_TRACEFILE "AFFLIB_TRACEFILE" // If set, send a record of all activity to the location +/* passphrases for single-key cryptography */ +#define AFFLIB_PASSPHRASE "AFFLIB_PASSPHRASE" +#define AFFLIB_PASSPHRASE_FILE "AFFLIB_PASSPHRASE_FILE" +#define AFFLIB_PASSPHRASE_FD "AFFLIB_PASSPHRASE_FD" + +/* passphrases for signing keys */ +#define AFFLIB_PEM_SIGNING_PASSPHRASE "AFFLIB_PEM_SIGNING_PASSPHRASE" + +/* passphrases for sealing keys */ + +#define AFFLIB_DECRYPTING_PRIVATE_KEYFILE "AFFLIB_DECRYPTING_PRIVATE_KEYFILE" + +extern FILE *af_trace; // fd to trace to + +/**************************************************************** + *** Not AFF functions at all, but placed here for convenience. + ****************************************************************/ +const char *af_hexbuf(char *dst,int dst_len,const unsigned char *bin,int bytes,int format_flag); + +/* af_hexbuf formats: */ +#define AF_HEXBUF_NO_SPACES 0 +#define AF_HEXBUF_SPACE2 0x0001 // space every 2 characters +#define AF_HEXBUF_SPACE4 0x0002 // space every 4 characters +#define AF_HEXBUF_UPPERCASE 0x1000 // uppercase +#define AF_HEXBUF_LINEBREAK 0x2000 // break every 80 cols + + +/**************************************************************** + *** Internal implementation details below. + ****************************************************************/ + + +#ifdef __never_defined__ +{ +#endif +#ifdef __cplusplus +} +#endif +#endif + + diff --git a/lib/afflib_i.h b/lib/afflib_i.h new file mode 100644 index 0000000..6208116 --- /dev/null +++ b/lib/afflib_i.h @@ -0,0 +1,704 @@ +/* + * afflib_i.h: + * The "master include file" of the AFF Library. + * Includes many fucntions that are not designed + * to be used by application programmers. + * + * + * Copyright (c) 2005-2006 + * Simson L. Garfinkel and Basis Technology, Inc. + * All rights reserved. + * + * This code is derrived from software contributed by + * Simson L. Garfinkel + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by Simson L. Garfinkel + * and Basis Technology Corp. + * 4. Neither the name of Simson Garfinkel, Basis Technology, or other + * contributors to this program may be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY SIMSON GARFINKEL, BASIS TECHNOLOGY, + * AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL SIMSON GARFINKEL, BAIS TECHNOLOGy, + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef AFFLIB_I_H +#define AFFLIB_I_H + +#ifdef KERNEL_LIBRARY +#ifdef __cplusplus +extern "C" { +#endif +void __cdecl AFDbgPrint (PCSTR Format,...); +#ifdef __cplusplus +} +#endif +#endif + + +/* Should we disable threading? */ +#ifdef DISABLE_PTHREAD +#undef HAVE_PTHREAD +#endif + +/* Standard includes */ +#ifdef HAVE_STRING_H +#include +#endif + +#ifdef HAVE_STDLIB_H +#include +#endif + +#ifdef HAVE_ZLIB_H +#include +#endif + +#ifdef HAVE_SYS_STAT_H +#include +#endif + +#ifdef HAVE_FCNTL_H +#include +#endif + +#ifdef HAVE_ASSERT_H +#include +#endif + +#ifdef HAVE_ERRNO_H +#include +#endif + +#ifdef HAVE_DIRENT_H +#include +#endif + +#ifdef HAVE_ERR_H +#include +#endif + +#ifdef HAVE_ALLOCA_H +#include +#endif + +#ifdef HAVE_LIBSSL +#include +#include // a standard part of OpenSSL +#include +#include +#include +#endif + +#ifdef HAVE_PTHREAD +#include +#define AF_READLOCK(af) pthread_rwlock_rdlock(&af->rwlock); +#define AF_WRLOCK(af) pthread_rwlock_wrlock(&af->rwlock); +#define AF_UNLOCK(af) pthread_rwlock_unlock(&af->rwlock); +#else +/* No threads */ +#define AF_READLOCK(af) {} +#define AF_WRLOCK(af) {} +#define AF_UNLOCK(af) {} +#endif + +#ifdef WIN32 +#if !defined(HAVE__MINGW_H) +#pragma warning(disable: 4996) /* Don't warn on Windows about using POSIX open() instead of _open() */ +#endif +#include +#include +#include // htonl() +#include +#define snprintf _snprintf +#define strcasecmp _stricmp +#define mkdir(path,mode) _mkdir(path) +#define random() rand() +#define access _access +#define strdup _strdup + +#ifndef ENOTSUP +#define ENOTSUP 65536 /* made up number */ +#endif + +#ifndef _MODE_T_ +#define _MODE_T_ +typedef unsigned short mode_t; +typedef unsigned short _mode_t; +#endif + +#ifndef S_ISDIR +#define S_ISDIR(m)(((m) & 0170000) == 0040000) +#endif + +#if !defined(HAVE__MINGW_H) +#define ftruncate(fd,size) _chsize_s(fd,size) +#define MAXPATHLEN 1024 +#endif + +#if defined(HAVE__MINGW_H) +#ifndef ftello +#define ftello ftello64 +#endif + +#ifndef fseeko +#define fseeko fseeko64 +#endif + +#else +#define ftello _ftelli64 /* replaces ftello64 in VC2008 */ +#define fseeko _fseeki64 +#endif + +#endif +/** END OF WIN32 DEFINES **/ + + +#ifdef HAVE_SYS_PARAM_H +#include +#endif + +#ifdef HAVE_ARPA_INET_H +#include +#endif + +#ifdef HAVE_UNISTD_H +#include +#endif + +/* Pick an appropriate POINTER_FMT; isn't there an ANSI standard for this? */ +#ifdef __APPLE__ +#define POINTER_FMT "%p" +#endif + +#ifdef linux +#define POINTER_FMT "%p" +#endif + +#ifndef POINTER_FMT +#define POINTER_FMT "%x" // guess +#endif + +/* Handle systems that are missing some #defines */ + +#ifndef O_BINARY +#define O_BINARY 0 // for Windows compatability +#endif + +#ifndef ENOTSUP +#define ENOTSUP EOPNOTSUPP +#endif + +#ifndef O_ACCMODE +#define O_ACCMODE 0x0003 +#endif + +/* If these functions do not exist, we need to create our own */ + +#ifndef HAVE_ERR +void err(int eval, const char *fmt, ...); +#endif + +#ifndef HAVE_ERRX +void errx(int eval, const char *fmt, ...); +#endif + +#ifndef HAVE_WARN +void warn(const char *fmt, ...); +#endif + +#ifndef HAVE_WARNX +void warnx(const char *fmt, ...); +#endif + +/* access function */ +#ifndef F_OK +#define F_OK 0 /* test for existence of file */ +#endif + +#ifndef X_OK +#define X_OK 0x01 /* test for execute or search permission */ +#endif + +#ifndef W_OK +#define W_OK 0x02 /* test for write permission */ +#endif + +#ifndef R_OK +#define R_OK 0x04 /* test for read permission */ +#endif + + +#if defined(WIN32) && !defined(HAVE__MINGW_H) +/**************************************************************** + *** Windows emulation of opendir()/readdir() + *** From php + ****************************************************************/ + +/* struct dirent - same as Unix */ + +struct dirent { + long d_ino; /* inode (always 1 in WIN32) */ + off_t d_off; /* offset to this dirent */ + int d_reclen; /* length of d_name; was unsigned short */ + char d_name[_MAX_FNAME + 1]; /* filename (null terminated) */ +}; + + +/* typedef DIR - not the same as Unix */ +typedef struct { + long handle; /* _findfirst/_findnext handle */ + short offset; /* offset into directory */ + short finished; /* 1 if there are not more files */ + struct _finddata_t fileinfo; /* from _findfirst/_findnext */ + char *dir; /* the dir we are reading */ + struct dirent dent; /* the dirent to return */ +} DIR; + +/* Function prototypes */ + + +#ifdef __cplusplus +extern "C" { +#endif + +DIR *opendir(const char *); +struct dirent *readdir(DIR *); +int readdir_r(DIR *, struct dirent *, struct dirent **); +int closedir(DIR *); +int rewinddir(DIR *); + + +#ifdef __cplusplus +} +#endif +#endif + + +/**************************************************************** + *** AFFLIB internal stuff follows. + ****************************************************************/ + +#ifdef __cplusplus +extern "C" { +#endif +#ifdef NEVER_DEFINED +} +#endif + +#if defined(HAVE_LIBEXPAT) +#define USE_AFFSIGS +#endif + +struct _AFFILE { + int version; // 2 + void *tag; // available to callers; unused by AFFLIB + + struct af_vnode *v; // which function table to use. + struct _AFFILE *parent; // for AFF file inside an AFD + + /* For all files */ + int openflags; // how it was opened + int openmode; // how we were asked to open it; more + int exists; // did file exist before open was called? + + /* From URLs */ + char *fname; // Filename of file; be sure to free when done + char *protocol; // could be "file" or "s3" + char *username; // optionally specified in URL + char *password; // from URL; erase after use. + char *hostname; // from URL + int port; // from URL + + /* Extended Logging */ + char error_str[64]; // what went wrong + + /* Implement a stream abstraction */ + uint64_t image_size; // last mappable byte of disk image + uint64_t image_size_in_file; // see if it was changed... + uint32_t image_pagesize; // the size of image data segments in this file + uint32_t image_sectorsize; + uint64_t pos; // location in stream; should be signed because of comparisons + + /* Page buffer cache */ + struct aff_pagebuf *pb; // the current page buffer + struct aff_pagebuf *pbcache; // array of pagebufs + int num_pbufs; // number of pagebufs; default is 1 + int afftime; // for updating last + int64_t cur_page; // used by vnode_raw to fake pages must be able to go negative. + + int debug; // for debugging, of course + unsigned int badflag_set:1; // is badflag set? + unsigned char *badflag; // bad sector flag + + + /****************************************************************/ + /* Right now the instance variables for each implementation are here, + * which is ugly but easier for development... + */ + + /* For AFF Segment Files; this could be moved into private storage... */ + FILE *aseg; + struct aff_toc_mem *toc; // table of contents + int toc_count; // number of directory elements + + /****************************************************************/ + + unsigned int write_md5:1; // automatically write the MD5 for each page + unsigned int write_sha1:1; + unsigned int write_sha256:1; + + + /* These are for optimizing updates; really this should go away and we should just + * exmaine the TOC to find a hole, but currently we don't do that. + */ + unsigned int direction_changes; // how many times have we changed directions? + int last_direction; // should be 1 or -1 + unsigned int random_access:1; // are we in random access mode? + + /* additional support for writing. */ + unsigned int compression_type; // preferred compression type + int compression_level; // 0 is no compression + + + /* w_callback: + * A callback that is called before and after each segment is written. + * Called with the arguments (i,0,0) at the beginning of the write operation. + * Called with the arguments (i,j,k) at the end of the write operation. + * i = segment number + * j = segment length + * If segment is being written with compresison, k = compressed length. + * If segment is written w/o compression, k = 0 + */ + void (*w_callback)(struct affcallback_info *acbi); + // called at start and end of compression. + + uint64_t maxsize; // maximum file size of a multi-segment files, + // or 0 if this is not a multi-segment file + + /* Performance Counters */ + uint64_t bytes_memcpy; // total number of bytes memcpy'ed + uint64_t pages_written; // total number of pages written + uint64_t pages_compressed; // total number of pages compressed + uint64_t pages_decompressed; + uint64_t pages_read; // total number of pages read + uint64_t bytes_written; + uint64_t cache_hits; + uint64_t cache_misses; // total number of pages flushed from cache + + void *vnodeprivate; // private storage for the vnode + void (*error_reporter)(const char *fmt, ...); + struct af_crypto *crypto; +#ifdef HAVE_PTHREAD + pthread_rwlock_t rwlock; // automatically created and destroyed if pthread exists +#endif + struct af_vnode_info *vni_cache; // vstat cache +}; + + + +/* af_crypto: + * copy of AES encrypt and decrypt keys. + */ +void af_crypto_allocate(AFFILE *af); +void af_crypto_deallocate(AFFILE *af); + +struct af_crypto { + uint32_t sealing_key_set:1; // encryption key has been set + uint32_t auto_encrypt:1; // encrypt segments when we write + uint32_t auto_decrypt:1; // automatically decrypto when we read +#ifdef AES_BLOCK_SIZE + AES_KEY ekey; // encrypt key + AES_KEY dkey; // decrypt key +#endif +#ifdef HAVE_OPENSSL_EVP_H + EVP_PKEY *sign_privkey; // signing private key (to write signatures) + EVP_PKEY *sign_pubkey; // signing public key (to verify signatures) + X509 *sign_cert; // signing certificate (for verifying signatures) + /* Sealing is kept locally and immediately turned into a dkey & ekey */ +#endif +}; + + +/* The AFF STREAM VNODE */ +struct af_vnode { + int type; // numeric vnode type + int flag; // file system flag type + const char *name; + int (*identify)(const char *fname,int exists); // returns 1 if file system is identified by implementation; + int (*open)(AFFILE *af); + int (*close)(AFFILE *af); + int (*vstat)(AFFILE *af,struct af_vnode_info *); // returns info about the vnode image file + int (*get_seg)(AFFILE *af,const char *name,uint32_t *arg, uint8_t *data,size_t *datalen); + int (*get_next_seg)(AFFILE *af,char *segname,size_t segname_len, + uint32_t *arg, uint8_t *data, size_t *datalen); + int (*rewind_seg)(AFFILE *af); + int (*update_seg)(AFFILE *af,const char *name,uint32_t arg, + const uint8_t *value,uint32_t vallen); + int (*del_seg)(AFFILE *af,const char *name); + int (*read)(AFFILE *af,uint8_t *buf,uint64_t offset,size_t count); + int (*write)(AFFILE *af,uint8_t *buf,uint64_t offset,size_t count); +}; + +/* VNODE Flags */ +#define AF_VNODE_TYPE_PRIMITIVE 0x01 // single-file implementation +#define AF_VNODE_TYPE_COMPOUND 0x02 // multi-file implementation +#define AF_VNODE_TYPE_RELIABLE 0x04 // writes are reliable; no need to verify them. +#define AF_VNODE_MAXSIZE_MULTIPLE 0x08 // maxsize must be multiple of pagesize (for AFM and splitraw) +#define AF_VNODE_NO_SIGNING 0x10 // vnode does not support signing (like raw) +#define AF_VNODE_NO_SEALING 0x20 // vnode does not support sealing (like raw and afd) + +#define AF_SEALING_VNODE(af) (!(af->v->flag & AF_VNODE_NO_SEALING)) +#define AF_SIGNING_VNODE(af) (!(af->v->flag & AF_VNODE_NO_SIGNING)) +#define AF_SEALING(af) AF_SEALING_VNODE(af) && af->crypto && af->crypto->sealing_key_set + + +/* The header for an AFF file. All binary numbers are stored in network byte order. */ +#define AF_HEADER "AFF10\r\n\000" +struct af_head { + char header[8]; // "AFF10\r\n\000" + /* segments follow */ +}; + + +/* The header of each segment */ +#define AF_SEGHEAD "AFF\000" +struct af_segment_head { + char magic[4]; // "AFF\000" + uint32_t name_len:32; // length of segment name + uint32_t data_len:32; // length of segment data, if any + uint32_t flag:32; // argument for name; + /* name follows, then data */ +}; + +/* The tail of each segment */ +#define AF_SEGTAIL "ATT\000" +struct af_segment_tail { + char magic[4]; // "ATT\000" + uint32_t segment_len:32; // includes head, tail, name & length +}; + + +/* How 64-bit values are stored in a segment */ +#pragma pack(1) +struct aff_quad { + uint32_t low:32; + uint32_t high:32; +}; +#pragma pack() + + +/* As it is kept in memory */ +struct aff_toc_mem { + char *name; // name of this directory entry + uint64_t offset; // offset from beginning of file. + uint64_t segment_len; // includes head, tail, name & length +}; + +/* How encryption keys are stored */ +struct affkey { + uint8_t version[4]; + uint8_t affkey_aes256[32]; // AFF key encrypted with SHA-256 of passphrase + // encrypted as two codebooks in a row; no need for CBC + uint8_t zeros_aes256[16]; // all zeros encrypted with SHA-256 of passphrase +}; +#define AFFKEY_SIZE 4+32+16 + + +void af_initialize(); // initialize the AFFLIB + // automatically called by af_open() + +/* Internal identification routines */ +int af_identify_file_type(const char *filename,int exists); // returns type of a file; if exists=1, file must exist +const char *af_identify_file_name(const char *filename,int exists); // returns name of a file type; +int split_raw_increment_fname (char *fn); /* exposed for testing in aftest */ + +/* AFF implementation types returned by af_identify_type() and af_identify_name()*/ + +#define AF_IDENTIFY_RAW 0 // file is a raw file +#define AF_IDENTIFY_AFF 1 // file is an AFF file +#define AF_IDENTIFY_AFD 2 // file is a directory of AFF files +#define AF_IDENTIFY_EVF 3 // file is an EnCase file +#define AF_IDENTIFY_EVD 4 // file is a .E01 file when there are more files following +#define AF_IDENTIFY_SPLIT_RAW 5 // file is a split raw file +#define AF_IDENTIFY_AFM 6 // file is raw file with metadata +#define AF_IDENTIFY_EWF 7 // libewf; deprecated +#define AF_IDENTIFY_S3 8 // is an s3:/// file +#define AF_IDENTIFY_VMDK 9 // QEMU support for VMDK format +#define AF_IDENTIFY_DMG 10 // QEMU support for Apple DMG format +#define AF_IDENTIFY_SPARSEIMAGE 11 // QEMU support for Apple SPARSEIMAGE format + + +#define AF_IDENTIFY_ERR -1 // error encountered on identify +#define AF_IDENTIFY_NOEXIST -2 // file does not exist + + +AFFILE *af_open_with(const char *filename,int flags,int mode, struct af_vnode *v); +extern struct af_vnode *af_vnode_array[]; // array of filesystems; last is a "0" + +int af_last_seg(AFFILE *af,char *last_segname,int last_segname_len,int64_t *pos); +int af_make_badflag(AFFILE *af); // creates a badflag and puts it +int af_make_gid(AFFILE *af); // created an AF_IMAGE_GID if it doesn't exist +extern char af_error_str[64]; + + +#define AFF_DEFAULT_PAGESIZE (1024*1024*16) + + +/* afflib_os.cpp: + * Operating-system specific code. + */ + +/* af_figure_media: + * Returns information about the media in a structure. + * Returns 0 if successful, -1 if error. + */ + +struct af_figure_media_buf { + int version; + int sector_size; + uint64_t total_sectors; + uint64_t max_read_blocks; // was previously 4-bytes; must be 8! +}; +int af_figure_media(int fd,struct af_figure_media_buf *); + +/**************************************************************** + *** Lowest-level routines for manipulating the AFF File... + ****************************************************************/ + +/* Navigating within the AFFILE */ +/* probe the next segment. + * Returns: 0 if success + * -1 if error + * -2 if segname_len was not large enough to hold segname + * - segname - the name of the next segment. + * - segsize - number of bytes the entire segment is. + * + * doesn't change af->aseg pointer if do_rewind is true, otherwise leaves stream + * positioned ready to read the data + */ + +int af_probe_next_seg(AFFILE *af,char *segname,size_t segname_len, + uint32_t *arg,size_t *datasize, size_t *segsize,int do_rewind); +int af_backspace(AFFILE *af); // back up one segment + + + +/**************************************************************** + *** Reading functions + ****************************************************************/ + + +/* Support for data pages. This is what the stream system is built upon. + * Note: pagename to string translation happens inside afflib.cpp, not inside + * the vnode driver. + */ +int af_page_size(AFFILE *af); // legacy (now is af_get_pagesize) +void af_read_sizes(AFFILE *af); // sets up values if we can get them. +int af_set_pagesize(AFFILE *af,uint32_t pagesize); // sets the pagesize; fails with -1 if imagesize >=0 +int af_set_sectorsize(AFFILE *AF,int sectorsize); // fails with -1 if imagesize>=0 +int af_get_sectorsize(AFFILE *AF); // returns sector size +int af_has_pages(AFFILE *af); // does the underlying system support pages? +int af_get_pagesize(AFFILE *af); // returns page size, or -1 +int af_get_page_raw(AFFILE *af,int64_t pagenum,uint32_t *arg,uint8_t *data,size_t *bytes); +int af_get_page(AFFILE *af,int64_t pagenum,uint8_t *data,size_t *bytes); +#define AF_SIGFLAG_NOSIG 0x0001 // do not write signatures with af_update_segf() +#define AF_SIGFLAG_NOSEAL 0x0002 // do not encrypt an af_update_segf() + +/**************************************************************** + *** Writing functions + ****************************************************************/ + +extern int af_cache_debug; // sets level of verbosity */ +int af_set_maxsize(AFFILE *af,int64_t size); // sets maximum AFF file size +int af_update_page(AFFILE *af,int64_t pagenum,uint8_t *data,int datalen); +int af_update_segf(AFFILE *af,const char *name, + uint32_t arg,const uint8_t *value,uint32_t vallen,uint32_t sigflag); + +void af_invalidate_vni_cache(AFFILE *af); +void af_cache_writethrough(AFFILE *af,int64_t pagenum, + const uint8_t *buf,int bufflen); +int af_cache_flush(AFFILE *af); // write buffers to disk +struct aff_pagebuf *af_cache_alloc(AFFILE *af,int64_t pagenum); + + +/**************************************************************** + ***/ + +/* afflib_util.cpp + */ +uint64_t af_decode_q(uint8_t buf[8]); // return buf[8] into an unsigned quad +const char *af_commas(char buf[64],int64_t val); +int af_hasmeta(const char *buf); // return 1 if buf has shell metacharacters +int af_is_filestream(const char *filename); // return 1 if file:// or filename +void af_parse_url(const char *url,char **protocol,char **hostname, + char **username,char **password,int *port,char **path); + +#ifndef HAVE_STRLCPY +size_t strlcpy(char *dest,const char *src,size_t dest_size); +#endif + +#ifndef HAVE_STRLCAT +size_t strlcat(char *dest,const char *src,size_t dest_size); +#endif + + +/**************************************************************** + *** Table of Contents + ****************************************************************/ + +/* Needs a rewrite for efficiency */ + +/* afflib_toc.cpp: + * Table of contents management routines + * Remember: all of these routines may fail, because the whole TOC may not + * fit in memory... + * + * This is all experimental right now. + */ + +int aff_segment_overhead(const char *segname); +int aff_toc_free(AFFILE *af); +void aff_toc_print(AFFILE *af); +int aff_toc_build(AFFILE *af); // build by scanning the AFFILE +struct aff_toc_mem *aff_toc(AFFILE *af,const char *segname); +int aff_toc_del(AFFILE *af,const char *segname); +void aff_toc_update(AFFILE *af,const char *segname,uint64_t offset,uint64_t datalen); + +/* lzma_glue.cpp: + * For the LZMA compression engine + */ +int lzma_compress(uint8_t *dest,size_t *destLen, const uint8_t *data,size_t datalen,int level); +int lzma_uncompress(uint8_t *buf,size_t *buflen, const uint8_t *cbuf,size_t cbuf_size); + +#ifdef NEVER_DEFINED +{ +#endif +#ifdef __cplusplus +} +#endif +#endif + + diff --git a/lib/afflib_os.cpp b/lib/afflib_os.cpp new file mode 100644 index 0000000..ecd2ab9 --- /dev/null +++ b/lib/afflib_os.cpp @@ -0,0 +1,172 @@ +/* + * afflib_os.cpp: + * + * The OS-specific features of AFFLIB + * + * This file is a work of a US government employee and as such is in the Public domain. + * Simson L. Garfinkel, March 12, 2012 + */ + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" + +#ifdef HAVE_SYS_PARAM_H +#include +#endif + +#if defined(__FreeBSD_version) && __FreeBSD_version<500000 +#undef HAVE_SYS_DISK_H +#endif + +#if defined(HAVE_SYS_DISK_H) +#include +#endif + +#ifdef HAVE_LINUX_FS_H +#include +#endif + +#ifdef HAVE_SYS_IOCTL_H +#include // needed for Linux +#endif + +/**************************************************************** + *** Extra code for Windows... + ****************************************************************/ + +/* No longer needed with VC2008 */ +#if 0 +#if defined(WIN32) and !defined(__MINGW_H) +#pragma warning(disable: 4996) +int64 ftello(FILE *stream) +{ + fpos_t pos; + if(fgetpos(stream,&pos) != 0){ + return -1; + } + return pos; +} + +int fseeko(FILE *stream,int64 offset,int whence) +{ + switch(whence){ + case SEEK_SET: + break; // jump down to fsetpos() + case SEEK_CUR: + offset += ftello(stream); + break; + case SEEK_END: + fseek(stream,0L,SEEK_END); // go to the end; hope this works for big files + offset = ftello(stream) - offset; + break; + default: + return -1; + } + return fsetpos(stream,&offset); +} +#endif +#endif + + + + +/****************************************************************/ + + +/* af_figure_media(): + * + * This is a function that returns, for the file handle of an open + * device, the sector_size, total_sectors, and maximum number of + * blocks that can be read at a time (or 0 if there is no max). There + * is a generic implementation at the bottom, and operating-specific + * versions above. + */ + + +int af_figure_media(int fd,struct af_figure_media_buf *afb) +{ + memset(afb,0,sizeof(*afb)); + afb->version = 1; + +#ifdef __APPLE__ +#define MEDIA_FIGURED + if(ioctl(fd,DKIOCGETBLOCKSIZE,&afb->sector_size)){ + afb->sector_size = 512; // assume 512 + } + if(ioctl(fd,DKIOCGETBLOCKCOUNT,&afb->total_sectors)){ + afb->total_sectors=0; // seeking not allowed on stdin + } + if(ioctl(fd,DKIOCGETMAXBLOCKCOUNTREAD,&afb->max_read_blocks)){ + afb->max_read_blocks = 0; // read all you want + } +#endif +#if defined(__FreeBSD__) && defined(DIOCGSECTORSIZE) +#define MEDIA_FIGURED + if(ioctl(fd,DIOCGSECTORSIZE,&afb->sector_size)){ + afb->sector_size = 512; // can't figure it out; go with the default + } + off_t inbytes=0; + if(ioctl(fd,DIOCGMEDIASIZE,&inbytes)){ + afb->total_sectors = 0; + } + if(inbytes % afb->sector_size != 0){ + fprintf(stderr,"ioctl(DIOCGSECTORSIZE) returns %d bytes\n", afb->sector_size); + fprintf(stderr,"ioctl(DIOCGMEDIASIZE) returns %d bytes\n", inbytes); + fprintf(stderr,"which is not an even number of sectors.\n"); + return -1; + } + afb->total_sectors = inbytes / afb->sector_size; + afb->max_read_blocks = 0; +#endif +#ifdef linux +#define MEDIA_FIGURED + /* With Linux, we figure out how many bytes there are and get sector size + * from a #define + */ + + afb->sector_size = BLOCK_SIZE; +#ifdef BLKGETSIZE64 + uint64_t total_bytes=0; + if(ioctl(fd,BLKGETSIZE64,&total_bytes)){ + total_bytes = 0; + } +#else + int total_bytes=0; + if(ioctl(fd,BLKGETSIZE,&total_bytes)){ + total_bytes = 0; + } +#endif + + + afb->total_sectors = total_bytes / afb->sector_size; + afb->max_read_blocks = 0; +#endif +#ifndef MEDIA_FIGURED + + /* Unknown OS type. Try our best-effort guess, + */ +#ifdef BLOCK_SIZE + afb->sector_size = BLOCK_SIZE; // it's a good guess +#else + afb->sector_size = 512; // it's a good guess +#endif + + /* Try seeking to the end of fd and ask where we are! */ + + off_t start_pos = lseek(fd,0,SEEK_CUR); // find where we are + off_t end_of_file = lseek(fd,0,SEEK_END); + + if(end_of_file==-1){ + end_of_file = 0; // can't figure it. + } + lseek(fd,start_pos,SEEK_SET); // go back to the starting position + + afb->total_sectors = end_of_file / afb->sector_size; + afb->max_read_blocks = 0; +#endif + return 0; +} + + + diff --git a/lib/afflib_pages.cpp b/lib/afflib_pages.cpp new file mode 100644 index 0000000..362f9a5 --- /dev/null +++ b/lib/afflib_pages.cpp @@ -0,0 +1,631 @@ +/* + * The AFFLIB page abstraction. + * Distributed under the Berkeley 4-part license + */ + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" + + +/* af_read_sizes: + * Get the page sizes if they are set in the file. + */ +void af_read_sizes(AFFILE *af) +{ + af_get_seg(af,AF_SECTORSIZE,&af->image_sectorsize,0,0); + if(af->image_sectorsize==0) af->image_sectorsize = 512; // reasonable default + + if(af_get_seg(af,AF_PAGESIZE,&af->image_pagesize,0,0)){ + af_get_seg(af,AF_SEGSIZE_D,&af->image_pagesize,0,0); // try old name + } + + /* Read the badflag if it is present */ + size_t sectorsize = af->image_sectorsize; + if(af->badflag==0) af->badflag = (unsigned char *)malloc(sectorsize); + if(af_get_seg(af,AF_BADFLAG,0,af->badflag,(size_t *)§orsize)==0){ + af->badflag_set = 1; + } + + /* Read the image file segment if it is present. + * If it isn't, scan through the disk image to figure out the size of the disk image. + */ + + if(af_get_segq(af,AF_IMAGESIZE,(int64_t *)&af->image_size)){ + + /* Calculate the imagesize by scanning all of the pages that are in + * the disk image and finding the highest page number. + * Then read that page to find the last allocated byte. + */ + char segname[AF_MAX_NAME_LEN]; + size_t datalen = 0; + af_rewind_seg(af); // start at the beginning + int64_t highest_page_number = 0; + while(af_get_next_seg(af,segname,sizeof(segname),0,0,&datalen)==0){ + if(segname[0]==0) continue; // ignore sector + int64_t pagenum = af_segname_page_number(segname); + if(pagenum > highest_page_number) highest_page_number = pagenum; + } + size_t highest_page_len = 0; + if(af_get_page(af,highest_page_number,0,&highest_page_len)==0){ + af->image_size = af->image_pagesize * highest_page_number + highest_page_len; + } + } + af->image_size_in_file = af->image_size; +} + + +int af_page_size(AFFILE *af) +{ + return af->image_pagesize; +} + +int af_get_pagesize(AFFILE *af) +{ + return af->image_pagesize; +} + +/* af_set_sectorsize: + * Sets the sectorsize. + * Fails with -1 if imagesize >=0 unless these changes permitted + */ +int af_set_sectorsize(AFFILE *af,int sectorsize) +{ + struct af_vnode_info vni; + af_vstat(af,&vni); + if(vni.changable_pagesize==0 && af->image_size>0){ + errno = EINVAL; + return -1; + } + af->image_sectorsize =sectorsize; + if(af->badflag==0) af->badflag = (unsigned char *)malloc(sectorsize); + else af->badflag = (unsigned char *)realloc(af->badflag,sectorsize); + af->badflag_set = 0; + + if(af_update_seg(af,AF_SECTORSIZE,sectorsize,0,0)){ + if(errno != ENOTSUP) return -1; + } + return 0; +} + +int af_get_sectorsize(AFFILE *af) // returns sector size +{ + return af->image_sectorsize; +} + +/* + * af_set_pagesize: + * Sets the pagesize. Fails with -1 if it can't be changed. + */ +int af_set_pagesize(AFFILE *af,uint32_t pagesize) +{ + /* Allow the pagesize to be changed if it hasn't been set yet + * and if this format doesn't support metadata updating (which is the raw formats) + */ + struct af_vnode_info vni; + + af_vstat(af,&vni); + + if(vni.changable_pagesize==0 && af->image_size>0){ + if(pagesize==af->image_pagesize) return 0; // it's already set to this, so let it pass + errno = EINVAL; + return -1; + } + if(pagesize % af->image_sectorsize != 0){ + (*af->error_reporter)("Cannot set pagesize to %d (sectorsize=%d)\n", + pagesize,af->image_sectorsize); + errno = EINVAL; + return -1; + } + + af->image_pagesize = pagesize; + if(af_update_seg(af,AF_PAGESIZE,pagesize,0,0)){ + if(errno != ENOTSUP) return -1; // error updating (don't report ENOTSUP); + } + return 0; +} + + +/**************************************************************** + *** page-level interface + ****************************************************************/ + +int af_get_page_raw(AFFILE *af,int64_t pagenum,uint32_t *arg, + unsigned char *data,size_t *bytes) +{ + char segname[AF_MAX_NAME_LEN]; + + memset(segname,0,sizeof(segname)); + sprintf(segname,AF_PAGE,pagenum); + int r = af_get_seg(af,segname,arg,data,bytes); + if(r!=0){ + /* Couldn't read with AF_PAGE; try AF_SEG_D. + * This is legacy for the old AFF files. Perhaps we should delete it. + */ + sprintf(segname,AF_SEG_D,pagenum); + r = af_get_seg(af,segname,arg,data,bytes); + } + /* Update the counters */ + if(r==0 && bytes && *bytes>0) af->pages_read++; // note that we read a page + return r; +} + +/* af_get_page: + * Get a page from its named segment. + * If the page is compressed, uncompress it. + * data points to a segmenet of at least *bytes; + * *bytes is then modified to indicate the actual amount of bytes read. + * Return 0 if success, -1 if fail. + */ + +int af_get_page(AFFILE *af,int64_t pagenum,unsigned char *data,size_t *bytes) +{ + uint32_t arg=0; + size_t page_len=0; + + if (af_trace){ + fprintf(af_trace,"af_get_page(%p,pagenum=%"I64d",buf=%p,bytes=%u)\n",af,pagenum,data,(int)*bytes); + } + + /* Find out the size of the segment and if it is compressed or not. + * If we can't find it with new nomenclature, try the old one... + */ + int r = af_get_page_raw(af,pagenum,&arg,0,&page_len); + if(r){ + /* Segment doesn't exist. + * If we have been provided with a buffer, + * fill buffer with the 'bad segment' flag and return. + */ + if (data && (af->openmode&AF_BADBLOCK_FILL)) { + for(size_t i = 0;i <= af->image_pagesize - af->image_sectorsize; + i+= af->image_sectorsize){ + memcpy(data+i,af->badflag,af->image_sectorsize); + af->bytes_memcpy += af->image_sectorsize; + } + } + return r; // segment doesn't exist + } + + + /* If the segment isn't compressed, just get it*/ + uint32_t pageflag = 0; + if((arg & AF_PAGE_COMPRESSED)==0){ + if(data==0){ // if no data provided, just return size of the segment if requested + if(bytes) *bytes = page_len; // set the number of bytes in the page if requested + return 0; + } + int ret = af_get_page_raw(af,pagenum,&pageflag,data,bytes); + if(*bytes > page_len) *bytes = page_len; // we only read this much + if(ret!=0) return ret; // some error happened? + } + else { + /* Allocate memory to hold the compressed segment */ + unsigned char *compressed_data = (unsigned char *)malloc(page_len); + size_t compressed_data_len = page_len; + if(compressed_data==0){ + return -2; // memory error + } + + /* Get the data */ + if(af_get_page_raw(af,pagenum,&pageflag,compressed_data,&compressed_data_len)){ + free(compressed_data); + return -3; // read error + } + + /* Now uncompress directly into the buffer provided by the caller, unless the caller didn't + * provide a buffer. If that happens, allocate our own... + */ + int res = -1; // 0 is success + bool free_data = false; + if(data==0){ + data = (unsigned char *)malloc(af->image_pagesize); + free_data = true; + *bytes = af->image_pagesize; // I can hold this much + } + + switch((pageflag & AF_PAGE_COMP_ALG_MASK)){ + case AF_PAGE_COMP_ALG_ZERO: + if(compressed_data_len != 4){ + (*af->error_reporter)("ALG_ZERO compressed data is %d bytes, expected 4.",compressed_data_len); + break; + } + memset(data,0,af->image_pagesize); + *bytes = ntohl(*(long *)compressed_data); + res = 0; // not very hard to decompress with the ZERO compressor. + break; + + case AF_PAGE_COMP_ALG_ZLIB: + res = uncompress(data,(uLongf *)bytes,compressed_data,compressed_data_len); + switch(res){ + case Z_OK: + break; + case Z_ERRNO: + (*af->error_reporter)("Z_ERRNOR decompressing segment %"I64d,pagenum); + case Z_STREAM_ERROR: + (*af->error_reporter)("Z_STREAM_ERROR decompressing segment %"I64d,pagenum); + case Z_DATA_ERROR: + (*af->error_reporter)("Z_DATA_ERROR decompressing segment %"I64d,pagenum); + case Z_MEM_ERROR: + (*af->error_reporter)("Z_MEM_ERROR decompressing segment %"I64d,pagenum); + case Z_BUF_ERROR: + (*af->error_reporter)("Z_BUF_ERROR decompressing segment %"I64d,pagenum); + case Z_VERSION_ERROR: + (*af->error_reporter)("Z_VERSION_ERROR decompressing segment %"I64d,pagenum); + default: + (*af->error_reporter)("uncompress returned an invalid value in get_segment"); + } + break; + +#ifdef USE_LZMA + case AF_PAGE_COMP_ALG_LZMA: + res = lzma_uncompress(data,bytes,compressed_data,compressed_data_len); + if (af_trace) fprintf(af_trace," LZMA decompressed page %"I64d". %d bytes => %u bytes\n", + pagenum,(int)compressed_data_len,(int)*bytes); + switch(res){ + case 0:break; // OK + case 1:(*af->error_reporter)("LZMA header error decompressing segment %"I64d"\n",pagenum); + break; + case 2:(*af->error_reporter)("LZMA memory error decompressing segment %"I64d"\n",pagenum); + break; + } + break; +#endif + + default: + (*af->error_reporter)("Unknown compression algorithm 0x%d", + pageflag & AF_PAGE_COMP_ALG_MASK); + break; + } + + if(free_data){ + free(data); + data = 0; // restore the way it was + } + free(compressed_data); // don't need this one anymore + af->pages_decompressed++; + if(res!=Z_OK) return -1; + } + + /* If the page size is larger than the sector_size, + * make sure that the rest of the sector is zeroed, and that the + * rest after that has the 'bad block' notation. + */ + if(data && (af->image_pagesize > af->image_sectorsize)){ + const int SECTOR_SIZE = af->image_sectorsize; // for ease of typing + size_t bytes_left_in_sector = (SECTOR_SIZE - (*bytes % SECTOR_SIZE)) % SECTOR_SIZE; + for(size_t i=0;iimage_pagesize-SECTOR_SIZE; i+=SECTOR_SIZE){ + memcpy(data+i,af->badflag,SECTOR_SIZE); + af->bytes_memcpy += SECTOR_SIZE; + } + } + return 0; +} + + +static bool is_buffer_zero(unsigned char *buf,int buflen) +{ + for(int i=0;icrypto && af->crypto->sign_privkey){ + af_sign_seg3(af,segname_buf,0,data,datalen,AF_SIGNATURE_MODE1); + } +#endif + +#ifdef HAVE_MD5 + /* Write out MD5 if requested */ + if(af->write_md5){ + unsigned char md5_buf[16]; + char md5name_buf[32]; + MD5(data,datalen,md5_buf); + snprintf(md5name_buf,sizeof(md5name_buf),AF_PAGE_MD5,pagenum); + af_update_segf(af,md5name_buf,0,md5_buf,sizeof(md5_buf),AF_SIGFLAG_NOSIG); // ignore failure + } +#endif +#ifdef HAVE_SHA1 + /* Write out SHA1 if requested */ + if(af->write_sha1){ + unsigned char sha1_buf[20]; + char sha1name_buf[32]; + SHA1(data,datalen,sha1_buf); + snprintf(sha1name_buf,sizeof(sha1name_buf),AF_PAGE_SHA1,pagenum); + af_update_segf(af,sha1name_buf,0,sha1_buf,sizeof(sha1_buf),AF_SIGFLAG_NOSIG); // ignore failure + } +#endif + /* Write out SHA256 if requested and if SHA256 is available */ + if(af->write_sha256){ + unsigned char sha256_buf[32]; + if(af_SHA256(data,datalen,sha256_buf)==0){ + char sha256name_buf[32]; + snprintf(sha256name_buf,sizeof(sha256name_buf),AF_PAGE_SHA256,pagenum); + af_update_segf(af,sha256name_buf,0,sha256_buf,sizeof(sha256_buf),AF_SIGFLAG_NOSIG); // ignore failure + } + } + + /* Check for bypass */ + if(af->v->write){ + int r = (*af->v->write)(af,data,af->image_pagesize * pagenum,datalen); + if(r!=datalen) return -1; + return 0; + } + + struct affcallback_info acbi; + int ret = 0; + uint64_t starting_pages_written = af->pages_written; + + /* Setup the callback structure */ + memset(&acbi,0,sizeof(acbi)); + acbi.info_version = 1; + acbi.af = af->parent ? af->parent : af; + acbi.pagenum = pagenum; + acbi.bytes_to_write = datalen; + + size_t destLen = af->image_pagesize; // it could be this big. + + /* Compress and write the data, if we are allowed to compress */ + if(af->compression_type != AF_COMPRESSION_ALG_NONE){ + unsigned char *cdata = (unsigned char *)malloc(destLen); // compressed data + uint32_t *ldata = (uint32_t *)cdata; // allows me to reference as a buffer of uint32_ts + if(cdata!=0){ // If data could be allocated + int cres = -1; // compression results + uint32_t flag = 0; // flag for data segment + int dont_compress = 0; + + /* Try zero compression first; it's the best algorithm we have */ + if(is_buffer_zero(data,datalen)){ + acbi.compression_alg = AF_PAGE_COMP_ALG_ZERO; + acbi.compression_level = AF_COMPRESSION_MAX; + + if(af->w_callback) { acbi.phase = 1; (*af->w_callback)(&acbi); } + + *ldata = htonl(datalen); // store the data length + destLen = 4; // 4 bytes + flag = AF_PAGE_COMPRESSED | AF_PAGE_COMP_ALG_ZERO | AF_PAGE_COMP_MAX; + cres = 0; + + acbi.compressed = 1; // it was compressed + if(af->w_callback) {acbi.phase = 2;(*af->w_callback)(&acbi);} + } + +#ifdef USE_LZMA + if(cres!=0 && af->compression_type==AF_COMPRESSION_ALG_LZMA){ // try to compress with LZMA + acbi.compression_alg = AF_PAGE_COMP_ALG_LZMA; + acbi.compression_level = 7; // right now, this is the level we use + if(af->w_callback) { acbi.phase = 1; (*af->w_callback)(&acbi); } + + cres = lzma_compress(cdata,&destLen,data,datalen,9); +#if 0 + switch(cres){ + case 0:break; // OKAY + case 1: (*af->error_reporter)("LZMA: Unspecified Error\n");break; + case 2: (*af->error_reporter)("LZMA: Memory Allocating Error\n");break; + case 3: (*af->error_reporter)("LZMA: Output buffer OVERFLOW\n"); break; + default: (*af->error_reporter)("LZMA: Unknown error %d\n",cres);break; + } +#endif + if(cres==0){ + flag = AF_PAGE_COMPRESSED | AF_PAGE_COMP_ALG_LZMA; + acbi.compressed = 1; + if(af->w_callback) {acbi.phase = 2;(*af->w_callback)(&acbi);} + } + else { + /* Don't bother reporting LZMA errors; we just won't compress */ + dont_compress = 1; + if(af->w_callback) {acbi.phase = 2;(*af->w_callback)(&acbi);} + } + } +#endif + + if(cres!=0 + && af->compression_type==AF_COMPRESSION_ALG_ZLIB + && dont_compress==0){ // try to compress with zlib + acbi.compression_alg = AF_PAGE_COMP_ALG_ZLIB; // only one that we support + acbi.compression_level = af->compression_level; + if(af->w_callback) { acbi.phase = 1; (*af->w_callback)(&acbi); } + + cres = compress2((Bytef *)cdata, (uLongf *)&destLen, + (Bytef *)data,datalen, af->compression_level); + + if(cres==0){ + flag = AF_PAGE_COMPRESSED | AF_PAGE_COMP_ALG_ZLIB; + if(af->compression_level == AF_COMPRESSION_MAX){ + flag |= AF_PAGE_COMP_MAX; // useful to know it can't be better + } + } + acbi.compressed = 1; // it was compressed (or not compressed) + if(af->w_callback) {acbi.phase = 2;(*af->w_callback)(&acbi);} + } + + if(cres==0 && destLen < af->image_pagesize){ + /* Prepare to write out the compressed segment with compression */ + if(af->w_callback) {acbi.phase = 3;(*af->w_callback)(&acbi);} + ret = af_update_segf(af,segname_buf,flag,cdata,destLen,AF_SIGFLAG_NOSIG); + acbi.bytes_written = destLen; + if(af->w_callback) {acbi.phase = 4;(*af->w_callback)(&acbi);} + if(ret==0){ + af->pages_written++; + af->pages_compressed++; + } + } + free(cdata); + cdata = 0; + } + } + + /* If a compressed segment was not written, write it uncompressed */ + if(af->pages_written == starting_pages_written){ + if(af->w_callback) {acbi.phase = 3;(*af->w_callback)(&acbi);} + ret = af_update_segf(af,segname_buf,0,data,datalen,AF_SIGFLAG_NOSIG); + acbi.bytes_written = datalen; + if(af->w_callback) {acbi.phase = 4;(*af->w_callback)(&acbi);} + if(ret==0){ + acbi.bytes_written = datalen; // because that is how much we wrote + af->pages_written++; + } + } + return ret; +} + +/**************************************************************** + *** Cache interface + ****************************************************************/ + +/* The page cache is a read/write cache. + * + * Pages that are read are cached after they are decompressed. + * When new pages are fetched, we check the cache first to see if they are there; + * if so, they are satsfied by the cache. + * + * Modifications are written to the cache, then dumped to the disk. + * + * The cache is managed by two functions: + * af_cache_flush(af) - (prevously af_purge) + * - Makes sure that all dirty buffers are written. + * - Sets af->pb=NULL (no current page) + * - (returns 0 if success, -1 if failure.) + * + * af_cache_writethrough(af,page,buf,buflen) + * - used for write bypass + * + */ + +static int cachetime = 0; + + +int af_cache_flush(AFFILE *af) +{ + if(af_trace) fprintf(af_trace,"af_cache_flush()\n"); + int ret = 0; + for(int i=0;inum_pbufs;i++){ + struct aff_pagebuf *p = &af->pbcache[i]; + if(p->pagebuf_valid && p->pagebuf_dirty){ + if(af_update_page(af,p->pagenum,p->pagebuf,p->pagebuf_bytes)){ + ret = -1; // got an error; keep going, though + } + p->pagebuf_dirty = 0; + if(af_trace) fprintf(af_trace,"af_cache_flush: slot %d page %"PRIu64" flushed.\n",i,p->pagenum); + } + } + return ret; // now return the error that I might have gotten +} + +/* If the page being written is in the cache, update it. + * Question: would it make sense to copy the data anyway? I don't think so, because + * the main use of writethrough is when imaging, and in that event you probably don't + * want the extra memcpy. + */ +void af_cache_writethrough(AFFILE *af,int64_t pagenum,const unsigned char *buf,int bufflen) +{ + for(int i=0;inum_pbufs;i++){ + struct aff_pagebuf *p = &af->pbcache[i]; + if(p->pagenum_valid && p->pagenum == pagenum){ + if(p->pagebuf_dirty){ + (*af->error_reporter)("af_cache_writethrough: overwriting page %"I64u".\n",pagenum); + exit(-1); // this shouldn't happen + } + memcpy(p->pagebuf,buf,bufflen); + memset(p->pagebuf+bufflen,0,af->image_pagesize-bufflen); // zero fill the rest + af->bytes_memcpy += bufflen; + p->pagebuf_valid = 1; // we have a copy of it now. + p->pagebuf_dirty = 0; // but it isn't dirty + p->last = cachetime++; + } + } +} + +#ifdef HAVE_MALLOC_H +#include +#endif + +#ifndef HAVE_VALLOC +#define valloc malloc +#endif + +struct aff_pagebuf *af_cache_alloc(AFFILE *af,int64_t pagenum) +{ + if(af_trace) fprintf(af_trace,"af_cache_alloc(%p,%"I64d")\n",af,pagenum); + af_cache_flush(af); // make sure nothing in cache is dirty + /* See if this page is already in the cache */ + for(int i=0;inum_pbufs;i++){ + struct aff_pagebuf *p = &af->pbcache[i]; + if(p->pagenum_valid && p->pagenum==pagenum){ + af->cache_hits++; + if(af_trace) fprintf(af_trace," page %"I64d" satisfied fromcache\n",pagenum); + p->last = cachetime++; + return p; + } + } + + af->cache_misses++; + int slot = -1; + /* See if there is an empty slot in the cache */ + for(int i=0;inum_pbufs;i++){ + struct aff_pagebuf *p = &af->pbcache[i]; + if(p->pagenum_valid==0){ + slot = i; + if(af_trace) fprintf(af_trace," slot %d given to page %"I64d"\n",slot,pagenum); + break; + } + } + if(slot==-1){ + /* Find the oldest cache entry */ + int oldest_i = 0; + int oldest_t = af->pbcache[0].last; + for(int i=1;inum_pbufs;i++){ + if(af->pbcache[i].last < oldest_t){ + oldest_t = af->pbcache[i].last; + oldest_i = i; + } + } + slot = oldest_i; + if(af_trace) fprintf(af_trace," slot %d assigned to page %"I64d"\n",slot,pagenum); + } + /* take over this slot */ + struct aff_pagebuf *p = &af->pbcache[slot]; + if(p->pagebuf==0){ + p->pagebuf = (unsigned char *)valloc(af->image_pagesize); // allocate to a page boundary + if(p->pagebuf==0){ + /* Malloc failed; See if we can just use the first slot */ + slot = 0; + if(af->pbcache[0].pagebuf==0) return 0; // ugh. Cannot malloc? + + /* First slot is available. Just use it. */ + p = &af->pbcache[0]; + } + } + memset(p->pagebuf,0,af->image_pagesize); // clean object reuse + p->pagenum = pagenum; + p->pagenum_valid = 1; + p->pagebuf_valid = 0; + p->pagebuf_dirty = 0; + p->last = cachetime++; + if(af_trace){ + fprintf(af_trace," current pages in cache: "); + for(int i=0;inum_pbufs;i++){ + fprintf(af_trace," %"I64d,af->pbcache[i].pagenum); + } + fprintf(af_trace,"\n"); + } + return p; +} + + + + diff --git a/lib/afflib_stream.cpp b/lib/afflib_stream.cpp new file mode 100644 index 0000000..e1e12d8 --- /dev/null +++ b/lib/afflib_stream.cpp @@ -0,0 +1,299 @@ +/* + * The AFFLIB data stream interface. + * Supports the page->segment name translation, and the actual file pointer. + * Distributed under the Berkeley 4-part license. + */ + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" + + +/**************************************************************** + *** Internal Functions. + ****************************************************************/ + +#ifdef _WIN32 +#define ASIZE SSIZE_T +#else +#define ASIZE ssize_t +#endif + + +/* + * af_set_maxsize + * Sets the maxsize. + Fails with -1 if imagesize >= 0 unless this is a raw or split_raw file + */ +int af_set_maxsize(AFFILE *af,int64_t maxsize) +{ + AF_WRLOCK(af); + if(af->image_size>0){ + (*af->error_reporter)("Cannot set maxsize as imagesize is already set (%"I64d")",af->image_size); + AF_UNLOCK(af); + return -1; // now allowed to set if imagesize is bigger than 0 + } + if((af->image_pagesize!=0) + && (af->v->type & AF_VNODE_MAXSIZE_MULTIPLE) + && (maxsize % af->image_pagesize != 0)){ + (*af->error_reporter)("Cannot set maxsize to %"I64d" --- not multiple of pagesize=%d\n", + maxsize,af->image_pagesize); + AF_UNLOCK(af); + return -1; + } + af->maxsize = maxsize; + AF_UNLOCK(af); + return 0; +} + +const unsigned char *af_badflag(AFFILE *af) +{ + return af->badflag; +} + + +/**************************************************************** + *** Stream-level interface + ****************************************************************/ + + +/* Throw out the current segment */ +int af_purge(AFFILE *af) +{ + AF_WRLOCK(af); + if (af_trace) fprintf(af_trace,"af_purge(%p)\n",af); + int ret = af_cache_flush(af); // flush the cache + af->pb = 0; // no longer have a current page + AF_UNLOCK(af); + return ret; +} + +extern "C" ASIZE af_read(AFFILE *af,unsigned char *buf,ASIZE count) +{ + int total = 0; + + AF_WRLOCK(af); // wrlock because cache may change + if (af_trace) fprintf(af_trace,"af_read(%p,%p,%d) (pos=%"I64d")\n",af,buf,(int)count,af->pos); + if (af->v->read){ // check for bypass + int r = (af->v->read)(af, buf, af->pos, count); + if(r>0) af->pos += r; + AF_UNLOCK(af); + return r; + } + + /* performance improvement: use af->image_size if it is set */ + uint64_t offset = af->pos; /* where to start */ + + if(af->image_size<0) {total=-1;goto done;} // error + if(af->image_size==0) {goto done;} // no data in file + if(af->pos > af->image_size) {goto done;} // seeked beyond end of file + if(af->pos+count > af->image_size) count = af->image_size - af->pos; // only this much left in file + + + /* Make sure we have a pagebuf if none was defined */ + if(af->image_pagesize==0){ // page size not defined + errno = EFAULT; + total=-1; + goto done; + } + + while(count>0){ + /* If the correct segment is not loaded, purge the segment */ + int64_t new_page = offset / af->image_pagesize; + + if(af->pb==0 || new_page != af->pb->pagenum){ + af_cache_flush(af); + af->pb = 0; + } + + /* If no segment is loaded in cache, load the current segment */ + if(af->pb==0){ + int64_t pagenum = offset / af->image_pagesize; + af->pb = af_cache_alloc(af,pagenum); + if(af->pb->pagebuf_valid==0){ + /* page buffer isn't valid; need to get it */ + af->pb->pagebuf_bytes = af->image_pagesize; // we can hold this much + if(af_get_page(af,af->pb->pagenum,af->pb->pagebuf, &af->pb->pagebuf_bytes)){ + /* Page doesn't exist; fill with NULs */ + memset(af->pb->pagebuf,0,af->pb->pagebuf_bytes); + /* TK: Should fill with BADBLOCK here if desired */ + /* previously had BREAK here */ + } + af->pb->pagebuf_valid = 1; // contents of the page buffer are valid + } + } + // Compute how many bytes can be copied... + // where we were reading from + u_int page_offset = (u_int)(offset - af->pb->pagenum * af->image_pagesize); + + if(page_offset > af->pb->pagebuf_bytes){ + /* Page is short. */ + /* Question - should we advance af->pos to the next page? */ + break; + } + + u_int page_left = af->pb->pagebuf_bytes - page_offset; // number we can get out + u_int bytes_to_read = count; + + if(bytes_to_read > page_left) bytes_to_read = page_left; + if(bytes_to_read > af->image_size - offset) bytes_to_read = (u_int)(af->image_size - offset); + + assert(bytes_to_read >= 0); // + if(bytes_to_read==0) break; // that's all we could get + + /* Copy out the bytes for the user */ + memcpy(buf,af->pb->pagebuf+page_offset,bytes_to_read); // copy out + af->bytes_memcpy += bytes_to_read; + buf += bytes_to_read; + offset += bytes_to_read; + count -= bytes_to_read; + total += bytes_to_read; + af->pos += bytes_to_read; + } + /* We have copied all of the user's requested data, so return */ + done: + AF_UNLOCK(af); + return total; +} + + +/* + * Handle writing to the file... + * af_write() --- returns the number of bytes written + * + */ + +int af_write(AFFILE *af,unsigned char *buf,size_t count) +{ + AF_WRLOCK(af); + if (af_trace){ + fprintf(af_trace,"af_write(af=%p,buf=%p,count=%d) pos=%"I64d"\n", af,buf,(int)count,af->pos); + } + /* Invalidate caches */ + af_invalidate_vni_cache(af); + + /* vnode write bypass: + * If a write function is defined, use it and avoid the page and cache business. + */ + if (af->v->write){ + int r = (af->v->write)(af, buf, af->pos, count); + if(r>0){ + af->pos += r; + af->bytes_written += r; + } + if(af->pos >= af->image_size) af->image_size = af->pos; + AF_UNLOCK(af); + return r; + } + + /* If no pagesize has been set, go with the default pagesize */ + if(af->image_pagesize==0){ + if(af_set_pagesize(af,AFF_DEFAULT_PAGESIZE)){ + AF_UNLOCK(af); + return -1; + } + } + + int64_t offset = af->pos; // where to start + + /* If the correct segment is not loaded, purge the current segment */ + int64_t write_page = offset / af->image_pagesize; + if(af->pb && af->pb->pagenum!=write_page){ + af_cache_flush(af); + af->pb = 0; + } + + int write_page_offset = (int)(offset % af->image_pagesize); + + /* Page Write Bypass: + * If no data has been written into the current page buffer, + * and if the position of the stream is byte-aligned on the page buffer, + * and if an entire page is being written, + * just write it out and update the pointers, then return. + */ + if(af->pb==0 && af->image_pagesize==(unsigned)count && write_page_offset == 0){ + // copy into cache if we have this page anywhere in our cache + af_cache_writethrough(af,write_page,buf,count); + int ret = af_update_page(af,write_page,buf,count); + if(ret==0){ // no error + af->pos += count; + if(af->pos > af->image_size) af->image_size = af->pos; + AF_UNLOCK(af); + return count; + } + AF_UNLOCK(af); + return -1; // error + } + + + /* Can't use high-speed optimization; write through the cache */ + int total = 0; + while(count>0){ + /* If no page is loaded, or the wrong page is loaded, load the correct page */ + int64_t pagenum = offset / af->image_pagesize; // will be the segment we want + if(af->pb==0 || af->pb->pagenum != pagenum){ + af->pb = af_cache_alloc(af,pagenum); + af->pb->pagebuf_bytes = af->image_pagesize; + assert(af->pb->pagenum == pagenum); + + /* Now try to load the page. + * If we can't load it, then we are creating a new page. + */ + if(af_get_page(af,af->pb->pagenum,af->pb->pagebuf, &af->pb->pagebuf_bytes)){ + /* Creating a new page; note that we have no bytes in this page */ + af->pb->pagebuf_bytes = 0; + } + } + // where writing to + u_int seg_offset = (u_int)(offset - af->pb->pagenum * af->image_pagesize); + + // number we can write into + u_int seg_left = af->image_pagesize - seg_offset; + u_int bytes_to_write = count; + + if(bytes_to_write > seg_left) bytes_to_write = seg_left; + + assert(bytes_to_write >= 0); // + if(bytes_to_write==0) break; // that's all we could get + + /* Copy out the bytes for the user */ + memcpy(af->pb->pagebuf+seg_offset,buf,bytes_to_write); // copy into the page cache + af->bytes_memcpy += bytes_to_write; + + if(af->pb->pagebuf_bytes < seg_offset+bytes_to_write){ + af->pb->pagebuf_bytes = seg_offset+bytes_to_write; // it has been extended. + } + + buf += bytes_to_write; + offset += bytes_to_write; + count -= bytes_to_write; + total += bytes_to_write; + af->pos += bytes_to_write; + af->pb->pagebuf_valid = 1; + af->pb->pagebuf_dirty = 1; + + /* If we wrote out all of the bytes that were left in the segment, + * then we are at the end of the segment, write it back... + */ + if(seg_left == bytes_to_write){ + if(af_cache_flush(af)){ + AF_UNLOCK(af); + return -1; + } + } + + /* If we have written more than the image size, update the image size */ + if((uint64_t)offset > af->image_size) af->image_size = offset; + } + /* We have copied all of the user's requested data, so return */ + AF_UNLOCK(af); + return total; +} + +/* No lock needed? */ +int af_is_badsector(AFFILE *af,const unsigned char *buf) +{ + if(af->badflag_set==0) return 0; + if(af->badflag==0) return 0; + return memcmp(af->badflag,buf,af->image_sectorsize)==0; +} diff --git a/lib/afflib_util.cpp b/lib/afflib_util.cpp new file mode 100644 index 0000000..9e32405 --- /dev/null +++ b/lib/afflib_util.cpp @@ -0,0 +1,318 @@ +/* + * utility functions used by AFFLIB. + * These functions do not actually read or write data into the AFF File. + * Distributed under the Berkeley 4-part license. + */ + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" + +#ifndef HAVE_ERR +#include +void err(int eval,const char *fmt,...) +{ + va_list ap; + va_start(ap,fmt); + vfprintf(stderr,fmt,ap); + va_end(ap); + fprintf(stderr,": %s\n",strerror(errno)); + exit(eval); +} +#endif + +void af_err(int eval,const char *fmt,...) +{ + va_list ap; + va_start(ap,fmt); + vfprintf(stderr,fmt,ap); + va_end(ap); + if(af_error_str[0]) fprintf(stderr,": %s",af_error_str); + if(errno) fprintf(stderr,": %s",strerror(errno)); + fputc('\n',stderr); + exit(eval); +} + + +#ifndef HAVE_ERRX +#include +void errx(int eval,const char *fmt,...) +{ + va_list ap; + va_start(ap,fmt); + vfprintf(stderr,fmt,ap); + fprintf(stderr,"%s\n",strerror(errno)); + va_end(ap); + exit(eval); +} +#endif + +#ifndef HAVE_WARN +#include +void warn(const char *fmt, ...) +{ + va_list args; + va_start(args,fmt); + vfprintf(stderr,fmt, args); + fprintf(stderr,": %s",strerror(errno)); +} +#endif + +#ifndef HAVE_WARNX +#include +void warnx(const char *fmt,...) +{ + va_list ap; + va_start(ap,fmt); + vfprintf(stderr,fmt,ap); + va_end(ap); +} +#endif + + + + +/* + * af_hexbuf: + * Turn a binay string into a hex string, optionally with spaces. + */ + +const char *af_hexbuf(char *dst,int dst_len,const unsigned char *bin,int bytes,int flag) +{ + int charcount = 0; + const char *start = dst; // remember where the start of the string is + const char *fmt = (flag & AF_HEXBUF_UPPERCASE) ? "%02X" : "%02x"; + + *dst = 0; // begin with null termination + while(bytes>0 && dst_len > 3){ + sprintf(dst,fmt,*bin); // convert the next byte + dst += 2; + bin += 1; + dst_len -= 2; + bytes--; + charcount++; // how many characters + + if((flag & AF_HEXBUF_SPACE4) && charcount%2==0){ + *dst++ = ' '; + *dst = '\000'; + dst_len -= 1; + } + } + return start; // return the start +} + + +/* Add commas + */ +const char *af_commas(char buf[64],int64_t val) +{ + char tmp[64]; + char t2[64]; + int negative = 0; + + buf[0] = 0; + if(val==0){ + strcpy(buf,"0"); + } + if(val<0){ + negative = 1; + val = -val; + } + + while(val>0){ + int digits = val % 1000; // get the residue + val = val / 1000; // and shift what's over + + if(val>0){ // we will still have more to do + sprintf(tmp,",%03d",digits); // so pad it with zeros and a comma + } + else { + sprintf(tmp,"%d",digits); // otherwise just get the value + } + strcpy(t2,buf); // copy out the buffer + strcpy(buf,tmp); // copy in what we just did + strcat(buf,t2); // and put back what was there + } + if(negative){ + strcpy(t2,buf); + strcpy(buf,"-"); + strcat(buf,t2); + } + return buf; +} + +uint64_t af_decode_q(unsigned char buf[8]) +{ + struct aff_quad *q = (struct aff_quad *)buf; // point q to buf. + + assert(sizeof(*q)==8); // be sure! + return (((uint64_t)ntohl(q->high)) << 32) + ((uint64_t)ntohl(q->low)); +} + +/* parse the segment number. + * The extra %c picks up characters that might be after the number, + * so that page5_hash doesn't match for page5. + */ +int64_t af_segname_page_number(const char *name) +{ +#ifdef KERNEL_LIBRARY +#define PAGE_NAME "page" + if(_strnicmp(name,PAGE_NAME,strlen(PAGE_NAME))==0) + { + int64_t pagenum; + for (int i=strlen(PAGE_NAME);i0){ + /* There is room; just copy over what we have and return */ + strcat(dest,src); + return strlen(dest); + } + /* Not room; figure out how many bytes we can copy... */ + int left = dest_size - (dest_len+1); + strncpy(dest+dest_len,src,left); + dest[dest_len-1] = '\000'; + return strlen(dest); +} +#endif + +/* Parse a URL. Allocate the parts if requested. Default protocol is "file", of course*/ +void af_parse_url(const char *url,char **protocol,char **hostname,char **username,char **password, + int *port,char **path) +{ + const char *p1 = strstr(url,"://"); + if(!p1){ + if(protocol) *protocol = strdup("file"); + if(path) *path = strdup(url); + return; + } + if(protocol){ + int len = p1-url; + *protocol = (char *)malloc(len+1); + strncpy(*protocol,url,len); + } + url = p1+3; // move past :// + + const char *at = strchr(url,'@'); + if(at){ // we may have a username and/or password + char *scratch = (char *)malloc(at-url+1); + strncpy(scratch,url,at-url); + scratch[at-url]='\000'; + char *colon = strchr(scratch,':'); + if(colon){ + *colon = '\000'; + } + if(username) *username = strdup(scratch); + if(colon){ + if(password) *password = strdup(colon+1); + } + free(scratch); + url = at+1; + } + + /* Process hostname if it exists */ + const char *slash = strchr(url,'/'); + if(slash){ + char *scratch = (char *)malloc(slash-url+1); + strncpy(scratch,url,slash-url); + scratch[slash-url]='\000'; + char *colon = strchr(scratch,':'); + if(colon){ + *colon = '\000'; + } + if(hostname) *hostname = strdup(scratch); + if(colon){ + if(port) *port = atoi(colon+1); + } + free(scratch); + url = slash+1; + } + if(path) *path = strdup(url); // remember file name +} diff --git a/lib/aftest.cpp b/lib/aftest.cpp new file mode 100644 index 0000000..5be9b4e --- /dev/null +++ b/lib/aftest.cpp @@ -0,0 +1,1004 @@ +/* + * atest.cpp: + * test suite for the AFF Library. + * This file is a work of a US government employee and as such is in the Public domain. + * Simson L. Garfinkel, March 12, 2012 + */ + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" +#include "base64.h" +#include "aftimer.h" + +#ifdef HAVE_GETOPT_H +#include +#endif + +#include "LzmaRam.h" +extern "C" { +#include "LzmaRamDecode.h" +} + +int MAX_FMTS = 10000; // how many formats we should write + +const char *fmt = "%8d Another format string.\n"; // must be constant size +const char *progname = 0; + +const char *opt_protocol = "file:///"; +const char *opt_ext = "aff"; +int opt_compression_level = AF_COMPRESSION_DEFAULT;// default compression level +int opt_compression_type = AF_COMPRESSION_ALG_ZLIB; // +const char *tempdir = "/tmp/"; + + +/* Create the segment that we need */ + +#ifndef MIN +#define MIN(x,y) ((x)<(y)?(x):(y)) +#endif + + +const char *filename(char *buf,int buflen,const char *base) +{ + snprintf(buf,buflen,"%s%s%s.%s",opt_protocol,tempdir,base,opt_ext); + return buf; +} + +AFFILE *open_testfile(const char *base,int wipe) +{ + int flags = O_CREAT|O_RDWR; + char fn[1024]; + + filename(fn,sizeof(fn),base); + printf("%s = %s\n",base,fn); + if(wipe){ + unlink(fn); // make sure it is gone + flags |= O_TRUNC; + } + AFFILE *af = af_open(fn,flags,0666); + if(!af) err(1,"af_open"); + if(wipe){ + af_enable_compression(af,opt_compression_type,opt_compression_level); + af_set_pagesize(af,1024); + af_set_maxsize(af,(int64_t)65536); // force splitting of raw and afd files + } + return af; +} + +int sequential_test() +{ + char buf[1024]; + const char *fmt = "this is line %d\n"; + + printf("Sequential test...\n"); + + AFFILE *af = open_testfile("test_sequential",1); + for(int i=0;i=0;i--){ + sprintf(wbuf,fmt,i); + af_seek(af,strlen(wbuf)*i,SEEK_SET); + if(pass==1){ + if(af_write(af,(unsigned char *)wbuf,strlen(wbuf))!=(int)strlen(wbuf)){ + err(1,"Attempt to write buffer %d failed\n",i); + } + } + if(pass==2){ + memset(rbuf,0,sizeof(rbuf)); + if(af_read(af,(unsigned char *)rbuf,strlen(wbuf))!=(int)strlen(wbuf)){ + err(1,"Attempt to read buffer %d failed\n",i); + } + if(strcmp(rbuf,wbuf)!=0){ + errx(1,"Data doesn't verify.\nWrote: '%s'\nRead: '%s'\n",wbuf,rbuf); + } + } + } + af_close(af); + } + + printf("\nReverse test passes.\n"); + printf("======================\n\n"); + return 0; +} + + +int random_write_test() +{ + char buf[1024]; + char *tally = (char *)calloc(MAX_FMTS,1); + int i; + + memset(tally,0,sizeof(tally)); + + /* Create the AFF file */ + sprintf(buf,fmt,0); // figure out how big fmt string is + int fmt_size = strlen(buf); + + printf("Random write test...\n"); + printf("Creating test file with %d byte records.\n", fmt_size); + + AFFILE *af = open_testfile("test_random",1); + + if(af_write(af,(unsigned char *)buf,fmt_size)!=fmt_size){ + err(1,"af_write"); + } + for(i=0;i0 && memcmp(buf,test,buflen)==0){ + errx(1,"Error: data fetched with wrong passphrase was not scrambled."); + } + af_close(af); + exit(0); +} + +void readfile_test(const char *fname) +{ + unsigned char buf[1024]; + memset(buf,0,sizeof(buf)); + AFFILE *af = af_open(fname,O_RDONLY,0666); + if(!af){ + af_perror(fname); + err(1,"af_open(%s)",fname); + } + printf("using '%s'\n",af->v->name); + printf("af_get_imagesize()=%"PRId64" errno=%d\n",af_get_imagesize(af),errno); + + int r = af_read(af,buf,sizeof(buf)); + printf("af_read(af,buf,1024)=%d errno=%d\n",r,errno); + r = fwrite(buf,1,512,stdout); + assert(r==512); + af_close(af); + exit(0); +} + +void zap(const char *fn) +{ + unsigned char buf[1024*1024]; + AFFILE *af = af_open(fn,O_RDWR,0666); + if(!af) err(1,"af_open(%s)",fn); + memset(buf,0,sizeof(buf)); + if(af_write(af,buf,sizeof(buf))!=sizeof(buf)){ + err(1,"af_write()"); + } + af_close(af); +} + +const char *fnames[] = {"foo.000","foo.001", + "foo.100","foo.101", + "bizmark.999","bizmark.A00", + "nutter.A99","nutter.AA0", + "bizmark.AZ9","bizmark.B00", + "glutten.afj","glutten.afk", + 0,0}; + +void bugs_test() +{ + for(int i=0;fnames[i];i+=2){ + char buf[256]; + strcpy(buf,fnames[i]); + if(split_raw_increment_fname(buf)){ + err(1,"split_raw_increment_fname(%s) failed",fnames[i]); + } + printf("%s=>%s\n",fnames[i],buf); + if(strcmp(buf,fnames[i+1])!=0){ + err(1,"split_raw_increment_fname(%s) should have returned %s", + fnames[i],fnames[i+1]); + } + } + + + const char *buf = "This is a test\n"; + int len = strlen(buf); + + AFFILE *af = af_open("bugs.aff",O_RDWR|O_CREAT|O_TRUNC,0666); + if(!af) err(1,"bugs.aff"); + int r = af_write(af,(unsigned char *)buf,strlen(buf)); + if(r!=len) err(1,"r=%d len=%d\n",r,len); + af_close(af); +} + + +void rsatest(); +void xmltest(const char *fn); + +void time_test() +{ + exit(0); +} + + +#include +#include + +void rsatest() +{ + const EVP_MD *sha256 = EVP_get_digestbyname("sha256"); + + if(!sha256){ + fprintf(stderr,"SHA256 not available\n"); + return; + } + + printf("Now try signing with X.509 certificates and EVP\n"); + + char ptext[16]; + memset(ptext,0,sizeof(ptext)); + strcpy(ptext,"Simson"); + + unsigned char sig[1024]; + uint32_t siglen = sizeof(sig); + + BIO *bp = BIO_new_file("signing_key.pem","r"); + + EVP_MD_CTX md; + EVP_PKEY *pkey = PEM_read_bio_PrivateKey(bp,0,0,0); + + EVP_SignInit(&md,sha256); + EVP_SignUpdate(&md,ptext,sizeof(ptext)); + EVP_SignFinal(&md,sig,&siglen,pkey); + + /* let's try to verify it */ + bp = BIO_new_file("signing_cert.pem","r"); + X509 *x = 0; + PEM_read_bio_X509(bp,&x,0,0); + EVP_PKEY *pubkey = X509_get_pubkey(x); + + printf("pubkey=%p\n",pubkey); + + EVP_VerifyInit(&md,sha256); + EVP_VerifyUpdate(&md,ptext,sizeof(ptext)); + int r = EVP_VerifyFinal(&md,sig,siglen,pubkey); + printf("r=%d\n",r); + + printf("do it again...\n"); + EVP_VerifyInit(&md,sha256); + EVP_VerifyUpdate(&md,ptext,sizeof(ptext)); + r = EVP_VerifyFinal(&md,sig,siglen,pubkey); + printf("r=%d\n",r); + + printf("make a tiny change...\n"); + ptext[0]='f'; + EVP_VerifyInit(&md,sha256); + EVP_VerifyUpdate(&md,ptext,sizeof(ptext)); + r = EVP_VerifyFinal(&md,sig,siglen,pubkey); + printf("r=%d\n",r); +} + +void xmlseg(BIO *bp,AFFILE *af,const char *segname) +{ + BIO_printf(bp," \n"); + BIO_printf(bp," %s\n",segname); + /* Get the signature and base64 it (if we can) */ + u_char sigbuf[1024]; + size_t sigbuf_len = sizeof(sigbuf); + char segname_sig[1024]; + strlcpy(segname_sig,segname,sizeof(segname_sig)); + strlcat(segname_sig,AF_SIG256_SUFFIX,sizeof(segname_sig)); + if(af_get_seg(af,segname_sig,0,sigbuf,&sigbuf_len)==0){ + char sigbuf48[2048]; + int sigbuf48_len = b64_ntop(sigbuf,sigbuf_len,sigbuf48,sizeof(sigbuf48)); + sigbuf48[sigbuf48_len] = 0; // null terminate + BIO_printf(bp," %s\n",sigbuf48); + } + BIO_printf(bp," \n"); +} + +void xmltest(const char *fn) +{ + BIO *bp = BIO_new(BIO_s_mem()); + AFFILE *af = af_open(fn,O_RDONLY,0); + if(!af) err(1,"%s",fn); + char segname[AF_MAX_NAME_LEN]; + while(af_get_next_seg(af,segname,sizeof(segname),0,0,0)==0){ + xmlseg(bp,af,segname); + } + char *buf=0; + ssize_t len = BIO_get_mem_data(bp,&buf); + int r = fwrite(buf,1,len,stdout); + assert(r==len); +} + + +void image_test() +{ + char fn[1024]; + printf("Imaging test...\n"); + filename(fn,sizeof(fn),"test_image"); + unlink(fn); // make sure it is gone + AFFILE *af = af_open(fn,O_CREAT|O_RDWR|O_TRUNC,0666); + if(!af) err(1,"af_open"); +} + +void usage() +{ + printf("usage: %s [options]\n",progname); + printf(" -e ext = use ext for extension (default is %s)\n",opt_ext); + printf(" -p protocol = use protocol for protocol (default is %s)\n",opt_protocol); + printf(" -a = do all tests (except -L)\n"); + printf(" -b = do the bugs test (tests things that were reported and fixed)\n"); + printf(" -1 = do sequential test\n"); + printf(" -2 = do reverse test\n"); + printf(" -3 = do random write test\n"); + printf(" -4 = do random read test\n"); + printf(" -5 = do maxsize multi-file test\n"); + printf(" -6 = sparse file test\n"); + printf(" -B = run large file test (needs 5GB of disk)\n"); + printf(" -L = use LZMA compression\n"); + printf(" -r = perform RSA tests\n"); + printf(" -d = use as the working dir for files (default is %s)\n",tempdir); + printf(" -D = write debugging trace to \n"); + printf(" -f = run af_figure_media on dev and print the results\n"); + printf(" -c filename = compress filename and output to stdout\n"); + printf(" -T = just test the LZMA compression\n"); + printf(" -q = quite; just report errors\n"); + printf(" -R filename = just try to read the first 1024 bytes of a file and print what happens...\n"); + printf(" -t = run some timing tests\n"); + printf(" -n nn = sets MAX_FMTS (default %d)\n",MAX_FMTS); + printf(" -i image write speed test (lots of small pages)\n"); + printf(" -v = verbose\n"); + printf(" -S filename = perform split-raw tests on filename\n"); +#ifdef HAVE_AES_ENCRYPT + printf(" -C = just test AES encryption\n"); +#endif + printf(" -x fn = xml test\n"); + printf(" -z fn = open up fn for writing and zap it.\n"); +} + +int split_raw_test(const char *fn) +{ + void srp_dump(AFFILE *af); + AFFILE *af = af_open(fn,O_RDONLY,0666); + if(!af) err(1,"af_open:%s",fn); + printf("split_raw imagesize: %"PRId64"\n",af_get_imagesize(af)); + srp_dump(af); + af_close(af); + return 0; +} + +int main(int argc,char **argv) +{ + progname = argv[0]; + int do_bugs = 0; + int do_sequential = 0; + int do_reverse = 0; + int do_random_write_test = 0; + int do_random_read_test = 0; + int do_large_file = 0; + int do_maxsize_test = 0; + int random_repeat = 1; + int do_sparse_test = 0; + int do_all=0; + int do_image_test =1; + int ch; + + const char *bigdir = getenv(AFFLIB_BIGTMP); // use by default + + if(bigdir) tempdir = bigdir; + + setvbuf(stdout,0,_IONBF,0); + //putenv(AFFLIB_CACHE_STATS"=1"); + + if(argc==1){ + printf("running all tests with -a option (exception bigfile test)\n"); + do_all = 1; + } + + while ((ch = getopt(argc, argv, "b123456aBLd:h?f:e:c:TCp:rx:R:z:tn:D:S:")) != -1) { + switch(ch){ + case 'D': af_trace = fopen(optarg,"w");break; + case 'R': readfile_test(optarg); break; + case 'b': + do_bugs = 1; + break; + case '1': + do_sequential = 1; + break; + case '2': + do_reverse = 1; + break; + case '3': + do_random_write_test = 1; + break; + case '4': + do_random_read_test = 1; + break; + case '5': + do_maxsize_test = 1; + break; + case '6': do_sparse_test = 1; break; + case 'l': + random_repeat = atoi(optarg); + break; + case 'B': + do_large_file = 1; + break; + case 'n': MAX_FMTS = atoi(optarg); break; + case 't': time_test(); break; + case 'L': opt_compression_type = AF_COMPRESSION_ALG_LZMA; break; + case 'T': lzma_test(); break; + case 'r': rsatest(); break; + case 'a': + do_all = 1; + break; + case 'z': + zap(optarg);break; + case 'd': tempdir = optarg; break; + case 'f': figure(optarg); break; + case 'e': opt_ext = optarg; break; + case 'c': compress(optarg); break; + + case 'p': opt_protocol = optarg; break; + case 'x': xmltest(optarg);break; + case 'C': aestest(); break; + case 'i': do_image_test=1;break; + case 'S': split_raw_test(optarg);exit(0); + case 'h': + case '?': + default: + usage(); + } + } + + if(do_bugs || do_all) bugs_test(); + if(do_sequential || do_all) sequential_test(); + if(do_reverse || do_all ) reverse_test(); + if(do_maxsize_test || do_all) maxsize_test(); + if(do_sparse_test || do_all) sparse_test(); + if(do_image_test || do_all) image_test(); + + for(int i=0;i +#include +#endif +#include +#include +#include + +class aftimer { + struct timeval t0; + bool running; + long total_sec; + long total_usec; + double lap_time_; // time from when we last did a "stop" +public: + aftimer():t0(),running(false),total_sec(0),total_usec(0),lap_time_(0){} + + void start(); // start the timer + void stop(); // stop the timer + + time_t tstart() { return t0.tv_sec;} // time we started + double elapsed_seconds(); // how long timer has been running, total + double lap_time(); // how long the timer is running this time + double eta(double fraction_done); // calculate ETA in seconds, given fraction + std::string hms(long t); // turn a number of seconds into h:m:s + std::string elapsed_text(); /* how long we have been running */ + std::string eta_text(double fraction_done); // h:m:s + std::string eta_time(double fraction_done); // the actual time +}; + +/* This code in part from + * http://social.msdn.microsoft.com/Forums/en/vcgeneral/thread/430449b3-f6dd-4e18-84de-eebd26a8d668 + */ + +#ifdef WIN32 +# include +# include +# ifndef DELTA_EPOCH_IN_MICROSECS +# if defined(_MSC_VER) || defined(_MSC_EXTENSIONS) +# define DELTA_EPOCH_IN_MICROSECS 11644473600000000Ui64 +# else +# define DELTA_EPOCH_IN_MICROSECS 11644473600000000ULL +# endif +# endif +#endif + +inline void timestamp(struct timeval *t) +{ +#ifdef WIN32 + FILETIME ft; + GetSystemTimeAsFileTime(&ft); + unsigned __int64 tmpres = 0; + tmpres |= ft.dwHighDateTime; + tmpres <<= 32; + tmpres |= ft.dwLowDateTime; + + /*converting file time to unix epoch*/ + tmpres -= DELTA_EPOCH_IN_MICROSECS; + tmpres /= 10; /*convert into microseconds*/ + t->tv_sec = (long)(tmpres / 1000000UL); + t->tv_usec = (long)(tmpres % 1000000UL); +#else + gettimeofday(t,NULL); +#endif +} + +inline void aftimer::start() +{ + timestamp(&t0); + running = 1; +} + +inline void aftimer::stop(){ + if(running){ + struct timeval t; + timestamp(&t); + total_sec += t.tv_sec - t0.tv_sec; + total_usec += t.tv_usec - t0.tv_usec; + lap_time_ = (double)(t.tv_sec - t0.tv_sec) + (double)(t.tv_usec - t0.tv_usec)/1000000.0; + running = false; + } +} + +inline double aftimer::lap_time() +{ + return lap_time_; +} + +inline double aftimer::elapsed_seconds() +{ + double ret = (double)total_sec + (double)total_usec/1000000.0; + if(running){ + struct timeval t; + timestamp(&t); + ret += t.tv_sec - t0.tv_sec; + ret += (t.tv_usec - t0.tv_usec) / 1000000.0; + } + return ret; +} + +inline std::string aftimer::hms(long t) +{ + char buf[64]; + int days = t / (60*60*24); + + t = t % (60*60*24); /* what's left */ + + int h = t / 3600; + int m = (t / 60) % 60; + int s = t % 60; + buf[0] = 0; + switch(days){ + case 0: + snprintf(buf,sizeof(buf),"%2d:%02d:%02d",h,m,s); + break; + case 1: + snprintf(buf,sizeof(buf),"%d day, %2d:%02d:%02d",days,h,m,s); + break; + default: + snprintf(buf,sizeof(buf),"%d days %2d:%02d:%02d",days,h,m,s); + } + return std::string(buf); +} + +inline std::string aftimer::elapsed_text() +{ + return hms((int)elapsed_seconds()); +} + +/** + * returns the number of seconds until the job is complete. + */ +inline double aftimer::eta(double fraction_done) +{ + double t = elapsed_seconds(); + if(t<=0) return -1; // can't figure it out + if(fraction_done<=0) return -1; // can't figure it out + return (t * 1.0/fraction_done - t); +} + +/** + * Retuns the number of hours:minutes:seconds until the job is done. + */ +inline std::string aftimer::eta_text(double fraction_done) +{ + double e = eta(fraction_done); + if(e<0) return std::string("n/a"); // can't figure it out + return hms((long)e); +} + +/** + * Returns the time when data is due. + */ +inline std::string aftimer::eta_time(double fraction_done) +{ + time_t when = time_t(eta(fraction_done)) + time(0); + struct tm tm; +#ifdef HAVE_LOCALTIME_R + localtime_r(&when,&tm); +#else + tm = *localtime(&when); +#endif + + char buf[64]; + snprintf(buf,sizeof(buf),"%02d:%02d:%02d",tm.tm_hour,tm.tm_min,tm.tm_sec); + return std::string(buf); +} + +#endif + +#endif diff --git a/lib/base64.cpp b/lib/base64.cpp new file mode 100755 index 0000000..fcbb67b --- /dev/null +++ b/lib/base64.cpp @@ -0,0 +1,337 @@ +/* + * Copyright (c) 1996-1999 by Internet Software Consortium. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SOFTWARE CONSORTIUM DISCLAIMS + * ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL INTERNET SOFTWARE + * CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS + * SOFTWARE. + */ + +/* + * Portions Copyright (c) 1995 by International Business Machines, Inc. + * + * International Business Machines, Inc. (hereinafter called IBM) grants + * permission under its copyrights to use, copy, modify, and distribute this + * Software with or without fee, provided that the above copyright notice and + * all paragraphs of this notice appear in all copies, and that the name of IBM + * not be used in connection with the marketing of any product incorporating + * the Software or modifications thereof, without specific, written prior + * permission. + * + * To the extent it has a right to do so, IBM grants an immunity from suit + * under its patents, if any, for the use, sale or manufacture of products to + * the extent that such products are used for performing Domain Name System + * dynamic updates in TCP/IP networks by means of the Software. No immunity is + * granted for any product per se or for any other function of any product. + * + * THE SOFTWARE IS PROVIDED "AS IS", AND IBM DISCLAIMS ALL WARRANTIES, + * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE. IN NO EVENT SHALL IBM BE LIABLE FOR ANY SPECIAL, + * DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER ARISING + * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE, EVEN + * IF IBM IS APPRISED OF THE POSSIBILITY OF SUCH DAMAGES. + */ + +#include +#include + +#ifdef HAVE_SYS_TYPES_H +#include +#else +typedef unsigned char u_char; +#endif + +#include +#include + +#include "base64.h" + +#define Assert(Cond) if (!(Cond)) abort() + +static const char Base64[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +static const char Pad64 = '='; + +/* (From RFC1521 and draft-ietf-dnssec-secext-03.txt) + The following encoding technique is taken from RFC 1521 by Borenstein + and Freed. It is reproduced here in a slightly edited form for + convenience. + + A 65-character subset of US-ASCII is used, enabling 6 bits to be + represented per printable character. (The extra 65th character, "=", + is used to signify a special processing function.) + + The encoding process represents 24-bit groups of input bits as output + strings of 4 encoded characters. Proceeding from left to right, a + 24-bit input group is formed by concatenating 3 8-bit input groups. + These 24 bits are then treated as 4 concatenated 6-bit groups, each + of which is translated into a single digit in the base64 alphabet. + + Each 6-bit group is used as an index into an array of 64 printable + characters. The character referenced by the index is placed in the + output string. + + Table 1: The Base64 Alphabet + + Value Encoding Value Encoding Value Encoding Value Encoding + 0 A 17 R 34 i 51 z + 1 B 18 S 35 j 52 0 + 2 C 19 T 36 k 53 1 + 3 D 20 U 37 l 54 2 + 4 E 21 V 38 m 55 3 + 5 F 22 W 39 n 56 4 + 6 G 23 X 40 o 57 5 + 7 H 24 Y 41 p 58 6 + 8 I 25 Z 42 q 59 7 + 9 J 26 a 43 r 60 8 + 10 K 27 b 44 s 61 9 + 11 L 28 c 45 t 62 + + 12 M 29 d 46 u 63 / + 13 N 30 e 47 v + 14 O 31 f 48 w (pad) = + 15 P 32 g 49 x + 16 Q 33 h 50 y + + Special processing is performed if fewer than 24 bits are available + at the end of the data being encoded. A full encoding quantum is + always completed at the end of a quantity. When fewer than 24 input + bits are available in an input group, zero bits are added (on the + right) to form an integral number of 6-bit groups. Padding at the + end of the data is performed using the '=' character. + + Since all base64 input is an integral number of octets, only the + ------------------------------------------------- + following cases can arise: + + (1) the final quantum of encoding input is an integral + multiple of 24 bits; here, the final unit of encoded + output will be an integral multiple of 4 characters + with no "=" padding, + (2) the final quantum of encoding input is exactly 8 bits; + here, the final unit of encoded output will be two + characters followed by two "=" padding characters, or + (3) the final quantum of encoding input is exactly 16 bits; + here, the final unit of encoded output will be three + characters followed by one "=" padding character. + */ + +extern "C" +int +b64_ntop(u_char const *src, size_t srclength, char *target, size_t targsize) { + size_t datalength = 0; + u_char input[3]; + u_char output[4]; + size_t i; + + while (2 < srclength) { + input[0] = *src++; + input[1] = *src++; + input[2] = *src++; + srclength -= 3; + + output[0] = input[0] >> 2; + output[1] = ((input[0] & 0x03) << 4) + (input[1] >> 4); + output[2] = ((input[1] & 0x0f) << 2) + (input[2] >> 6); + output[3] = input[2] & 0x3f; + Assert(output[0] < 64); + Assert(output[1] < 64); + Assert(output[2] < 64); + Assert(output[3] < 64); + + if (datalength + 4 > targsize) + return (-1); + target[datalength++] = Base64[output[0]]; + target[datalength++] = Base64[output[1]]; + target[datalength++] = Base64[output[2]]; + target[datalength++] = Base64[output[3]]; + } + + /* Now we worry about padding. */ + if (0 != srclength) { + /* Get what's left. */ + input[0] = input[1] = input[2] = '\0'; + for (i = 0; i < srclength; i++) + input[i] = *src++; + + output[0] = input[0] >> 2; + output[1] = ((input[0] & 0x03) << 4) + (input[1] >> 4); + output[2] = ((input[1] & 0x0f) << 2) + (input[2] >> 6); + Assert(output[0] < 64); + Assert(output[1] < 64); + Assert(output[2] < 64); + + if (datalength + 4 > targsize) + return (-1); + target[datalength++] = Base64[output[0]]; + target[datalength++] = Base64[output[1]]; + if (srclength == 1) + target[datalength++] = Pad64; + else + target[datalength++] = Base64[output[2]]; + target[datalength++] = Pad64; + } + if (datalength >= targsize) + return (-1); + target[datalength] = '\0'; /* Returned value doesn't count \0. */ + return (datalength); +} + +/* skips all whitespace anywhere. + converts characters, four at a time, starting at (or after) + src from base - 64 numbers into three 8 bit bytes in the target area. + it returns the number of data bytes stored at the target, or -1 on error. + */ + +extern "C" +int +b64_pton_slg(char const *src, int srclen, u_char *target, size_t targsize) +{ + int tarindex, state, ch; + const char *pos; + + state = 0; + tarindex = 0; + + while ((ch = *src++) != '\0' && srclen>0){ + srclen--; + if (isspace(ch)) /* Skip whitespace anywhere. */ + continue; + + if (ch == Pad64) + break; + + pos = strchr(Base64, ch); + if (pos == 0){ /* A non-base64 character. */ + puts("B64 Fail at 1"); + return (-1); + } + + switch (state) { + case 0: + if (target) { + if ((size_t)tarindex >= targsize){ + puts("B64 fail at 2"); + return (-1); + } + target[tarindex] = (pos - Base64) << 2; + } + state = 1; + break; + case 1: + if (target) { + if ((size_t)tarindex + 1 >= targsize){ + puts("B64 fail at 3"); + return (-1); + } + target[tarindex] |= (pos - Base64) >> 4; + target[tarindex+1] = ((pos - Base64) & 0x0f) + << 4 ; + } + tarindex++; + state = 2; + break; + case 2: + if (target) { + if ((size_t)tarindex + 1 >= targsize){ + puts("B64 fail at 4"); + return (-1); + } + target[tarindex] |= (pos - Base64) >> 2; + target[tarindex+1] = ((pos - Base64) & 0x03) + << 6; + } + tarindex++; + state = 3; + break; + case 3: + if (target) { + if ((size_t)tarindex >= targsize){ + puts("B64 fail at 5"); + return (-1); + } + target[tarindex] |= (pos - Base64); + } + tarindex++; + state = 0; + break; + default: + abort(); + } + } + + /* + * We are done decoding Base-64 chars. Let's see if we ended + * on a byte boundary, and/or with erroneous trailing characters. + */ + + if (ch == Pad64) { /* We got a pad char. */ + ch = *src++; /* Skip it, get next. */ + srclen--; + + switch (state) { + case 0: /* Invalid = in first position */ + case 1: /* Invalid = in second position */ + puts("B64 fail at 6"); + /* return (-1);*/ + return tarindex; /* slg is nicer */ + + case 2: /* Valid, means one byte of info */ + /* Skip any number of spaces. */ + for ((void)NULL; ch != '\0' && srclen>0 ; ch = *src++,srclen--){ + if (!isspace(ch)){ + break; + } + } + /* Make sure there is another trailing = sign. */ + if (ch != Pad64){ + puts("B64 fail at 7"); + return (-1); + } + ch = *src++; /* Skip the = */ + srclen--; + /* Fall through to "single trailing =" case. */ + /* FALLTHROUGH */ + + case 3: /* Valid, means two bytes of info */ + /* + * We know this char is an =. Is there anything but + * whitespace after it? + */ + for ((void)NULL; ch != '\0' && srclen>0; ch = *src++,srclen--) + if (!isspace(ch)){ + puts("B64 fail at 8"); + return (-1); + } + + /* + * Now make sure for cases 2 and 3 that the "extra" + * bits that slopped past the last full byte were + * zeros. If we don't check them, they become a + * subliminal channel. + */ + if (target && target[tarindex] != 0){ + puts("B64 fail at 9"); + return (-1); + } + } + } else { + /* + * We ended by seeing the end of the string. Make sure we + * have no partial bytes lying around. + */ + if (state != 0){ + puts("B64 fail at 10"); + return (-1); + } + } + + return (tarindex); +} diff --git a/lib/base64.h b/lib/base64.h new file mode 100755 index 0000000..7313197 --- /dev/null +++ b/lib/base64.h @@ -0,0 +1,39 @@ +/* + * Base64 conversion. + * + * From RFC1521 and draft-ietf-dnssec-secext-03.txt. + * + * Implementation (C) 1996-1999 by Internet Software Consortium. + */ + + + +#ifndef BASE64_H +#define BASE64_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif +#ifdef NEVER_DEFINED +} +#endif +/* Convert from printable base64 to binary. + * Returns number of bytes converted + */ +int b64_pton_slg(const char *str,int srclen,unsigned char *target,size_t targsize); + +/* Convert from binary to printable base 64. + * returns size of printable thing. + */ +int b64_ntop(const unsigned char *str,size_t srclength,char *target,size_t targsize); + +#ifdef NEVER_DEFINED +{ +#endif +#ifdef __cplusplus +} +#endif + +#endif diff --git a/lib/benchmark.sh b/lib/benchmark.sh new file mode 100644 index 0000000..e39ce30 --- /dev/null +++ b/lib/benchmark.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# perform some performance testing +# This file is a work of a US government employee and as such is in the Public domain. +# Simson L. Garfinkel, March 12, 2012 +# +echo Building aftest with profiling +(cd .. ; make distclean; ./configure CFLAGS="-p -pg" CPPFLAGS="-p -pg" CXXFLAGS="-p -pg" ; make) +cp aftest aftest.prof +./aftest.prof -1 +gprof -c aftest.prof gmon.out > aftest.gmon.txt + +echo Re-Building aftest without profiling +(cd .. ; make distclean; ./configure ; make) + +echo Running time tests + +echo try it with auditing +sudo dtruss ./aftest -1 2>aftest.dtruss +echo number of lseeks: `grep lseek aftest.dtruss | wc -l` +echo number of reads: `grep read_nocancel aftest.dtruss | wc -l` +echo number of writes: `grep write_nocancel aftest.dtruss | wc -l` + diff --git a/lib/crypto.cpp b/lib/crypto.cpp new file mode 100644 index 0000000..75c0f6b --- /dev/null +++ b/lib/crypto.cpp @@ -0,0 +1,979 @@ +/* + * This file is a work of a US government employee and as such is in the Public domain. + * Simson L. Garfinkel, March 12, 2012 + */ + + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" +#include "utils.h" + +#ifdef HAVE_OPENSSL_PEM_H +#include +#include +#endif + +#ifdef HAVE_STL +#include +#include +#include +using namespace std; +#endif + +#ifdef HAVE_CSTRING +#include +#endif + + + + + +/**************************************************************** + *** LOW LEVEL ROUTINES + ****************************************************************/ + +/** + * Returns TRUE if the segment named 'buf' has the suffixi indicating + * that it is an encrypted segment. + */ +int af_is_encrypted_segment(const char *segname){ + if(strcmp(segname,AF_AFFKEY)==0) return 1; + if(aff::ends_with(segname,AF_AES256_SUFFIX)) return 1; + if(strncmp(segname,AF_AFFKEY_EVP,strlen(AF_AFFKEY_EVP)-1)==0) return 1; + return 0; +} + +/** + * Returns TRUE if the segment named 'buf' has the suffix indicating + * that it is a signature segment. + * + * @param segname - segment to check + */ +int af_is_signature_segment(const char *segname){ + int num = 0; + char cc; + if(aff::ends_with(segname,AF_SIG256_SUFFIX)) return 1; + if(sscanf(segname,"affbom%d%c",&num,&cc)==1) return 1; // it's a bom segment + return 0; +} + + +/**************************************************************** + *** AES ENCRYPTION LAYER + ****************************************************************/ + +static const char *aff_cannot_sign = "AFFLIB: OpenSSL does not have SHA256! "\ + "AFF segments cannot be signed. "\ + "See http://www.afflib.org/requirements.php for additional information."; + +void af_crypto_allocate(AFFILE *af) +{ + af->crypto = (struct af_crypto *)calloc(sizeof(struct af_crypto),1); // give space +} + + +/** compute SHA256. + * Return 0 if success, -1 if error. + */ +int af_SHA256(const unsigned char *data,size_t datalen,unsigned char md[32]) +{ + const EVP_MD *sha256 = EVP_get_digestbyname("SHA256"); + if(!sha256) return -1; + + uint32_t sha256_buflen = 32; + EVP_MD_CTX ctx; + EVP_DigestInit(&ctx,sha256); + EVP_DigestUpdate(&ctx,data,datalen); + if(EVP_DigestFinal(&ctx,md,&sha256_buflen)!=1) return -1; // EVP_DigestFinal returns 1 for success + return 0; +} + +void af_crypto_deallocate(AFFILE *af) +{ +#ifdef AES_BLOCK_SIZE + memset(&af->crypto->ekey,0,sizeof(af->crypto->ekey)); + memset(&af->crypto->dkey,0,sizeof(af->crypto->dkey)); +#endif +#ifdef HAVE_PEM_READ_BIO_RSA_PUBKEY + if(af->crypto->sign_privkey){ + EVP_PKEY_free(af->crypto->sign_privkey); + af->crypto->sign_privkey = 0; + } + if(af->crypto->sign_pubkey){ + EVP_PKEY_free(af->crypto->sign_pubkey); + af->crypto->sign_pubkey = 0; + } + if(af->crypto->sign_cert){ + X509_free(af->crypto->sign_cert); + af->crypto->sign_cert = 0; + } +#endif + free(af->crypto); + af->crypto = 0; +} + + +int af_set_aes_key(AFFILE *af,const unsigned char *userKey,const int bits) +{ +#ifdef HAVE_AES_ENCRYPT + if(af->crypto->sealing_key_set){ + if(userKey==0){ // key was set and it is being cleared + af->crypto->sealing_key_set = 0; + return 0; + } + return AF_ERROR_KEY_SET; // key is already set + } + int r; + r = AES_set_encrypt_key(userKey,bits,&af->crypto->ekey); + if(r) return r; + + r = AES_set_decrypt_key(userKey,bits,&af->crypto->dkey); + if(r) return r; + + af->crypto->sealing_key_set = 1; + af->crypto->auto_encrypt = 1; // default + af->crypto->auto_decrypt = 1; // default + af_invalidate_vni_cache(af); // invalidate the cache, because now we can read encrypted values + return 0; +#else + return AF_ERROR_NO_AES; +#endif +} + + + +/** + * Take an unencrypted AFFKEY, encrypt it with the SHA256 of the passphrase, + * and save it in the appropriate segment. + */ + +int af_save_aes_key_with_passphrase(AFFILE *af,const char *passphrase, const u_char affkey[32]) +{ +#if defined(HAVE_AES_ENCRYPT) + if(af->crypto->sealing_key_set) return AF_ERROR_KEY_SET; // already enabled + + /* Make an encrypted copy of the AFFkey */ + unsigned char passphrase_hash[32]; + af_SHA256((const unsigned char *)passphrase, strlen(passphrase), passphrase_hash); + + struct affkey affkey_seg; + assert(sizeof(affkey_seg)==AFFKEY_SIZE); + memset((unsigned char *)&affkey_seg,0,sizeof(affkey_seg)); + + uint32_t version_number = htonl(1); // version 1 + memcpy(affkey_seg.version,(u_char *)&version_number,4); + memcpy(affkey_seg.affkey_aes256,affkey,32); + + /* Use the hash to encrypt the key and all zeros */ + AES_KEY ekey; + AES_set_encrypt_key(passphrase_hash,256,&ekey); + AES_encrypt(affkey_seg.affkey_aes256, + affkey_seg.affkey_aes256,&ekey); + AES_encrypt(affkey_seg.affkey_aes256+AES_BLOCK_SIZE, + affkey_seg.affkey_aes256+AES_BLOCK_SIZE,&ekey); + AES_encrypt(affkey_seg.zeros_aes256,affkey_seg.zeros_aes256,&ekey); + + /* Write this to a segment */ + if(af_update_seg(af,AF_AFFKEY,0,(const u_char *)&affkey_seg,sizeof(affkey_seg))) return -1; + memset((unsigned char *)&affkey_seg,0,sizeof(affkey_seg)); // erase the temp data + return 0; +#endif +#if !defined(HAVE_AES_ENCRYPT) + return AF_ERROR_NO_AES; +#endif +} + +/** MacOS 10.5 with GCC 4.0.1 packed affkey at 52 bytes. + ** Linux GCC 4.1.2 packed affkey at 56 bytes. It should be 52 bytes + ** --- 4 bytes for the version number, 32 bytes for the affkey, 16 bytes for encryption of zeros. + ** original code specified the version as uint32_t version:32, for which the + ** compiler allocated 64 bits... + ** So this code needs to be willing to accept a 52-byte or 56-byte affkey. + **/ +/* Legacy - this version of the structure was improperly used in AFFLIB prior to + * 3.1.6. Unfortunately, the structure didn't pack properly, resulting in some images + * in which the affkey structure was too large. + */ +struct affkey_legacy { + uint32_t version:32; + u_char affkey_aes256[32]; // AFF key encrypted with SHA-256 of passphrase + // encrypted as two codebooks in a row; no need for CBC + u_char zeros_aes256[16]; // all zeros encrypted with SHA-256 of passphrase +}; + +int af_get_aes_key_from_passphrase(AFFILE *af,const char *passphrase, + unsigned char affkey[32]) +{ +#if defined(HAVE_AES_ENCRYPT) + if(af->crypto->sealing_key_set) return AF_ERROR_KEY_SET; // already enabled + + /* Get the segment with the key in it. It should be AFFKEY_SIZE + * but there are a few images out there with the wrong key size due + * to a compiler packing bug. Automatically handle those. + */ + struct affkey affkey_seg; // in-memory copy + u_char kbuf[1024]; + size_t klen=sizeof(kbuf); + uint32_t version; + int kversion=0; + + /* Try to get the segment */ + if(af_get_seg(af,AF_AFFKEY,0,kbuf,&klen)) return AF_ERROR_AFFKEY_NOT_EXIST; + + if(sizeof(affkey_seg)==klen){ + // On-disk structure is correct; copy it over + memcpy(&affkey_seg,kbuf,klen); + memcpy((char *)&version,affkey_seg.version,4); + kversion = ntohl(version); + } else { + // Try to figure it out manually + memcpy((char *)&version,kbuf,4); + kversion = ntohl(version); + memcpy(affkey_seg.affkey_aes256,kbuf+4,sizeof(affkey_seg.affkey_aes256)); + memcpy(affkey_seg.zeros_aes256,kbuf+36,sizeof(affkey_seg.zeros_aes256)); + } + + /* make sure version is correct */ + if(kversion != 1){ + errno = EINVAL; + return AF_ERROR_AFFKEY_WRONG_VERSION; + } + + /* hash the passphrase */ + unsigned char passphrase_hash[32]; + if(af_SHA256((const unsigned char *)passphrase,strlen(passphrase), passphrase_hash)){ + return AF_ERROR_NO_SHA256; + } + + /* Try to decrypt the key */ + + AES_KEY dkey; + AES_set_decrypt_key(passphrase_hash,256,&dkey); + AES_decrypt(affkey_seg.affkey_aes256, + affkey_seg.affkey_aes256,&dkey); + AES_decrypt(affkey_seg.affkey_aes256+AES_BLOCK_SIZE, + affkey_seg.affkey_aes256+AES_BLOCK_SIZE,&dkey); + AES_decrypt(affkey_seg.zeros_aes256,affkey_seg.zeros_aes256,&dkey); + + /* See if its zero? */ + for(u_int i=0;icrypto->sealing_key_set) return AF_ERROR_KEY_SET; // already enabled + + /* Can only establish a passphrase if the encryption segment doesn't exist */ + if(af_get_seg(af,AF_AFFKEY,0,0,0)==0) return AF_ERROR_AFFKEY_EXISTS; + + /* Check to make sure it wasn't public key encrypted */ + char segname[AF_MAX_NAME_LEN]; + snprintf(segname,sizeof(segname),AF_AFFKEY_EVP,0); + if(af_get_seg(af,segname,0,0,0)==0) return AF_ERROR_AFFKEY_EXISTS; + + /* Okay; make a random key and encrypt it with the passphrase */ + unsigned char affkey[32]; + int r = RAND_bytes(affkey,sizeof(affkey)); // makes a random key; with REAL random bytes + if(r!=1) r = RAND_pseudo_bytes(affkey,sizeof(affkey)); // true random not supported + if(r!=1) return AF_ERROR_RNG_FAIL; // pretty bad... + + /* I have the key, now save it */ + r = af_save_aes_key_with_passphrase(af,passphrase,affkey); + memset(affkey,0,sizeof(affkey)); /* Erase the encryption key in memory */ + return r; +#else + return AF_ERROR_NO_AES; +#endif +} + + +/** Like the one above, this public interface actually wipes the key after it is created. + * @param passphrase - Passphrae, use NULL to erase the encryption key. + * This can only be done if the file is opened read-only. + */ +int af_use_aes_passphrase(AFFILE *af,const char *passphrase) +{ + af_invalidate_vni_cache(af); + if(passphrase==0 && !(af->openflags & O_RDWR)){ + af->crypto->sealing_key_set = 0; + return 0; + } + + if(af->crypto->sealing_key_set) return AF_ERROR_KEY_SET; // already enabled + + unsigned char affkey[32]; + int r = af_get_aes_key_from_passphrase(af,passphrase,affkey); + if(r) return r; // wrong keyphrase + r = af_set_aes_key(af,affkey,256); /* Set the encryption key */ + memset(affkey,0,sizeof(affkey)); /* Erase the encryption key in memory */ + return r; +} + + +/* gets the key with the old phrase and then changes it to the new one */ +int af_change_aes_passphrase(AFFILE *af,const char *oldphrase,const char *newphrase) +{ + if(af->crypto->sealing_key_set) return AF_ERROR_KEY_SET; // already enabled + + unsigned char affkey[32]; + int r = af_get_aes_key_from_passphrase(af,oldphrase,affkey); + + if(r) return r; + r = af_save_aes_key_with_passphrase(af,newphrase,affkey); + memset(affkey,0,sizeof(affkey)); // erase the temp data + return r; +} + + +int af_has_encrypted_segments(AFFILE *af) +{ + struct af_vnode_info vni; + af_vstat(af,&vni); + return vni.segment_count_encrypted>0; +} + +/** + * Returns true if there are segments that cannot be decrypted + * (other than key segments) + */ +int af_cannot_decrypt(AFFILE *af){ + if(af_has_encrypted_segments(af)==0) return 0; // no encrypted segments to decrypt + /* Now start at the beginning and see if any segments are read which are encrypted. + * If they are encrypted, then we don't have the encryption key. + */ + if(af_rewind_seg(af)) return -1; + char segname[AF_MAX_NAME_LEN]; + memset(segname,0,sizeof(segname)); + while(af_get_next_seg(af,segname,sizeof(segname),0,0,0)==0){ + if(aff::ends_with(segname,AF_AES256_SUFFIX)) return 1; // we shouldn't see these. + } + return 0; +} + +/**************************************************************** + *** + *** Signature Routines + *** + ****************************************************************/ + +/** See if the public key and private key match by dial a trial encryption and decryption. + * + * @param pubkey + * @param privkey + * @returns 0 if successful, -1 if failure. + */ +static int check_keys(EVP_PKEY *privkey,EVP_PKEY *pubkey) +{ + char ptext[16]; /* plaintext of a 128-bit message */ + unsigned char sig[1024]; /* signature; bigger than needed */ + uint32_t siglen = sizeof(sig); /* length of signature */ + + const EVP_MD *sha256 = EVP_get_digestbyname("SHA256"); + if(!sha256) return -1; // no SHA256. + + EVP_MD_CTX md; /* EVP message digest */ + + + /* make the plaintext message */ + memset(ptext,0,sizeof(ptext)); + strcpy(ptext,"Test Message"); + EVP_SignInit(&md,sha256); + EVP_SignUpdate(&md,ptext,sizeof(ptext)); + EVP_SignFinal(&md,sig,&siglen,privkey); + + /* Verify the message */ + EVP_VerifyInit(&md,sha256); + EVP_VerifyUpdate(&md,ptext,sizeof(ptext)); + if(EVP_VerifyFinal(&md,sig,siglen,pubkey)!=1){ + return -3; + } + return 0; +} + + +/** + * af_set_sign_files: + * + * Load the private key & certificate, make sure they are matched, and + * write to the AFF. This requirest not just AES256, but EVP_SHA256 + * because we use the openSSL signature functions. + * + * @param af - The open AFFILE + * @param keyfile - The filename of the key file to read + * @param certfile - The filename of the certificate file to read + */ + + +int af_set_sign_files(AFFILE *af,const char *keyfile,const char *certfile) +{ + const EVP_MD *sha256 = EVP_get_digestbyname("SHA256"); + if(!sha256){ + (*af->error_reporter)(aff_cannot_sign); + return AF_ERROR_NO_SHA256; // + } + + BIO *bp = BIO_new_file(keyfile,"r"); + if(!bp) return -1; + af->crypto->sign_privkey = PEM_read_bio_PrivateKey(bp,0,0,NULL); + BIO_free(bp); + if(!af->crypto->sign_privkey) return -2; // can't decode keyfile + + bp = BIO_new_file(certfile,"r"); + if(!bp) return -1; + PEM_read_bio_X509(bp,&af->crypto->sign_cert,0,0); + if(af->crypto->sign_cert==0){ + EVP_PKEY_free(af->crypto->sign_privkey); + af->crypto->sign_privkey = 0; + return -3; + } + af->crypto->sign_pubkey = X509_get_pubkey(af->crypto->sign_cert); + BIO_free(bp); + + if(check_keys(af->crypto->sign_privkey,af->crypto->sign_pubkey)){ + /* private key doesn't match certificate */ + EVP_PKEY_free(af->crypto->sign_privkey); af->crypto->sign_privkey = 0; + EVP_PKEY_free(af->crypto->sign_pubkey); af->crypto->sign_pubkey = 0; + return -4; + } + + /* Looks good; save the cert in a segment */ + BIO *xbp = BIO_new(BIO_s_mem()); // where we are writing + PEM_write_bio_X509(xbp,af->crypto->sign_cert); + af_update_seg_frombio(af,AF_SIGN256_CERT,0,xbp); + BIO_free(xbp); + return 0; +} + +/* Sign the segment with the signing key. Signatures are calculated + * by taking the SHA256 of the following concatenated together: + * segment name + * segment arg (in network byte order) + * segment data + */ +int af_sign_seg3(AFFILE *af,const char *segname, + uint32_t arg,const unsigned char *data,uint32_t datalen, + uint32_t signmode) +{ + const EVP_MD *sha256 = EVP_get_digestbyname("SHA256"); + if(!sha256){ + (*af->error_reporter)(aff_cannot_sign); + return AF_ERROR_NO_SHA256; // + } + + + if(af->crypto->sign_privkey==0) return -1; // can't sign; no signing key + + if(strlen(segname)+strlen(AF_SIG256_SUFFIX)+1 > AF_MAX_NAME_LEN) return -1; // too long + + char signed_segname[AF_MAX_NAME_LEN]; + strlcpy(signed_segname,segname,AF_MAX_NAME_LEN); + strlcat(signed_segname,AF_SIG256_SUFFIX,AF_MAX_NAME_LEN); + + if(signmode==AF_SIGNATURE_DELETE){ + af_del_seg(af,signed_segname); + return 0; + } + + uint32_t arg_net = htonl(arg); + unsigned char sig[1024]; /* signature; bigger than needed */ + uint32_t siglen = sizeof(sig); /* length of signature */ + + EVP_MD_CTX md; /* EVP message digest */ + EVP_SignInit(&md,sha256); + EVP_SignUpdate(&md,(const unsigned char *)segname,strlen(segname)+1); + EVP_SignUpdate(&md,(const unsigned char *)&arg_net,sizeof(arg_net)); + EVP_SignUpdate(&md,data,datalen); + EVP_SignFinal(&md,sig,&siglen,af->crypto->sign_privkey); + return (*af->v->update_seg)(af,signed_segname,signmode,sig,siglen); +} + + +int af_sign_seg(AFFILE *af,const char *segname) +{ + size_t datalen = 0; + + /* Now get the data to verify */ + if(af_get_seg(af,segname,0,0,&datalen)){ + return AF_ERROR_SIG_DATAREAD_ERROR; // can't read the segment length + } + + /* Now read the segment */ + unsigned char *data=(unsigned char *)malloc(datalen); + if(data==0) return AF_ERROR_SIG_MALLOC; + + uint32_t arg=0; + if(af_get_seg(af,segname,&arg,data,&datalen)){ + free(data); + return AF_ERROR_SIG_DATAREAD_ERROR; // can't read the segment length + } + + /* Note: it woudl be wrong to detect pages and sign them in mode1, because we don't really + * have access to the uncompressed data... + */ + int r = af_sign_seg3(af,segname,arg,data,datalen,AF_SIGNATURE_MODE0); + free(data); + return r; +} + + +#ifdef HAVE_STL +/** Returns number of segments that were signed. + * Returns -1 if there is an error. + */ +int af_sign_all_unsigned_segments(AFFILE *af) +{ + vector segs; + setsigs; + char name[AF_MAX_NAME_LEN]; + int count=0; + + /* Get a list of all the segments and all the signatures */ + if(af_rewind_seg(af)) return -1; + while(af_get_next_seg(af,name,sizeof(name),0,0,0)==0){ + if(name[0]==0) continue; // don't sign the empty segments + if(aff::ends_with(name,AF_SIG256_SUFFIX)==0){ + segs.push_back(name); + } + else{ + sigs.insert(name); + } + } + /* Sign the ones that are unsigned. */ + for(vector::const_iterator s = segs.begin(); + s != segs.end(); + s++){ + /* Compute name of the signature */ + string signame = *s + AF_SIG256_SUFFIX; + if(sigs.find(signame) == sigs.end()){ + if(af_sign_seg(af,s->c_str())){ + (*af->error_reporter)("AFFLIB: Could not sign segment '%s'",s->c_str()); + return -1; + } + count++; + } + } + return count; +} +#endif + +/* Verify a segment against a particular signature and public key */ +int af_hash_verify_seg2(AFFILE *af,const char *segname,u_char *sigbuf_,size_t sigbuf_len_,int sigmode) +{ + const EVP_MD *sha256 = EVP_get_digestbyname("SHA256"); + if(!sha256){ + (*af->error_reporter)(aff_cannot_sign); + return AF_ERROR_NO_SHA256; // + } + + /* Now get the data to verify */ + size_t seglen = 0; + unsigned char *segbuf = 0; + uint32_t arg=0; + + /* Do we need to get the page */ + if(sigmode==AF_SIGNATURE_MODE1){ + int64_t pagenumber = af_segname_page_number(segname); + if(pagenumber>=0){ + seglen = af_page_size(af); + segbuf = (unsigned char *)malloc(seglen); + if(segbuf==0) return AF_ERROR_SIG_MALLOC; + if(af_get_page(af,pagenumber,segbuf,&seglen)){ + free(segbuf); + return -1; + } + } + } + if(segbuf==0){ // get the raw segment + if(af_get_seg(af,segname,0,0,&seglen)){ + return AF_ERROR_SIG_DATAREAD_ERROR; // can't read the segment length + } + + /* Now read the segment */ + segbuf=(unsigned char *)malloc(seglen); + if(segbuf==0) return AF_ERROR_SIG_MALLOC; + + if(af_get_seg(af,segname,&arg,segbuf,&seglen)){ + free(segbuf); + return AF_ERROR_SIG_DATAREAD_ERROR; // can't read the segment length + } + } + + /* Verify the signature*/ + uint8_t sigbuf[1024]; + uint32_t sigbuf_len = sizeof(sigbuf); + uint32_t arg_net = htonl(arg); + EVP_MD_CTX md; /* EVP message digest */ + EVP_DigestInit(&md,sha256); + EVP_DigestUpdate(&md,(const unsigned char *)segname,strlen(segname)+1); + EVP_DigestUpdate(&md,(const unsigned char *)&arg_net,sizeof(arg_net)); + EVP_DigestUpdate(&md,segbuf,seglen); + EVP_DigestFinal(&md,sigbuf,&sigbuf_len); + int r = memcmp(sigbuf,sigbuf_,sigbuf_len); + if(sigbuf_len != sigbuf_len_) r = -1; // doesn't match + free(segbuf); + + if(r==0) return 0; // verifies + return AF_ERROR_SIG_BAD; // doesn't verify +} + +/* Verify a segment against a particular signature and public key */ +int af_sig_verify_seg2(AFFILE *af,const char *segname,EVP_PKEY *pubkey,u_char *sigbuf,size_t sigbuf_len,int sigmode) +{ + const EVP_MD *sha256 = EVP_get_digestbyname("SHA256"); + if(!sha256){ + (*af->error_reporter)(aff_cannot_sign); + return AF_ERROR_NO_SHA256; // + } + + + /* Now get the data to verify */ + size_t seglen = 0; + unsigned char *segbuf = 0; + uint32_t arg=0; + + /* Do we need to get the page */ + if(sigmode==AF_SIGNATURE_MODE1){ + int64_t pagenumber = af_segname_page_number(segname); + if(pagenumber>=0){ + seglen = af_page_size(af); + segbuf = (unsigned char *)malloc(seglen); + if(segbuf==0) return AF_ERROR_SIG_MALLOC; + if(af_get_page(af,pagenumber,segbuf,&seglen)){ + free(segbuf); + return -1; + } + } + } + if(segbuf==0){ // get the raw segment + if(af_get_seg(af,segname,0,0,&seglen)){ + return AF_ERROR_SIG_DATAREAD_ERROR; // can't read the segment length + } + + /* Now read the segment */ + segbuf=(unsigned char *)malloc(seglen); + if(segbuf==0) return AF_ERROR_SIG_MALLOC; + + if(af_get_seg(af,segname,&arg,segbuf,&seglen)){ + free(segbuf); + return AF_ERROR_SIG_DATAREAD_ERROR; // can't read the segment length + } + } + + /* Verify the signature*/ + uint32_t arg_net = htonl(arg); + EVP_MD_CTX md; /* EVP message digest */ + EVP_VerifyInit(&md,sha256); + EVP_VerifyUpdate(&md,(const unsigned char *)segname,strlen(segname)+1); + EVP_VerifyUpdate(&md,(const unsigned char *)&arg_net,sizeof(arg_net)); + EVP_VerifyUpdate(&md,segbuf,seglen); + int r = EVP_VerifyFinal(&md,sigbuf,sigbuf_len,af->crypto->sign_pubkey); + free(segbuf); + + if(r==1) return 0; // verifies + return AF_ERROR_SIG_BAD; // doesn't verify +} + + + +int af_sig_verify_seg(AFFILE *af,const char *segname) +{ +#ifdef USE_AFFSIGS + if(aff::ends_with(segname,AF_SIG256_SUFFIX)){ + return AF_ERROR_SIG_SIG_SEG; // don't verify the signature segments + } + + /* Need the public key if I don't have it */ + if(af->crypto->sign_pubkey==0){ + unsigned char certbuf[65536]; + size_t certbuf_len = sizeof(certbuf); + if(af_get_seg(af,AF_SIGN256_CERT,0,certbuf,&certbuf_len)!=0){ + return AF_ERROR_SIG_NO_CERT; + } + af->crypto->sign_cert = 0; + BIO *cert_bio = BIO_new_mem_buf(certbuf,certbuf_len); + PEM_read_bio_X509(cert_bio,&af->crypto->sign_cert,0,0); + BIO_free(cert_bio); + af->crypto->sign_pubkey = X509_get_pubkey(af->crypto->sign_cert); + } + + /* Figure out the signature segment name */ + char sigseg[AF_MAX_NAME_LEN + 1 + sizeof(AF_SIG256_SUFFIX)]; + strlcpy(sigseg,segname,sizeof(sigseg)); + strlcat(sigseg,AF_SIG256_SUFFIX,sizeof(sigseg)); + + /* Get the signature (it says how we need to handle the data) */ + unsigned char sigbuf[2048]; // big enough to hold any conceivable signature + size_t sigbuf_len=sizeof(sigbuf); + uint32_t sigmode=0; + if(af_get_seg(af,sigseg,&sigmode,sigbuf,&sigbuf_len)){ + return AF_ERROR_SIG_READ_ERROR; + } + + return af_sig_verify_seg2(af,segname,af->crypto->sign_pubkey,sigbuf,sigbuf_len,sigmode); +#else + return AF_ERROR_SIG_NOT_COMPILED; // sig support not compiled in +#endif +} + +/**************************************************************** + *** PUBLIC KEY ENCRYPION ROUTINES + ****************************************************************/ + +/** + * af_set_seal_certfiles + * + * Specifies the certific file(s) to use for creating a new affkey. + * If an affkey is already on the disk, this returns with an error. + * + * @param af - The open AFFILE + * @param certfile - The filename of the certificate file to read + */ + +int af_set_seal_certificates(AFFILE *af,const char *certfiles[],int numcertfiles) +{ + const EVP_MD *sha256 = EVP_get_digestbyname("SHA256"); + if(!sha256){ + (*af->error_reporter)(aff_cannot_sign); + return AF_ERROR_NO_SHA256; // + } + + char evp0[AF_MAX_NAME_LEN]; // segment where we will store the encrypted session key + snprintf(evp0,sizeof(evp0),AF_AFFKEY_EVP,0); + + /* If an affkey has not been created, create one if there is a public key(s)... + * todo: this should probably see if there is ANY evp segment. + */ + if(af_get_seg(af,evp0,0,0,0)==0) return -1; // make sure no encrypted EVP exists + if(af_get_seg(af,AF_AFFKEY,0,0,0)==0) return -1; // make sure no passphrase exists + if(certfiles==0 || numcertfiles==0) return -1; // make sure the user supplied a certificate + + /* First make the affkey */ + unsigned char affkey[32]; + int r = RAND_bytes(affkey,sizeof(affkey)); + if(r!=1) r = RAND_pseudo_bytes(affkey,sizeof(affkey)); // true random not supported + if(r!=1) return AF_ERROR_RNG_FAIL; // pretty bad... + + af_seal_affkey_using_certificates(af, certfiles, numcertfiles, affkey); + return 0; +} + +/** + * af_seal_affkey_using_certificates + * + * Encrypt the provided affkey. + * + * + */ + +int af_seal_affkey_using_certificates(AFFILE *af,const char *certfiles[],int numcertfiles, unsigned char affkey[32]) +{ + /* Repeat for each public key.. */ + int r; + for(int i=0;i buflen) size = buflen - ptr; // max that can be read + memcpy(data,buf+ptr,size); + ptr += size; + if(processedSize) *processedSize = size; + return S_OK; + } +}; + + +class COutMemoryStream: public ISequentialOutStream, public CMyUnknownImp { +public: + unsigned char *buf; + size_t buflen; + size_t ptr; + size_t *notify; + virtual ~COutMemoryStream(){} + COutMemoryStream(unsigned char *buf_,size_t len,size_t *notify_){ + buf = buf_; + buflen = len; + ptr = 0; + notify = notify_; + } + + MY_UNKNOWN_IMP1(IOutStream) STDMETHOD(Write)(const void *data, UInt32 size, + UInt32 *processedSize){ + if(ptr+size > buflen) return E_FAIL; + memcpy(buf+ptr,data,size); + ptr += size; + if(processedSize) *processedSize = size; + if(notify) *notify = ptr; + return S_OK; + } +}; + +/* + * Attempt to compress. Return -1 if fail. + * (Fails if compression results in expansion. + */ + +int lzma_compress(unsigned char *dest,size_t *destLen,const unsigned char *data,size_t datalen,int level) +{ + PROPID propIDs[] = { + NCoderPropID::kDictionarySize, + NCoderPropID::kPosStateBits, + NCoderPropID::kLitContextBits, + NCoderPropID::kLitPosBits, + NCoderPropID::kAlgorithm, + NCoderPropID::kNumFastBytes, + NCoderPropID::kMatchFinder, + NCoderPropID::kEndMarker + }; + const int nprops = sizeof(propIDs) / sizeof(propIDs[0]); + PROPVARIANT p[nprops]; + + p[0].vt = VT_UI4; p[0].ulVal = UInt32(1 << 24); + p[1].vt = VT_UI4; p[1].ulVal = UInt32(2); // posBits + p[2].vt = VT_UI4; p[2].ulVal = UInt32(3); // literal context bits + p[3].vt = VT_UI4; p[3].ulVal = UInt32(0); // literal pos bits + p[4].vt = VT_UI4; p[4].ulVal = UInt32(2); // compression mode + p[5].vt = VT_UI4; p[5].ulVal = UInt32(128); // fast_bytes + + // old code generates warnings now + //p[6].vt = VT_BSTR; p[6].bstrVal = L"bt4"; // it's okay; we won't change it + + // new code + const void *temp = L"bt4"; + p[6].vt = VT_BSTR; p[6].bstrVal = (OLECHAR *)temp; // it's okay; we won't change it + + p[7].vt = VT_BOOL; p[7].boolVal = VARIANT_FALSE; + + NCompress::NLZMA::CEncoder *encoder = new NCompress::NLZMA::CEncoder; + + if (encoder->SetCoderProperties(propIDs, p, nprops) != S_OK){ + return -1; /* Couldn't set encoder properties */ + } + + /* Open and configure the output stream */ + UInt64 fileSize = datalen; + COutMemoryStream *outStream = new COutMemoryStream(dest,*destLen,destLen); + outStream->AddRef(); + + encoder->WriteCoderProperties(outStream); + + for (int i = 0; i < 8; i++) { + Byte b = Byte(fileSize >> (8 * i)); + if (outStream->Write(&b, sizeof(b), 0) != S_OK){ + outStream->Release(); + return -1; /* Write error while encoding */ + } + } + + CInMemoryStream *inStream = new CInMemoryStream(data,datalen); + inStream->AddRef(); + HRESULT result = encoder->Code(inStream, outStream, 0, 0, 0); + inStream->Release(); + outStream->Release(); + delete(encoder); + + return result; +} + + +int lzma_uncompress(unsigned char *buf,size_t *buflen, const unsigned char *cbuf,size_t cbuf_size) +{ + CInMemoryStream *inStream = new CInMemoryStream(cbuf,cbuf_size); + inStream->AddRef(); + + const UInt32 kPropertiesSize = 5; + Byte properties[kPropertiesSize]; + UInt32 processedSize; + UInt64 fileSize = 0; + NCompress::NLZMA::CDecoder decoderSpec; + + if (inStream->Read(properties, kPropertiesSize, &processedSize) != S_OK){ + inStream->Release(); + return -1; + } + if (processedSize != kPropertiesSize) return -1; + if (decoderSpec.SetDecoderProperties2(properties, kPropertiesSize) != S_OK){ + inStream->Release(); + return -1; + } + + for (int i = 0; i < 8; i++) { + Byte b; + if (inStream->Read(&b, sizeof(b), &processedSize) != S_OK) return -1; + if (processedSize != 1){ + inStream->Release(); + return -1; + } + fileSize |= ((UInt64)b) << (8 * i); + } + + COutMemoryStream *outStream = new COutMemoryStream(buf,*buflen,buflen); + outStream->AddRef(); + int r = decoderSpec.Code(inStream, outStream, 0, &fileSize, 0); + inStream->Release(); + outStream->Release(); + return r; +} + diff --git a/lib/qemu/aes.h b/lib/qemu/aes.h new file mode 100644 index 0000000..2386733 --- /dev/null +++ b/lib/qemu/aes.h @@ -0,0 +1 @@ +#include diff --git a/lib/qemu/aff-block-vmdk.h b/lib/qemu/aff-block-vmdk.h new file mode 100644 index 0000000..14564c7 --- /dev/null +++ b/lib/qemu/aff-block-vmdk.h @@ -0,0 +1,5 @@ +/* Simson Garfinkel's fake header to make block-vmdk.c compile */ +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" + diff --git a/lib/qemu/block-bochs.c b/lib/qemu/block-bochs.c new file mode 100644 index 0000000..1727d17 --- /dev/null +++ b/lib/qemu/block-bochs.c @@ -0,0 +1,254 @@ +/* + * Block driver for the various disk image formats used by Bochs + * Currently only for "growing" type in read-only mode + * + * Copyright (c) 2005 Alex Beregszaszi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include "qemu-common.h" +#include "block_int.h" + +/**************************************************************/ + +#define HEADER_MAGIC "Bochs Virtual HD Image" +#define HEADER_VERSION 0x00020000 +#define HEADER_V1 0x00010000 +#define HEADER_SIZE 512 + +#define REDOLOG_TYPE "Redolog" +#define GROWING_TYPE "Growing" + +// not allocated: 0xffffffff + +// always little-endian +struct bochs_header_v1 { + char magic[32]; // "Bochs Virtual HD Image" + char type[16]; // "Redolog" + char subtype[16]; // "Undoable" / "Volatile" / "Growing" + uint32_t version; + uint32_t header; // size of header + + union { + struct { + uint32_t catalog; // num of entries + uint32_t bitmap; // bitmap size + uint32_t extent; // extent size + uint64_t disk; // disk size + char padding[HEADER_SIZE - 64 - 8 - 20]; + } redolog; + char padding[HEADER_SIZE - 64 - 8]; + } extra; +}; + +// always little-endian +struct bochs_header { + char magic[32]; // "Bochs Virtual HD Image" + char type[16]; // "Redolog" + char subtype[16]; // "Undoable" / "Volatile" / "Growing" + uint32_t version; + uint32_t header; // size of header + + union { + struct { + uint32_t catalog; // num of entries + uint32_t bitmap; // bitmap size + uint32_t extent; // extent size + uint32_t reserved; // for ??? + uint64_t disk; // disk size + char padding[HEADER_SIZE - 64 - 8 - 24]; + } redolog; + char padding[HEADER_SIZE - 64 - 8]; + } extra; +}; + +typedef struct BDRVBochsState { + int fd; + + uint32_t *catalog_bitmap; + int catalog_size; + + int data_offset; + + int bitmap_blocks; + int extent_blocks; + int extent_size; +} BDRVBochsState; + +static int bochs_probe(const uint8_t *buf, int buf_size, const char *filename) +{ + const struct bochs_header *bochs = (const void *)buf; + + if (buf_size < HEADER_SIZE) + return 0; + + if (!strcmp(bochs->magic, HEADER_MAGIC) && + !strcmp(bochs->type, REDOLOG_TYPE) && + !strcmp(bochs->subtype, GROWING_TYPE) && + ((le32_to_cpu(bochs->version) == HEADER_VERSION) || + (le32_to_cpu(bochs->version) == HEADER_V1))) + return 100; + + return 0; +} + +static int bochs_open(BlockDriverState *bs, const char *filename, int flags) +{ + BDRVBochsState *s = bs->opaque; + int fd, i; + struct bochs_header bochs; + struct bochs_header_v1 header_v1; + + fd = open(filename, O_RDWR | O_BINARY); + if (fd < 0) { + fd = open(filename, O_RDONLY | O_BINARY); + if (fd < 0) + return -1; + } + + bs->read_only = 1; // no write support yet + + s->fd = fd; + + if (read(fd, &bochs, sizeof(bochs)) != sizeof(bochs)) { + goto fail; + } + + if (strcmp(bochs.magic, HEADER_MAGIC) || + strcmp(bochs.type, REDOLOG_TYPE) || + strcmp(bochs.subtype, GROWING_TYPE) || + ((le32_to_cpu(bochs.version) != HEADER_VERSION) && + (le32_to_cpu(bochs.version) != HEADER_V1))) { + goto fail; + } + + if (le32_to_cpu(bochs.version) == HEADER_V1) { + memcpy(&header_v1, &bochs, sizeof(bochs)); + bs->total_sectors = le64_to_cpu(header_v1.extra.redolog.disk) / 512; + } else { + bs->total_sectors = le64_to_cpu(bochs.extra.redolog.disk) / 512; + } + + lseek(s->fd, le32_to_cpu(bochs.header), SEEK_SET); + + s->catalog_size = le32_to_cpu(bochs.extra.redolog.catalog); + s->catalog_bitmap = qemu_malloc(s->catalog_size * 4); + if (!s->catalog_bitmap) + goto fail; + if (read(s->fd, s->catalog_bitmap, s->catalog_size * 4) != + s->catalog_size * 4) + goto fail; + for (i = 0; i < s->catalog_size; i++) + le32_to_cpus(&s->catalog_bitmap[i]); + + s->data_offset = le32_to_cpu(bochs.header) + (s->catalog_size * 4); + + s->bitmap_blocks = 1 + (le32_to_cpu(bochs.extra.redolog.bitmap) - 1) / 512; + s->extent_blocks = 1 + (le32_to_cpu(bochs.extra.redolog.extent) - 1) / 512; + + s->extent_size = le32_to_cpu(bochs.extra.redolog.extent); + + return 0; + fail: + close(fd); + return -1; +} + +static inline int seek_to_sector(BlockDriverState *bs, int64_t sector_num) +{ + BDRVBochsState *s = bs->opaque; + int64_t offset = sector_num * 512; + int64_t extent_index, extent_offset, bitmap_offset, block_offset; + char bitmap_entry; + + // seek to sector + extent_index = offset / s->extent_size; + extent_offset = (offset % s->extent_size) / 512; + + if (s->catalog_bitmap[extent_index] == 0xffffffff) + { +// fprintf(stderr, "page not allocated [%x - %x:%x]\n", +// sector_num, extent_index, extent_offset); + return -1; // not allocated + } + + bitmap_offset = s->data_offset + (512 * s->catalog_bitmap[extent_index] * + (s->extent_blocks + s->bitmap_blocks)); + block_offset = bitmap_offset + (512 * (s->bitmap_blocks + extent_offset)); + +// fprintf(stderr, "sect: %x [ext i: %x o: %x] -> %x bitmap: %x block: %x\n", +// sector_num, extent_index, extent_offset, +// le32_to_cpu(s->catalog_bitmap[extent_index]), +// bitmap_offset, block_offset); + + // read in bitmap for current extent + lseek(s->fd, bitmap_offset + (extent_offset / 8), SEEK_SET); + + if(read(s->fd, &bitmap_entry, 1)!=1) return -1; + + if (!((bitmap_entry >> (extent_offset % 8)) & 1)) + { +// fprintf(stderr, "sector (%x) in bitmap not allocated\n", +// sector_num); + return -1; // not allocated + } + + lseek(s->fd, block_offset, SEEK_SET); + + return 0; +} + +static int bochs_read(BlockDriverState *bs, int64_t sector_num, + uint8_t *buf, int nb_sectors) +{ + BDRVBochsState *s = bs->opaque; + int ret; + + while (nb_sectors > 0) { + if (!seek_to_sector(bs, sector_num)) + { + ret = read(s->fd, buf, 512); + if (ret != 512) + return -1; + } + else + memset(buf, 0, 512); + nb_sectors--; + sector_num++; + buf += 512; + } + return 0; +} + +static void bochs_close(BlockDriverState *bs) +{ + BDRVBochsState *s = bs->opaque; + qemu_free(s->catalog_bitmap); + close(s->fd); +} + +BlockDriver bdrv_bochs = { + "bochs", + sizeof(BDRVBochsState), + bochs_probe, + bochs_open, + bochs_read, + NULL, + bochs_close, +}; diff --git a/lib/qemu/block-cloop.c b/lib/qemu/block-cloop.c new file mode 100644 index 0000000..43d3801 --- /dev/null +++ b/lib/qemu/block-cloop.c @@ -0,0 +1,169 @@ +/* + * QEMU Block driver for CLOOP images + * + * Copyright (c) 2004 Johannes E. Schindelin + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include "qemu-common.h" +#include "block_int.h" +#include + +typedef struct BDRVCloopState { + int fd; + uint32_t block_size; + uint32_t n_blocks; + uint64_t* offsets; + uint32_t sectors_per_block; + uint32_t current_block; + uint8_t *compressed_block; + uint8_t *uncompressed_block; + z_stream zstream; +} BDRVCloopState; + +static int cloop_probe(const uint8_t *buf, int buf_size, const char *filename) +{ + const char* magic_version_2_0="#!/bin/sh\n" + "#V2.0 Format\n" + "modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1\n"; + int length=strlen(magic_version_2_0); + if(length>buf_size) + length=buf_size; + if(!memcmp(magic_version_2_0,buf,length)) + return 2; + return 0; +} + +static int cloop_open(BlockDriverState *bs, const char *filename, int flags) +{ + BDRVCloopState *s = bs->opaque; + uint32_t offsets_size,max_compressed_block_size=1,i; + + s->fd = open(filename, O_RDONLY | O_BINARY); + if (s->fd < 0) + return -errno; + bs->read_only = 1; + + /* read header */ + if(lseek(s->fd,128,SEEK_SET)<0) { +cloop_close: + close(s->fd); + return -1; + } + if(read(s->fd,&s->block_size,4)<4) + goto cloop_close; + s->block_size=be32_to_cpu(s->block_size); + if(read(s->fd,&s->n_blocks,4)<4) + goto cloop_close; + s->n_blocks=be32_to_cpu(s->n_blocks); + + /* read offsets */ + offsets_size=s->n_blocks*sizeof(uint64_t); + if(!(s->offsets=(uint64_t*)malloc(offsets_size))) + goto cloop_close; + if(read(s->fd,s->offsets,offsets_size)n_blocks;i++) { + s->offsets[i]=be64_to_cpu(s->offsets[i]); + if(i>0) { + uint32_t size=s->offsets[i]-s->offsets[i-1]; + if(size>max_compressed_block_size) + max_compressed_block_size=size; + } + } + + /* initialize zlib engine */ + if(!(s->compressed_block = malloc(max_compressed_block_size+1))) + goto cloop_close; + if(!(s->uncompressed_block = malloc(s->block_size))) + goto cloop_close; + if(inflateInit(&s->zstream) != Z_OK) + goto cloop_close; + s->current_block=s->n_blocks; + + s->sectors_per_block = s->block_size/512; + bs->total_sectors = s->n_blocks*s->sectors_per_block; + return 0; +} + +static inline int cloop_read_block(BDRVCloopState *s,int block_num) +{ + if(s->current_block != block_num) { + int ret; + uint32_t bytes = s->offsets[block_num+1]-s->offsets[block_num]; + + lseek(s->fd, s->offsets[block_num], SEEK_SET); + ret = read(s->fd, s->compressed_block, bytes); + if (ret != bytes) + return -1; + + s->zstream.next_in = s->compressed_block; + s->zstream.avail_in = bytes; + s->zstream.next_out = s->uncompressed_block; + s->zstream.avail_out = s->block_size; + ret = inflateReset(&s->zstream); + if(ret != Z_OK) + return -1; + ret = inflate(&s->zstream, Z_FINISH); + if(ret != Z_STREAM_END || s->zstream.total_out != s->block_size) + return -1; + + s->current_block = block_num; + } + return 0; +} + +static int cloop_read(BlockDriverState *bs, int64_t sector_num, + uint8_t *buf, int nb_sectors) +{ + BDRVCloopState *s = bs->opaque; + int i; + + for(i=0;isectors_per_block), + block_num=(sector_num+i)/s->sectors_per_block; + if(cloop_read_block(s, block_num) != 0) + return -1; + memcpy(buf+i*512,s->uncompressed_block+sector_offset_in_block*512,512); + } + return 0; +} + +static void cloop_close(BlockDriverState *bs) +{ + BDRVCloopState *s = bs->opaque; + close(s->fd); + if(s->n_blocks>0) + free(s->offsets); + free(s->compressed_block); + free(s->uncompressed_block); + inflateEnd(&s->zstream); +} + +BlockDriver bdrv_cloop = { + "cloop", + sizeof(BDRVCloopState), + cloop_probe, + cloop_open, + cloop_read, + NULL, + cloop_close, +}; + + diff --git a/lib/qemu/block-cow.c b/lib/qemu/block-cow.c new file mode 100644 index 0000000..473157e --- /dev/null +++ b/lib/qemu/block-cow.c @@ -0,0 +1,267 @@ +/* + * Block driver for the COW format + * + * Copyright (c) 2004 Fabrice Bellard + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef _WIN32 +#include "qemu-common.h" +#include "block_int.h" +#include + +/**************************************************************/ +/* COW block driver using file system holes */ + +/* user mode linux compatible COW file */ +#define COW_MAGIC 0x4f4f4f4d /* MOOO */ +#define COW_VERSION 2 + +struct cow_header_v2 { + uint32_t magic; + uint32_t version; + char backing_file[1024]; + int32_t mtime; + uint64_t size; + uint32_t sectorsize; +}; + +typedef struct BDRVCowState { + int fd; + uint8_t *cow_bitmap; /* if non NULL, COW mappings are used first */ + uint8_t *cow_bitmap_addr; /* mmap address of cow_bitmap */ + int cow_bitmap_size; + int64_t cow_sectors_offset; +} BDRVCowState; + +static int cow_probe(const uint8_t *buf, int buf_size, const char *filename) +{ + const struct cow_header_v2 *cow_header = (const void *)buf; + + if (buf_size >= sizeof(struct cow_header_v2) && + be32_to_cpu(cow_header->magic) == COW_MAGIC && + be32_to_cpu(cow_header->version) == COW_VERSION) + return 100; + else + return 0; +} + +static int cow_open(BlockDriverState *bs, const char *filename, int flags) +{ + BDRVCowState *s = bs->opaque; + int fd; + struct cow_header_v2 cow_header; + int64_t size; + + fd = open(filename, O_RDWR | O_BINARY | O_LARGEFILE); + if (fd < 0) { + fd = open(filename, O_RDONLY | O_BINARY | O_LARGEFILE); + if (fd < 0) + return -1; + } + s->fd = fd; + /* see if it is a cow image */ + if (read(fd, &cow_header, sizeof(cow_header)) != sizeof(cow_header)) { + goto fail; + } + + if (be32_to_cpu(cow_header.magic) != COW_MAGIC || + be32_to_cpu(cow_header.version) != COW_VERSION) { + goto fail; + } + + /* cow image found */ + size = be64_to_cpu(cow_header.size); + bs->total_sectors = size / 512; + + pstrcpy(bs->backing_file, sizeof(bs->backing_file), + cow_header.backing_file); + + /* mmap the bitmap */ + s->cow_bitmap_size = ((bs->total_sectors + 7) >> 3) + sizeof(cow_header); + s->cow_bitmap_addr = mmap(get_mmap_addr(s->cow_bitmap_size), + s->cow_bitmap_size, + PROT_READ | PROT_WRITE, + MAP_SHARED, s->fd, 0); + if (s->cow_bitmap_addr == MAP_FAILED) + goto fail; + s->cow_bitmap = s->cow_bitmap_addr + sizeof(cow_header); + s->cow_sectors_offset = (s->cow_bitmap_size + 511) & ~511; + return 0; + fail: + close(fd); + return -1; +} + +static inline void cow_set_bit(uint8_t *bitmap, int64_t bitnum) +{ + bitmap[bitnum / 8] |= (1 << (bitnum%8)); +} + +static inline int is_bit_set(const uint8_t *bitmap, int64_t bitnum) +{ + return !!(bitmap[bitnum / 8] & (1 << (bitnum%8))); +} + + +/* Return true if first block has been changed (ie. current version is + * in COW file). Set the number of continuous blocks for which that + * is true. */ +static inline int is_changed(uint8_t *bitmap, + int64_t sector_num, int nb_sectors, + int *num_same) +{ + int changed; + + if (!bitmap || nb_sectors == 0) { + *num_same = nb_sectors; + return 0; + } + + changed = is_bit_set(bitmap, sector_num); + for (*num_same = 1; *num_same < nb_sectors; (*num_same)++) { + if (is_bit_set(bitmap, sector_num + *num_same) != changed) + break; + } + + return changed; +} + +static int cow_is_allocated(BlockDriverState *bs, int64_t sector_num, + int nb_sectors, int *pnum) +{ + BDRVCowState *s = bs->opaque; + return is_changed(s->cow_bitmap, sector_num, nb_sectors, pnum); +} + +static int cow_read(BlockDriverState *bs, int64_t sector_num, + uint8_t *buf, int nb_sectors) +{ + BDRVCowState *s = bs->opaque; + int ret, n; + + while (nb_sectors > 0) { + if (is_changed(s->cow_bitmap, sector_num, nb_sectors, &n)) { + lseek(s->fd, s->cow_sectors_offset + sector_num * 512, SEEK_SET); + ret = read(s->fd, buf, n * 512); + if (ret != n * 512) + return -1; + } else { + if (bs->backing_hd) { + /* read from the base image */ + ret = bdrv_read(bs->backing_hd, sector_num, buf, n); + if (ret < 0) + return -1; + } else { + memset(buf, 0, n * 512); + } + } + nb_sectors -= n; + sector_num += n; + buf += n * 512; + } + return 0; +} + +static int cow_write(BlockDriverState *bs, int64_t sector_num, + const uint8_t *buf, int nb_sectors) +{ + BDRVCowState *s = bs->opaque; + int ret, i; + + lseek(s->fd, s->cow_sectors_offset + sector_num * 512, SEEK_SET); + ret = write(s->fd, buf, nb_sectors * 512); + if (ret != nb_sectors * 512) + return -1; + for (i = 0; i < nb_sectors; i++) + cow_set_bit(s->cow_bitmap, sector_num + i); + return 0; +} + +static void cow_close(BlockDriverState *bs) +{ + BDRVCowState *s = bs->opaque; + munmap(s->cow_bitmap_addr, s->cow_bitmap_size); + close(s->fd); +} + +static int cow_create(const char *filename, int64_t image_sectors, + const char *image_filename, int flags) +{ + int fd, cow_fd; + struct cow_header_v2 cow_header; + struct stat st; + + if (flags) + return -ENOTSUP; + + cow_fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, + 0644); + if (cow_fd < 0) + return -1; + memset(&cow_header, 0, sizeof(cow_header)); + cow_header.magic = cpu_to_be32(COW_MAGIC); + cow_header.version = cpu_to_be32(COW_VERSION); + if (image_filename) { + /* Note: if no file, we put a dummy mtime */ + cow_header.mtime = cpu_to_be32(0); + + fd = open(image_filename, O_RDONLY | O_BINARY); + if (fd < 0) { + close(cow_fd); + goto mtime_fail; + } + if (fstat(fd, &st) != 0) { + close(fd); + goto mtime_fail; + } + close(fd); + cow_header.mtime = cpu_to_be32(st.st_mtime); + mtime_fail: + pstrcpy(cow_header.backing_file, sizeof(cow_header.backing_file), + image_filename); + } + cow_header.sectorsize = cpu_to_be32(512); + cow_header.size = cpu_to_be64(image_sectors * 512); + if(write(cow_fd, &cow_header, sizeof(cow_header))!=sizeof(cow_header)) return -1; + /* resize to include at least all the bitmap */ + if(ftruncate(cow_fd, sizeof(cow_header) + ((image_sectors + 7) >> 3))<0) return -1; + close(cow_fd); + return 0; +} + +static void cow_flush(BlockDriverState *bs) +{ + BDRVCowState *s = bs->opaque; + fsync(s->fd); +} + +BlockDriver bdrv_cow = { + "cow", + sizeof(BDRVCowState), + cow_probe, + cow_open, + cow_read, + cow_write, + cow_close, + cow_create, + cow_flush, + cow_is_allocated, +}; +#endif diff --git a/lib/qemu/block-dmg.c b/lib/qemu/block-dmg.c new file mode 100644 index 0000000..62117c9 --- /dev/null +++ b/lib/qemu/block-dmg.c @@ -0,0 +1,297 @@ +/* + * QEMU Block driver for DMG images + * + * Copyright (c) 2004 Johannes E. Schindelin + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include "qemu-common.h" +#include "block_int.h" +#include "bswap.h" +#include + +typedef struct BDRVDMGState { + int fd; + + /* each chunk contains a certain number of sectors, + * offsets[i] is the offset in the .dmg file, + * lengths[i] is the length of the compressed chunk, + * sectors[i] is the sector beginning at offsets[i], + * sectorcounts[i] is the number of sectors in that chunk, + * the sectors array is ordered + * 0<=i4 && !strcmp(filename+len-4,".dmg")) + return 2; + return 0; +} + +static off_t read_off(int fd) +{ + uint64_t buffer; + if(read(fd,&buffer,8)<8) + return 0; + return be64_to_cpu(buffer); +} + +static off_t read_uint32(int fd) +{ + uint32_t buffer; + if(read(fd,&buffer,4)<4) + return 0; + return be32_to_cpu(buffer); +} + +static int dmg_open(BlockDriverState *bs, const char *filename, int flags) +{ + BDRVDMGState *s = bs->opaque; + off_t info_begin,info_end,last_in_offset,last_out_offset; + uint32_t count; + uint32_t max_compressed_size=1,max_sectors_per_chunk=1,i; + + s->fd = open(filename, O_RDONLY | O_BINARY); + if (s->fd < 0) + return -errno; + bs->read_only = 1; + s->n_chunks = 0; + s->offsets = s->lengths = s->sectors = s->sectorcounts = 0; + + /* read offset of info blocks */ + if(lseek(s->fd,-0x1d8,SEEK_END)<0) { +dmg_close: + close(s->fd); + /* open raw instead */ + bs->drv=&bdrv_raw; + return bs->drv->bdrv_open(bs, filename, flags); + } + info_begin=read_off(s->fd); + if(info_begin==0) + goto dmg_close; + if(lseek(s->fd,info_begin,SEEK_SET)<0) + goto dmg_close; + if(read_uint32(s->fd)!=0x100) + goto dmg_close; + if((count = read_uint32(s->fd))==0) + goto dmg_close; + info_end = info_begin+count; + if(lseek(s->fd,0xf8,SEEK_CUR)<0) + goto dmg_close; + + /* read offsets */ + last_in_offset = last_out_offset = 0; + while(lseek(s->fd,0,SEEK_CUR)fd); + if(count==0) + goto dmg_close; + type = read_uint32(s->fd); + if(type!=0x6d697368 || count<244) + lseek(s->fd,count-4,SEEK_CUR); + else { + int new_size, chunk_count; + if(lseek(s->fd,200,SEEK_CUR)<0) + goto dmg_close; + chunk_count = (count-204)/40; + new_size = sizeof(uint64_t) * (s->n_chunks + chunk_count); + s->types = realloc(s->types, new_size/2); + s->offsets = realloc(s->offsets, new_size); + s->lengths = realloc(s->lengths, new_size); + s->sectors = realloc(s->sectors, new_size); + s->sectorcounts = realloc(s->sectorcounts, new_size); + + for(i=s->n_chunks;in_chunks+chunk_count;i++) { + s->types[i] = read_uint32(s->fd); + if(s->types[i]!=0x80000005 && s->types[i]!=1 && s->types[i]!=2) { + if(s->types[i]==0xffffffff) { + last_in_offset = s->offsets[i-1]+s->lengths[i-1]; + last_out_offset = s->sectors[i-1]+s->sectorcounts[i-1]; + } + chunk_count--; + i--; + if(lseek(s->fd,36,SEEK_CUR)<0) + goto dmg_close; + continue; + } + read_uint32(s->fd); + s->sectors[i] = last_out_offset+read_off(s->fd); + s->sectorcounts[i] = read_off(s->fd); + s->offsets[i] = last_in_offset+read_off(s->fd); + s->lengths[i] = read_off(s->fd); + if(s->lengths[i]>max_compressed_size) + max_compressed_size = s->lengths[i]; + if(s->sectorcounts[i]>max_sectors_per_chunk) + max_sectors_per_chunk = s->sectorcounts[i]; + } + s->n_chunks+=chunk_count; + } + } + + /* initialize zlib engine */ + if(!(s->compressed_chunk = malloc(max_compressed_size+1))) + goto dmg_close; + if(!(s->uncompressed_chunk = malloc(512*max_sectors_per_chunk))) + goto dmg_close; + if(inflateInit(&s->zstream) != Z_OK) + goto dmg_close; + + s->current_chunk = s->n_chunks; + + return 0; +} + +static inline int is_sector_in_chunk(BDRVDMGState* s, + uint32_t chunk_num,int sector_num) +{ + if(chunk_num>=s->n_chunks || s->sectors[chunk_num]>sector_num || + s->sectors[chunk_num]+s->sectorcounts[chunk_num]<=sector_num) + return 0; + else + return -1; +} + +static inline uint32_t search_chunk(BDRVDMGState* s,int sector_num) +{ + /* binary search */ + uint32_t chunk1=0,chunk2=s->n_chunks,chunk3; + while(chunk1!=chunk2) { + chunk3 = (chunk1+chunk2)/2; + if(s->sectors[chunk3]>sector_num) + chunk2 = chunk3; + else if(s->sectors[chunk3]+s->sectorcounts[chunk3]>sector_num) + return chunk3; + else + chunk1 = chunk3; + } + return s->n_chunks; /* error */ +} + +static inline int dmg_read_chunk(BDRVDMGState *s,int sector_num) +{ + if(!is_sector_in_chunk(s,s->current_chunk,sector_num)) { + int ret; + uint32_t chunk = search_chunk(s,sector_num); + + if(chunk>=s->n_chunks) + return -1; + + s->current_chunk = s->n_chunks; + switch(s->types[chunk]) { + case 0x80000005: { /* zlib compressed */ + int i; + + ret = lseek(s->fd, s->offsets[chunk], SEEK_SET); + if(ret<0) + return -1; + + /* we need to buffer, because only the chunk as whole can be + * inflated. */ + i=0; + do { + ret = read(s->fd, s->compressed_chunk+i, s->lengths[chunk]-i); + if(ret<0 && errno==EINTR) + ret=0; + i+=ret; + } while(ret>=0 && ret+ilengths[chunk]); + + if (ret != s->lengths[chunk]) + return -1; + + s->zstream.next_in = s->compressed_chunk; + s->zstream.avail_in = s->lengths[chunk]; + s->zstream.next_out = s->uncompressed_chunk; + s->zstream.avail_out = 512*s->sectorcounts[chunk]; + ret = inflateReset(&s->zstream); + if(ret != Z_OK) + return -1; + ret = inflate(&s->zstream, Z_FINISH); + if(ret != Z_STREAM_END || s->zstream.total_out != 512*s->sectorcounts[chunk]) + return -1; + break; } + case 1: /* copy */ + ret = read(s->fd, s->uncompressed_chunk, s->lengths[chunk]); + if (ret != s->lengths[chunk]) + return -1; + break; + case 2: /* zero */ + memset(s->uncompressed_chunk, 0, 512*s->sectorcounts[chunk]); + break; + } + s->current_chunk = chunk; + } + return 0; +} + +static int dmg_read(BlockDriverState *bs, int64_t sector_num, + uint8_t *buf, int nb_sectors) +{ + BDRVDMGState *s = bs->opaque; + int i; + + for(i=0;isectors[s->current_chunk]; + memcpy(buf+i*512,s->uncompressed_chunk+sector_offset_in_chunk*512,512); + } + return 0; +} + +static void dmg_close(BlockDriverState *bs) +{ + BDRVDMGState *s = bs->opaque; + close(s->fd); + if(s->n_chunks>0) { + free(s->types); + free(s->offsets); + free(s->lengths); + free(s->sectors); + free(s->sectorcounts); + } + free(s->compressed_chunk); + free(s->uncompressed_chunk); + inflateEnd(&s->zstream); +} + +BlockDriver bdrv_dmg = { + "dmg", + sizeof(BDRVDMGState), + dmg_probe, + dmg_open, + dmg_read, + NULL, + dmg_close, +}; + diff --git a/lib/qemu/block-parallels.c b/lib/qemu/block-parallels.c new file mode 100644 index 0000000..4654b07 --- /dev/null +++ b/lib/qemu/block-parallels.c @@ -0,0 +1,176 @@ +/* + * Block driver for Parallels disk image format + * + * Copyright (c) 2007 Alex Beregszaszi + * + * This code is based on comparing different disk images created by Parallels. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include "qemu-common.h" +#include "block_int.h" + +/**************************************************************/ + +#define HEADER_MAGIC "WithoutFreeSpace" +#define HEADER_VERSION 2 +#define HEADER_SIZE 64 + +// always little-endian +struct parallels_header { + char magic[16]; // "WithoutFreeSpace" + uint32_t version; + uint32_t heads; + uint32_t cylinders; + uint32_t tracks; + uint32_t catalog_entries; + uint32_t nb_sectors; + char padding[24]; +} __attribute__((packed)); + +typedef struct BDRVParallelsState { + int fd; + + uint32_t *catalog_bitmap; + int catalog_size; + + int tracks; +} BDRVParallelsState; + +static int parallels_probe(const uint8_t *buf, int buf_size, const char *filename) +{ + const struct parallels_header *ph = (const void *)buf; + + if (buf_size < HEADER_SIZE) + return 0; + + if (!memcmp(ph->magic, HEADER_MAGIC, 16) && + (le32_to_cpu(ph->version) == HEADER_VERSION)) + return 100; + + return 0; +} + +static int parallels_open(BlockDriverState *bs, const char *filename, int flags) +{ + BDRVParallelsState *s = bs->opaque; + int fd, i; + struct parallels_header ph; + + fd = open(filename, O_RDWR | O_BINARY | O_LARGEFILE); + if (fd < 0) { + fd = open(filename, O_RDONLY | O_BINARY | O_LARGEFILE); + if (fd < 0) + return -1; + } + + bs->read_only = 1; // no write support yet + + s->fd = fd; + + if (read(fd, &ph, sizeof(ph)) != sizeof(ph)) + goto fail; + + if (memcmp(ph.magic, HEADER_MAGIC, 16) || + (le32_to_cpu(ph.version) != HEADER_VERSION)) { + goto fail; + } + + bs->total_sectors = le32_to_cpu(ph.nb_sectors); + + if (lseek(s->fd, 64, SEEK_SET) != 64) + goto fail; + + s->tracks = le32_to_cpu(ph.tracks); + + s->catalog_size = le32_to_cpu(ph.catalog_entries); + s->catalog_bitmap = qemu_malloc(s->catalog_size * 4); + if (!s->catalog_bitmap) + goto fail; + if (read(s->fd, s->catalog_bitmap, s->catalog_size * 4) != + s->catalog_size * 4) + goto fail; + for (i = 0; i < s->catalog_size; i++) + le32_to_cpus(&s->catalog_bitmap[i]); + + return 0; +fail: + if (s->catalog_bitmap) + qemu_free(s->catalog_bitmap); + close(fd); + return -1; +} + +static inline int seek_to_sector(BlockDriverState *bs, int64_t sector_num) +{ + BDRVParallelsState *s = bs->opaque; + uint32_t index, offset, position; + + index = sector_num / s->tracks; + offset = sector_num % s->tracks; + + // not allocated + if ((index > s->catalog_size) || (s->catalog_bitmap[index] == 0)) + return -1; + + position = (s->catalog_bitmap[index] + offset) * 512; + +// fprintf(stderr, "sector: %llx index=%x offset=%x pointer=%x position=%x\n", +// sector_num, index, offset, s->catalog_bitmap[index], position); + + if (lseek(s->fd, position, SEEK_SET) != position) + return -1; + + return 0; +} + +static int parallels_read(BlockDriverState *bs, int64_t sector_num, + uint8_t *buf, int nb_sectors) +{ + BDRVParallelsState *s = bs->opaque; + + while (nb_sectors > 0) { + if (!seek_to_sector(bs, sector_num)) { + if (read(s->fd, buf, 512) != 512) + return -1; + } else + memset(buf, 0, 512); + nb_sectors--; + sector_num++; + buf += 512; + } + return 0; +} + +static void parallels_close(BlockDriverState *bs) +{ + BDRVParallelsState *s = bs->opaque; + qemu_free(s->catalog_bitmap); + close(s->fd); +} + +BlockDriver bdrv_parallels = { + "parallels", + sizeof(BDRVParallelsState), + parallels_probe, + parallels_open, + parallels_read, + NULL, + parallels_close, +}; diff --git a/lib/qemu/block-qcow.c b/lib/qemu/block-qcow.c new file mode 100644 index 0000000..c7b526f --- /dev/null +++ b/lib/qemu/block-qcow.c @@ -0,0 +1,905 @@ +/* + * Block driver for the QCOW format + * + * Copyright (c) 2004-2006 Fabrice Bellard + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include "qemu-common.h" +#include "block_int.h" +#include +#include "aes.h" + +/**************************************************************/ +/* QEMU COW block driver with compression and encryption support */ + +#define QCOW_MAGIC (('Q' << 24) | ('F' << 16) | ('I' << 8) | 0xfb) +#define QCOW_VERSION 1 + +#define QCOW_CRYPT_NONE 0 +#define QCOW_CRYPT_AES 1 + +#define QCOW_OFLAG_COMPRESSED (1LL << 63) + +typedef struct QCowHeader { + uint32_t magic; + uint32_t version; + uint64_t backing_file_offset; + uint32_t backing_file_size; + uint32_t mtime; + uint64_t size; /* in bytes */ + uint8_t cluster_bits; + uint8_t l2_bits; + uint32_t crypt_method; + uint64_t l1_table_offset; +} QCowHeader; + +#define L2_CACHE_SIZE 16 + +typedef struct BDRVQcowState { + BlockDriverState *hd; + int cluster_bits; + int cluster_size; + int cluster_sectors; + int l2_bits; + int l2_size; + int l1_size; + uint64_t cluster_offset_mask; + uint64_t l1_table_offset; + uint64_t *l1_table; + uint64_t *l2_cache; + uint64_t l2_cache_offsets[L2_CACHE_SIZE]; + uint32_t l2_cache_counts[L2_CACHE_SIZE]; + uint8_t *cluster_cache; + uint8_t *cluster_data; + uint64_t cluster_cache_offset; + uint32_t crypt_method; /* current crypt method, 0 if no key yet */ + uint32_t crypt_method_header; + AES_KEY aes_encrypt_key; + AES_KEY aes_decrypt_key; +} BDRVQcowState; + +static int decompress_cluster(BDRVQcowState *s, uint64_t cluster_offset); + +static int qcow_probe(const uint8_t *buf, int buf_size, const char *filename) +{ + const QCowHeader *cow_header = (const void *)buf; + + if (buf_size >= sizeof(QCowHeader) && + be32_to_cpu(cow_header->magic) == QCOW_MAGIC && + be32_to_cpu(cow_header->version) == QCOW_VERSION) + return 100; + else + return 0; +} + +static int qcow_open(BlockDriverState *bs, const char *filename, int flags) +{ + BDRVQcowState *s = bs->opaque; + int len, i, shift, ret; + QCowHeader header; + + ret = bdrv_file_open(&s->hd, filename, flags); + if (ret < 0) + return ret; + if (bdrv_pread(s->hd, 0, &header, sizeof(header)) != sizeof(header)) + goto fail; + be32_to_cpus(&header.magic); + be32_to_cpus(&header.version); + be64_to_cpus(&header.backing_file_offset); + be32_to_cpus(&header.backing_file_size); + be32_to_cpus(&header.mtime); + be64_to_cpus(&header.size); + be32_to_cpus(&header.crypt_method); + be64_to_cpus(&header.l1_table_offset); + + if (header.magic != QCOW_MAGIC || header.version != QCOW_VERSION) + goto fail; + if (header.size <= 1 || header.cluster_bits < 9) + goto fail; + if (header.crypt_method > QCOW_CRYPT_AES) + goto fail; + s->crypt_method_header = header.crypt_method; + if (s->crypt_method_header) + bs->encrypted = 1; + s->cluster_bits = header.cluster_bits; + s->cluster_size = 1 << s->cluster_bits; + s->cluster_sectors = 1 << (s->cluster_bits - 9); + s->l2_bits = header.l2_bits; + s->l2_size = 1 << s->l2_bits; + bs->total_sectors = header.size / 512; + s->cluster_offset_mask = (1LL << (63 - s->cluster_bits)) - 1; + + /* read the level 1 table */ + shift = s->cluster_bits + s->l2_bits; + s->l1_size = (header.size + (1LL << shift) - 1) >> shift; + + s->l1_table_offset = header.l1_table_offset; + s->l1_table = qemu_malloc(s->l1_size * sizeof(uint64_t)); + if (!s->l1_table) + goto fail; + if (bdrv_pread(s->hd, s->l1_table_offset, s->l1_table, s->l1_size * sizeof(uint64_t)) != + s->l1_size * sizeof(uint64_t)) + goto fail; + for(i = 0;i < s->l1_size; i++) { + be64_to_cpus(&s->l1_table[i]); + } + /* alloc L2 cache */ + s->l2_cache = qemu_malloc(s->l2_size * L2_CACHE_SIZE * sizeof(uint64_t)); + if (!s->l2_cache) + goto fail; + s->cluster_cache = qemu_malloc(s->cluster_size); + if (!s->cluster_cache) + goto fail; + s->cluster_data = qemu_malloc(s->cluster_size); + if (!s->cluster_data) + goto fail; + s->cluster_cache_offset = -1; + + /* read the backing file name */ + if (header.backing_file_offset != 0) { + len = header.backing_file_size; + if (len > 1023) + len = 1023; + if (bdrv_pread(s->hd, header.backing_file_offset, bs->backing_file, len) != len) + goto fail; + bs->backing_file[len] = '\0'; + } + return 0; + + fail: + qemu_free(s->l1_table); + qemu_free(s->l2_cache); + qemu_free(s->cluster_cache); + qemu_free(s->cluster_data); + bdrv_delete(s->hd); + return -1; +} + +static int qcow_set_key(BlockDriverState *bs, const char *key) +{ + BDRVQcowState *s = bs->opaque; + uint8_t keybuf[16]; + int len, i; + + memset(keybuf, 0, 16); + len = strlen(key); + if (len > 16) + len = 16; + /* XXX: we could compress the chars to 7 bits to increase + entropy */ + for(i = 0;i < len;i++) { + keybuf[i] = key[i]; + } + s->crypt_method = s->crypt_method_header; + + if (AES_set_encrypt_key(keybuf, 128, &s->aes_encrypt_key) != 0) + return -1; + if (AES_set_decrypt_key(keybuf, 128, &s->aes_decrypt_key) != 0) + return -1; +#if 0 + /* test */ + { + uint8_t in[16]; + uint8_t out[16]; + uint8_t tmp[16]; + for(i=0;i<16;i++) + in[i] = i; + AES_encrypt(in, tmp, &s->aes_encrypt_key); + AES_decrypt(tmp, out, &s->aes_decrypt_key); + for(i = 0; i < 16; i++) + printf(" %02x", tmp[i]); + printf("\n"); + for(i = 0; i < 16; i++) + printf(" %02x", out[i]); + printf("\n"); + } +#endif + return 0; +} + +/* The crypt function is compatible with the linux cryptoloop + algorithm for < 4 GB images. NOTE: out_buf == in_buf is + supported */ +static void encrypt_sectors(BDRVQcowState *s, int64_t sector_num, + uint8_t *out_buf, const uint8_t *in_buf, + int nb_sectors, int enc, + const AES_KEY *key) +{ + union { + uint64_t ll[2]; + uint8_t b[16]; + } ivec; + int i; + + for(i = 0; i < nb_sectors; i++) { + ivec.ll[0] = cpu_to_le64(sector_num); + ivec.ll[1] = 0; + AES_cbc_encrypt(in_buf, out_buf, 512, key, + ivec.b, enc); + sector_num++; + in_buf += 512; + out_buf += 512; + } +} + +/* 'allocate' is: + * + * 0 to not allocate. + * + * 1 to allocate a normal cluster (for sector indexes 'n_start' to + * 'n_end') + * + * 2 to allocate a compressed cluster of size + * 'compressed_size'. 'compressed_size' must be > 0 and < + * cluster_size + * + * return 0 if not allocated. + */ +static uint64_t get_cluster_offset(BlockDriverState *bs, + uint64_t offset, int allocate, + int compressed_size, + int n_start, int n_end) +{ + BDRVQcowState *s = bs->opaque; + int min_index, i, j, l1_index, l2_index; + uint64_t l2_offset, *l2_table, cluster_offset, tmp; + uint32_t min_count; + int new_l2_table; + + l1_index = offset >> (s->l2_bits + s->cluster_bits); + l2_offset = s->l1_table[l1_index]; + new_l2_table = 0; + if (!l2_offset) { + if (!allocate) + return 0; + /* allocate a new l2 entry */ + l2_offset = bdrv_getlength(s->hd); + /* round to cluster size */ + l2_offset = (l2_offset + s->cluster_size - 1) & ~(s->cluster_size - 1); + /* update the L1 entry */ + s->l1_table[l1_index] = l2_offset; + tmp = cpu_to_be64(l2_offset); + if (bdrv_pwrite(s->hd, s->l1_table_offset + l1_index * sizeof(tmp), + &tmp, sizeof(tmp)) != sizeof(tmp)) + return 0; + new_l2_table = 1; + } + for(i = 0; i < L2_CACHE_SIZE; i++) { + if (l2_offset == s->l2_cache_offsets[i]) { + /* increment the hit count */ + if (++s->l2_cache_counts[i] == 0xffffffff) { + for(j = 0; j < L2_CACHE_SIZE; j++) { + s->l2_cache_counts[j] >>= 1; + } + } + l2_table = s->l2_cache + (i << s->l2_bits); + goto found; + } + } + /* not found: load a new entry in the least used one */ + min_index = 0; + min_count = 0xffffffff; + for(i = 0; i < L2_CACHE_SIZE; i++) { + if (s->l2_cache_counts[i] < min_count) { + min_count = s->l2_cache_counts[i]; + min_index = i; + } + } + l2_table = s->l2_cache + (min_index << s->l2_bits); + if (new_l2_table) { + memset(l2_table, 0, s->l2_size * sizeof(uint64_t)); + if (bdrv_pwrite(s->hd, l2_offset, l2_table, s->l2_size * sizeof(uint64_t)) != + s->l2_size * sizeof(uint64_t)) + return 0; + } else { + if (bdrv_pread(s->hd, l2_offset, l2_table, s->l2_size * sizeof(uint64_t)) != + s->l2_size * sizeof(uint64_t)) + return 0; + } + s->l2_cache_offsets[min_index] = l2_offset; + s->l2_cache_counts[min_index] = 1; + found: + l2_index = (offset >> s->cluster_bits) & (s->l2_size - 1); + cluster_offset = be64_to_cpu(l2_table[l2_index]); + if (!cluster_offset || + ((cluster_offset & QCOW_OFLAG_COMPRESSED) && allocate == 1)) { + if (!allocate) + return 0; + /* allocate a new cluster */ + if ((cluster_offset & QCOW_OFLAG_COMPRESSED) && + (n_end - n_start) < s->cluster_sectors) { + /* if the cluster is already compressed, we must + decompress it in the case it is not completely + overwritten */ + if (decompress_cluster(s, cluster_offset) < 0) + return 0; + cluster_offset = bdrv_getlength(s->hd); + cluster_offset = (cluster_offset + s->cluster_size - 1) & + ~(s->cluster_size - 1); + /* write the cluster content */ + if (bdrv_pwrite(s->hd, cluster_offset, s->cluster_cache, s->cluster_size) != + s->cluster_size) + return -1; + } else { + cluster_offset = bdrv_getlength(s->hd); + if (allocate == 1) { + /* round to cluster size */ + cluster_offset = (cluster_offset + s->cluster_size - 1) & + ~(s->cluster_size - 1); + bdrv_truncate(s->hd, cluster_offset + s->cluster_size); + /* if encrypted, we must initialize the cluster + content which won't be written */ + if (s->crypt_method && + (n_end - n_start) < s->cluster_sectors) { + uint64_t start_sect; + start_sect = (offset & ~(s->cluster_size - 1)) >> 9; + memset(s->cluster_data + 512, 0x00, 512); + for(i = 0; i < s->cluster_sectors; i++) { + if (i < n_start || i >= n_end) { + encrypt_sectors(s, start_sect + i, + s->cluster_data, + s->cluster_data + 512, 1, 1, + &s->aes_encrypt_key); + if (bdrv_pwrite(s->hd, cluster_offset + i * 512, + s->cluster_data, 512) != 512) + return -1; + } + } + } + } else { + cluster_offset |= QCOW_OFLAG_COMPRESSED | + (uint64_t)compressed_size << (63 - s->cluster_bits); + } + } + /* update L2 table */ + tmp = cpu_to_be64(cluster_offset); + l2_table[l2_index] = tmp; + if (bdrv_pwrite(s->hd, + l2_offset + l2_index * sizeof(tmp), &tmp, sizeof(tmp)) != sizeof(tmp)) + return 0; + } + return cluster_offset; +} + +static int qcow_is_allocated(BlockDriverState *bs, int64_t sector_num, + int nb_sectors, int *pnum) +{ + BDRVQcowState *s = bs->opaque; + int index_in_cluster, n; + uint64_t cluster_offset; + + cluster_offset = get_cluster_offset(bs, sector_num << 9, 0, 0, 0, 0); + index_in_cluster = sector_num & (s->cluster_sectors - 1); + n = s->cluster_sectors - index_in_cluster; + if (n > nb_sectors) + n = nb_sectors; + *pnum = n; + return (cluster_offset != 0); +} + +static int decompress_buffer(uint8_t *out_buf, int out_buf_size, + const uint8_t *buf, int buf_size) +{ + z_stream strm1, *strm = &strm1; + int ret, out_len; + + memset(strm, 0, sizeof(*strm)); + + strm->next_in = (uint8_t *)buf; + strm->avail_in = buf_size; + strm->next_out = out_buf; + strm->avail_out = out_buf_size; + + ret = inflateInit2(strm, -12); + if (ret != Z_OK) + return -1; + ret = inflate(strm, Z_FINISH); + out_len = strm->next_out - out_buf; + if ((ret != Z_STREAM_END && ret != Z_BUF_ERROR) || + out_len != out_buf_size) { + inflateEnd(strm); + return -1; + } + inflateEnd(strm); + return 0; +} + +static int decompress_cluster(BDRVQcowState *s, uint64_t cluster_offset) +{ + int ret, csize; + uint64_t coffset; + + coffset = cluster_offset & s->cluster_offset_mask; + if (s->cluster_cache_offset != coffset) { + csize = cluster_offset >> (63 - s->cluster_bits); + csize &= (s->cluster_size - 1); + ret = bdrv_pread(s->hd, coffset, s->cluster_data, csize); + if (ret != csize) + return -1; + if (decompress_buffer(s->cluster_cache, s->cluster_size, + s->cluster_data, csize) < 0) { + return -1; + } + s->cluster_cache_offset = coffset; + } + return 0; +} + +#if 0 + +static int qcow_read(BlockDriverState *bs, int64_t sector_num, + uint8_t *buf, int nb_sectors) +{ + BDRVQcowState *s = bs->opaque; + int ret, index_in_cluster, n; + uint64_t cluster_offset; + + while (nb_sectors > 0) { + cluster_offset = get_cluster_offset(bs, sector_num << 9, 0, 0, 0, 0); + index_in_cluster = sector_num & (s->cluster_sectors - 1); + n = s->cluster_sectors - index_in_cluster; + if (n > nb_sectors) + n = nb_sectors; + if (!cluster_offset) { + if (bs->backing_hd) { + /* read from the base image */ + ret = bdrv_read(bs->backing_hd, sector_num, buf, n); + if (ret < 0) + return -1; + } else { + memset(buf, 0, 512 * n); + } + } else if (cluster_offset & QCOW_OFLAG_COMPRESSED) { + if (decompress_cluster(s, cluster_offset) < 0) + return -1; + memcpy(buf, s->cluster_cache + index_in_cluster * 512, 512 * n); + } else { + ret = bdrv_pread(s->hd, cluster_offset + index_in_cluster * 512, buf, n * 512); + if (ret != n * 512) + return -1; + if (s->crypt_method) { + encrypt_sectors(s, sector_num, buf, buf, n, 0, + &s->aes_decrypt_key); + } + } + nb_sectors -= n; + sector_num += n; + buf += n * 512; + } + return 0; +} +#endif + +static int qcow_write(BlockDriverState *bs, int64_t sector_num, + const uint8_t *buf, int nb_sectors) +{ + BDRVQcowState *s = bs->opaque; + int ret, index_in_cluster, n; + uint64_t cluster_offset; + + while (nb_sectors > 0) { + index_in_cluster = sector_num & (s->cluster_sectors - 1); + n = s->cluster_sectors - index_in_cluster; + if (n > nb_sectors) + n = nb_sectors; + cluster_offset = get_cluster_offset(bs, sector_num << 9, 1, 0, + index_in_cluster, + index_in_cluster + n); + if (!cluster_offset) + return -1; + if (s->crypt_method) { + encrypt_sectors(s, sector_num, s->cluster_data, buf, n, 1, + &s->aes_encrypt_key); + ret = bdrv_pwrite(s->hd, cluster_offset + index_in_cluster * 512, + s->cluster_data, n * 512); + } else { + ret = bdrv_pwrite(s->hd, cluster_offset + index_in_cluster * 512, buf, n * 512); + } + if (ret != n * 512) + return -1; + nb_sectors -= n; + sector_num += n; + buf += n * 512; + } + s->cluster_cache_offset = -1; /* disable compressed cache */ + return 0; +} + +typedef struct QCowAIOCB { + BlockDriverAIOCB common; + int64_t sector_num; + uint8_t *buf; + int nb_sectors; + int n; + uint64_t cluster_offset; + uint8_t *cluster_data; + BlockDriverAIOCB *hd_aiocb; +} QCowAIOCB; + +static void qcow_aio_read_cb(void *opaque, int ret) +{ + QCowAIOCB *acb = opaque; + BlockDriverState *bs = acb->common.bs; + BDRVQcowState *s = bs->opaque; + int index_in_cluster; + + acb->hd_aiocb = NULL; + if (ret < 0) { + fail: + acb->common.cb(acb->common.opaque, ret); + qemu_aio_release(acb); + return; + } + + redo: + /* post process the read buffer */ + if (!acb->cluster_offset) { + /* nothing to do */ + } else if (acb->cluster_offset & QCOW_OFLAG_COMPRESSED) { + /* nothing to do */ + } else { + if (s->crypt_method) { + encrypt_sectors(s, acb->sector_num, acb->buf, acb->buf, + acb->n, 0, + &s->aes_decrypt_key); + } + } + + acb->nb_sectors -= acb->n; + acb->sector_num += acb->n; + acb->buf += acb->n * 512; + + if (acb->nb_sectors == 0) { + /* request completed */ + acb->common.cb(acb->common.opaque, 0); + qemu_aio_release(acb); + return; + } + + /* prepare next AIO request */ + acb->cluster_offset = get_cluster_offset(bs, acb->sector_num << 9, + 0, 0, 0, 0); + index_in_cluster = acb->sector_num & (s->cluster_sectors - 1); + acb->n = s->cluster_sectors - index_in_cluster; + if (acb->n > acb->nb_sectors) + acb->n = acb->nb_sectors; + + if (!acb->cluster_offset) { + if (bs->backing_hd) { + /* read from the base image */ + acb->hd_aiocb = bdrv_aio_read(bs->backing_hd, + acb->sector_num, acb->buf, acb->n, qcow_aio_read_cb, acb); + if (acb->hd_aiocb == NULL) + goto fail; + } else { + /* Note: in this case, no need to wait */ + memset(acb->buf, 0, 512 * acb->n); + goto redo; + } + } else if (acb->cluster_offset & QCOW_OFLAG_COMPRESSED) { + /* add AIO support for compressed blocks ? */ + if (decompress_cluster(s, acb->cluster_offset) < 0) + goto fail; + memcpy(acb->buf, + s->cluster_cache + index_in_cluster * 512, 512 * acb->n); + goto redo; + } else { + if ((acb->cluster_offset & 511) != 0) { + ret = -EIO; + goto fail; + } + acb->hd_aiocb = bdrv_aio_read(s->hd, + (acb->cluster_offset >> 9) + index_in_cluster, + acb->buf, acb->n, qcow_aio_read_cb, acb); + if (acb->hd_aiocb == NULL) + goto fail; + } +} + +static BlockDriverAIOCB *qcow_aio_read(BlockDriverState *bs, + int64_t sector_num, uint8_t *buf, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque) +{ + QCowAIOCB *acb; + + acb = qemu_aio_get(bs, cb, opaque); + if (!acb) + return NULL; + acb->hd_aiocb = NULL; + acb->sector_num = sector_num; + acb->buf = buf; + acb->nb_sectors = nb_sectors; + acb->n = 0; + acb->cluster_offset = 0; + + qcow_aio_read_cb(acb, 0); + return &acb->common; +} + +static void qcow_aio_write_cb(void *opaque, int ret) +{ + QCowAIOCB *acb = opaque; + BlockDriverState *bs = acb->common.bs; + BDRVQcowState *s = bs->opaque; + int index_in_cluster; + uint64_t cluster_offset; + const uint8_t *src_buf; + + acb->hd_aiocb = NULL; + + if (ret < 0) { + fail: + acb->common.cb(acb->common.opaque, ret); + qemu_aio_release(acb); + return; + } + + acb->nb_sectors -= acb->n; + acb->sector_num += acb->n; + acb->buf += acb->n * 512; + + if (acb->nb_sectors == 0) { + /* request completed */ + acb->common.cb(acb->common.opaque, 0); + qemu_aio_release(acb); + return; + } + + index_in_cluster = acb->sector_num & (s->cluster_sectors - 1); + acb->n = s->cluster_sectors - index_in_cluster; + if (acb->n > acb->nb_sectors) + acb->n = acb->nb_sectors; + cluster_offset = get_cluster_offset(bs, acb->sector_num << 9, 1, 0, + index_in_cluster, + index_in_cluster + acb->n); + if (!cluster_offset || (cluster_offset & 511) != 0) { + ret = -EIO; + goto fail; + } + if (s->crypt_method) { + if (!acb->cluster_data) { + acb->cluster_data = qemu_mallocz(s->cluster_size); + if (!acb->cluster_data) { + ret = -ENOMEM; + goto fail; + } + } + encrypt_sectors(s, acb->sector_num, acb->cluster_data, acb->buf, + acb->n, 1, &s->aes_encrypt_key); + src_buf = acb->cluster_data; + } else { + src_buf = acb->buf; + } + acb->hd_aiocb = bdrv_aio_write(s->hd, + (cluster_offset >> 9) + index_in_cluster, + src_buf, acb->n, + qcow_aio_write_cb, acb); + if (acb->hd_aiocb == NULL) + goto fail; +} + +static BlockDriverAIOCB *qcow_aio_write(BlockDriverState *bs, + int64_t sector_num, const uint8_t *buf, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque) +{ + BDRVQcowState *s = bs->opaque; + QCowAIOCB *acb; + + s->cluster_cache_offset = -1; /* disable compressed cache */ + + acb = qemu_aio_get(bs, cb, opaque); + if (!acb) + return NULL; + acb->hd_aiocb = NULL; + acb->sector_num = sector_num; + acb->buf = (uint8_t *)buf; + acb->nb_sectors = nb_sectors; + acb->n = 0; + + qcow_aio_write_cb(acb, 0); + return &acb->common; +} + +static void qcow_aio_cancel(BlockDriverAIOCB *blockacb) +{ + QCowAIOCB *acb = (QCowAIOCB *)blockacb; + if (acb->hd_aiocb) + bdrv_aio_cancel(acb->hd_aiocb); + qemu_aio_release(acb); +} + +static void qcow_close(BlockDriverState *bs) +{ + BDRVQcowState *s = bs->opaque; + qemu_free(s->l1_table); + qemu_free(s->l2_cache); + qemu_free(s->cluster_cache); + qemu_free(s->cluster_data); + bdrv_delete(s->hd); +} + +static int qcow_create(const char *filename, int64_t total_size, + const char *backing_file, int flags) +{ + int fd, header_size, backing_filename_len, l1_size, i, shift; + QCowHeader header; + uint64_t tmp; + + fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0644); + if (fd < 0) + return -1; + memset(&header, 0, sizeof(header)); + header.magic = cpu_to_be32(QCOW_MAGIC); + header.version = cpu_to_be32(QCOW_VERSION); + header.size = cpu_to_be64(total_size * 512); + header_size = sizeof(header); + backing_filename_len = 0; + if (backing_file) { + header.backing_file_offset = cpu_to_be64(header_size); + backing_filename_len = strlen(backing_file); + header.backing_file_size = cpu_to_be32(backing_filename_len); + header_size += backing_filename_len; + header.mtime = cpu_to_be32(0); + header.cluster_bits = 9; /* 512 byte cluster to avoid copying + unmodifyed sectors */ + header.l2_bits = 12; /* 32 KB L2 tables */ + } else { + header.cluster_bits = 12; /* 4 KB clusters */ + header.l2_bits = 9; /* 4 KB L2 tables */ + } + header_size = (header_size + 7) & ~7; + shift = header.cluster_bits + header.l2_bits; + l1_size = ((total_size * 512) + (1LL << shift) - 1) >> shift; + + header.l1_table_offset = cpu_to_be64(header_size); + if (flags & BLOCK_FLAG_ENCRYPT) { + header.crypt_method = cpu_to_be32(QCOW_CRYPT_AES); + } else { + header.crypt_method = cpu_to_be32(QCOW_CRYPT_NONE); + } + + /* write all the data */ + if(write(fd, &header, sizeof(header))!=sizeof(header)) return -1; + if (backing_file) { + if(write(fd, backing_file, backing_filename_len)!=backing_filename_len) return -1; + } + lseek(fd, header_size, SEEK_SET); + tmp = 0; + for(i = 0;i < l1_size; i++) { + if(write(fd, &tmp, sizeof(tmp))!=sizeof(tmp)) return -1; + } + close(fd); + return 0; +} + +static int qcow_make_empty(BlockDriverState *bs) +{ + BDRVQcowState *s = bs->opaque; + uint32_t l1_length = s->l1_size * sizeof(uint64_t); + int ret; + + memset(s->l1_table, 0, l1_length); + if (bdrv_pwrite(s->hd, s->l1_table_offset, s->l1_table, l1_length) < 0) + return -1; + ret = bdrv_truncate(s->hd, s->l1_table_offset + l1_length); + if (ret < 0) + return ret; + + memset(s->l2_cache, 0, s->l2_size * L2_CACHE_SIZE * sizeof(uint64_t)); + memset(s->l2_cache_offsets, 0, L2_CACHE_SIZE * sizeof(uint64_t)); + memset(s->l2_cache_counts, 0, L2_CACHE_SIZE * sizeof(uint32_t)); + + return 0; +} + +/* XXX: put compressed sectors first, then all the cluster aligned + tables to avoid losing bytes in alignment */ +static int qcow_write_compressed(BlockDriverState *bs, int64_t sector_num, + const uint8_t *buf, int nb_sectors) +{ + BDRVQcowState *s = bs->opaque; + z_stream strm; + int ret, out_len; + uint8_t *out_buf; + uint64_t cluster_offset; + + if (nb_sectors != s->cluster_sectors) + return -EINVAL; + + out_buf = qemu_malloc(s->cluster_size + (s->cluster_size / 1000) + 128); + if (!out_buf) + return -1; + + /* best compression, small window, no zlib header */ + memset(&strm, 0, sizeof(strm)); + ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, + Z_DEFLATED, -12, + 9, Z_DEFAULT_STRATEGY); + if (ret != 0) { + qemu_free(out_buf); + return -1; + } + + strm.avail_in = s->cluster_size; + strm.next_in = (uint8_t *)buf; + strm.avail_out = s->cluster_size; + strm.next_out = out_buf; + + ret = deflate(&strm, Z_FINISH); + if (ret != Z_STREAM_END && ret != Z_OK) { + qemu_free(out_buf); + deflateEnd(&strm); + return -1; + } + out_len = strm.next_out - out_buf; + + deflateEnd(&strm); + + if (ret != Z_STREAM_END || out_len >= s->cluster_size) { + /* could not compress: write normal cluster */ + qcow_write(bs, sector_num, buf, s->cluster_sectors); + } else { + cluster_offset = get_cluster_offset(bs, sector_num << 9, 2, + out_len, 0, 0); + cluster_offset &= s->cluster_offset_mask; + if (bdrv_pwrite(s->hd, cluster_offset, out_buf, out_len) != out_len) { + qemu_free(out_buf); + return -1; + } + } + + qemu_free(out_buf); + return 0; +} + +static void qcow_flush(BlockDriverState *bs) +{ + BDRVQcowState *s = bs->opaque; + bdrv_flush(s->hd); +} + +static int qcow_get_info(BlockDriverState *bs, BlockDriverInfo *bdi) +{ + BDRVQcowState *s = bs->opaque; + bdi->cluster_size = s->cluster_size; + return 0; +} + +BlockDriver bdrv_qcow = { + "qcow", + sizeof(BDRVQcowState), + qcow_probe, + qcow_open, + NULL, + NULL, + qcow_close, + qcow_create, + qcow_flush, + qcow_is_allocated, + qcow_set_key, + qcow_make_empty, + + .bdrv_aio_read = qcow_aio_read, + .bdrv_aio_write = qcow_aio_write, + .bdrv_aio_cancel = qcow_aio_cancel, + .aiocb_size = sizeof(QCowAIOCB), + .bdrv_write_compressed = qcow_write_compressed, + .bdrv_get_info = qcow_get_info, +}; diff --git a/lib/qemu/block-qcow2.c b/lib/qemu/block-qcow2.c new file mode 100644 index 0000000..1eaa214 --- /dev/null +++ b/lib/qemu/block-qcow2.c @@ -0,0 +1,2252 @@ +/* + * Block driver for the QCOW version 2 format + * + * Copyright (c) 2004-2006 Fabrice Bellard + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include "qemu-common.h" +#include "block_int.h" +#include +#include "aes.h" +#include + +/* + Differences with QCOW: + + - Support for multiple incremental snapshots. + - Memory management by reference counts. + - Clusters which have a reference count of one have the bit + QCOW_OFLAG_COPIED to optimize write performance. + - Size of compressed clusters is stored in sectors to reduce bit usage + in the cluster offsets. + - Support for storing additional data (such as the VM state) in the + snapshots. + - If a backing store is used, the cluster size is not constrained + (could be backported to QCOW). + - L2 tables have always a size of one cluster. +*/ + +//#define DEBUG_ALLOC +//#define DEBUG_ALLOC2 + +#define QCOW_MAGIC (('Q' << 24) | ('F' << 16) | ('I' << 8) | 0xfb) +#define QCOW_VERSION 2 + +#define QCOW_CRYPT_NONE 0 +#define QCOW_CRYPT_AES 1 + +/* indicate that the refcount of the referenced cluster is exactly one. */ +#define QCOW_OFLAG_COPIED (1LL << 63) +/* indicate that the cluster is compressed (they never have the copied flag) */ +#define QCOW_OFLAG_COMPRESSED (1LL << 62) + +#define REFCOUNT_SHIFT 1 /* refcount size is 2 bytes */ + +#ifndef offsetof +#define offsetof(type, field) ((size_t) &((type *)0)->field) +#endif + +typedef struct QCowHeader { + uint32_t magic; + uint32_t version; + uint64_t backing_file_offset; + uint32_t backing_file_size; + uint32_t cluster_bits; + uint64_t size; /* in bytes */ + uint32_t crypt_method; + uint32_t l1_size; /* XXX: save number of clusters instead ? */ + uint64_t l1_table_offset; + uint64_t refcount_table_offset; + uint32_t refcount_table_clusters; + uint32_t nb_snapshots; + uint64_t snapshots_offset; +} QCowHeader; + +typedef struct __attribute__((packed)) QCowSnapshotHeader { + /* header is 8 byte aligned */ + uint64_t l1_table_offset; + + uint32_t l1_size; + uint16_t id_str_size; + uint16_t name_size; + + uint32_t date_sec; + uint32_t date_nsec; + + uint64_t vm_clock_nsec; + + uint32_t vm_state_size; + uint32_t extra_data_size; /* for extension */ + /* extra data follows */ + /* id_str follows */ + /* name follows */ +} QCowSnapshotHeader; + +#define L2_CACHE_SIZE 16 + +typedef struct QCowSnapshot { + uint64_t l1_table_offset; + uint32_t l1_size; + char *id_str; + char *name; + uint32_t vm_state_size; + uint32_t date_sec; + uint32_t date_nsec; + uint64_t vm_clock_nsec; +} QCowSnapshot; + +typedef struct BDRVQcowState { + BlockDriverState *hd; + int cluster_bits; + int cluster_size; + int cluster_sectors; + int l2_bits; + int l2_size; + int l1_size; + int l1_vm_state_index; + int csize_shift; + int csize_mask; + uint64_t cluster_offset_mask; + uint64_t l1_table_offset; + uint64_t *l1_table; + uint64_t *l2_cache; + uint64_t l2_cache_offsets[L2_CACHE_SIZE]; + uint32_t l2_cache_counts[L2_CACHE_SIZE]; + uint8_t *cluster_cache; + uint8_t *cluster_data; + uint64_t cluster_cache_offset; + + uint64_t *refcount_table; + uint64_t refcount_table_offset; + uint32_t refcount_table_size; + uint64_t refcount_block_cache_offset; + uint16_t *refcount_block_cache; + int64_t free_cluster_index; + int64_t free_byte_offset; + + uint32_t crypt_method; /* current crypt method, 0 if no key yet */ + uint32_t crypt_method_header; + AES_KEY aes_encrypt_key; + AES_KEY aes_decrypt_key; + uint64_t snapshots_offset; + int snapshots_size; + int nb_snapshots; + QCowSnapshot *snapshots; +} BDRVQcowState; + +static int decompress_cluster(BDRVQcowState *s, uint64_t cluster_offset); +static int qcow_read(BlockDriverState *bs, int64_t sector_num, + uint8_t *buf, int nb_sectors); +static int qcow_read_snapshots(BlockDriverState *bs); +static void qcow_free_snapshots(BlockDriverState *bs); +static int refcount_init(BlockDriverState *bs); +static void refcount_close(BlockDriverState *bs); +static int get_refcount(BlockDriverState *bs, int64_t cluster_index); +static int update_cluster_refcount(BlockDriverState *bs, + int64_t cluster_index, + int addend); +static void update_refcount(BlockDriverState *bs, + int64_t offset, int64_t length, + int addend); +static int64_t alloc_clusters(BlockDriverState *bs, int64_t size); +static int64_t alloc_bytes(BlockDriverState *bs, int size); +static void free_clusters(BlockDriverState *bs, + int64_t offset, int64_t size); +#ifdef DEBUG_ALLOC +static void check_refcounts(BlockDriverState *bs); +#endif + +static int qcow_probe(const uint8_t *buf, int buf_size, const char *filename) +{ + const QCowHeader *cow_header = (const void *)buf; + + if (buf_size >= sizeof(QCowHeader) && + be32_to_cpu(cow_header->magic) == QCOW_MAGIC && + be32_to_cpu(cow_header->version) == QCOW_VERSION) + return 100; + else + return 0; +} + +static int qcow_open(BlockDriverState *bs, const char *filename, int flags) +{ + BDRVQcowState *s = bs->opaque; + int len, i, shift, ret; + QCowHeader header; + + ret = bdrv_file_open(&s->hd, filename, flags); + if (ret < 0) + return ret; + if (bdrv_pread(s->hd, 0, &header, sizeof(header)) != sizeof(header)) + goto fail; + be32_to_cpus(&header.magic); + be32_to_cpus(&header.version); + be64_to_cpus(&header.backing_file_offset); + be32_to_cpus(&header.backing_file_size); + be64_to_cpus(&header.size); + be32_to_cpus(&header.cluster_bits); + be32_to_cpus(&header.crypt_method); + be64_to_cpus(&header.l1_table_offset); + be32_to_cpus(&header.l1_size); + be64_to_cpus(&header.refcount_table_offset); + be32_to_cpus(&header.refcount_table_clusters); + be64_to_cpus(&header.snapshots_offset); + be32_to_cpus(&header.nb_snapshots); + + if (header.magic != QCOW_MAGIC || header.version != QCOW_VERSION) + goto fail; + if (header.size <= 1 || + header.cluster_bits < 9 || + header.cluster_bits > 16) + goto fail; + if (header.crypt_method > QCOW_CRYPT_AES) + goto fail; + s->crypt_method_header = header.crypt_method; + if (s->crypt_method_header) + bs->encrypted = 1; + s->cluster_bits = header.cluster_bits; + s->cluster_size = 1 << s->cluster_bits; + s->cluster_sectors = 1 << (s->cluster_bits - 9); + s->l2_bits = s->cluster_bits - 3; /* L2 is always one cluster */ + s->l2_size = 1 << s->l2_bits; + bs->total_sectors = header.size / 512; + s->csize_shift = (62 - (s->cluster_bits - 8)); + s->csize_mask = (1 << (s->cluster_bits - 8)) - 1; + s->cluster_offset_mask = (1LL << s->csize_shift) - 1; + s->refcount_table_offset = header.refcount_table_offset; + s->refcount_table_size = + header.refcount_table_clusters << (s->cluster_bits - 3); + + s->snapshots_offset = header.snapshots_offset; + s->nb_snapshots = header.nb_snapshots; + + /* read the level 1 table */ + s->l1_size = header.l1_size; + shift = s->cluster_bits + s->l2_bits; + s->l1_vm_state_index = (header.size + (1LL << shift) - 1) >> shift; + /* the L1 table must contain at least enough entries to put + header.size bytes */ + if (s->l1_size < s->l1_vm_state_index) + goto fail; + s->l1_table_offset = header.l1_table_offset; + s->l1_table = qemu_malloc(s->l1_size * sizeof(uint64_t)); + if (!s->l1_table) + goto fail; + if (bdrv_pread(s->hd, s->l1_table_offset, s->l1_table, s->l1_size * sizeof(uint64_t)) != + s->l1_size * sizeof(uint64_t)) + goto fail; + for(i = 0;i < s->l1_size; i++) { + be64_to_cpus(&s->l1_table[i]); + } + /* alloc L2 cache */ + s->l2_cache = qemu_malloc(s->l2_size * L2_CACHE_SIZE * sizeof(uint64_t)); + if (!s->l2_cache) + goto fail; + s->cluster_cache = qemu_malloc(s->cluster_size); + if (!s->cluster_cache) + goto fail; + /* one more sector for decompressed data alignment */ + s->cluster_data = qemu_malloc(s->cluster_size + 512); + if (!s->cluster_data) + goto fail; + s->cluster_cache_offset = -1; + + if (refcount_init(bs) < 0) + goto fail; + + /* read the backing file name */ + if (header.backing_file_offset != 0) { + len = header.backing_file_size; + if (len > 1023) + len = 1023; + if (bdrv_pread(s->hd, header.backing_file_offset, bs->backing_file, len) != len) + goto fail; + bs->backing_file[len] = '\0'; + } + if (qcow_read_snapshots(bs) < 0) + goto fail; + +#ifdef DEBUG_ALLOC + check_refcounts(bs); +#endif + return 0; + + fail: + qcow_free_snapshots(bs); + refcount_close(bs); + qemu_free(s->l1_table); + qemu_free(s->l2_cache); + qemu_free(s->cluster_cache); + qemu_free(s->cluster_data); + bdrv_delete(s->hd); + return -1; +} + +static int qcow_set_key(BlockDriverState *bs, const char *key) +{ + BDRVQcowState *s = bs->opaque; + uint8_t keybuf[16]; + int len, i; + + memset(keybuf, 0, 16); + len = strlen(key); + if (len > 16) + len = 16; + /* XXX: we could compress the chars to 7 bits to increase + entropy */ + for(i = 0;i < len;i++) { + keybuf[i] = key[i]; + } + s->crypt_method = s->crypt_method_header; + + if (AES_set_encrypt_key(keybuf, 128, &s->aes_encrypt_key) != 0) + return -1; + if (AES_set_decrypt_key(keybuf, 128, &s->aes_decrypt_key) != 0) + return -1; +#if 0 + /* test */ + { + uint8_t in[16]; + uint8_t out[16]; + uint8_t tmp[16]; + for(i=0;i<16;i++) + in[i] = i; + AES_encrypt(in, tmp, &s->aes_encrypt_key); + AES_decrypt(tmp, out, &s->aes_decrypt_key); + for(i = 0; i < 16; i++) + printf(" %02x", tmp[i]); + printf("\n"); + for(i = 0; i < 16; i++) + printf(" %02x", out[i]); + printf("\n"); + } +#endif + return 0; +} + +/* The crypt function is compatible with the linux cryptoloop + algorithm for < 4 GB images. NOTE: out_buf == in_buf is + supported */ +static void encrypt_sectors(BDRVQcowState *s, int64_t sector_num, + uint8_t *out_buf, const uint8_t *in_buf, + int nb_sectors, int enc, + const AES_KEY *key) +{ + union { + uint64_t ll[2]; + uint8_t b[16]; + } ivec; + int i; + + for(i = 0; i < nb_sectors; i++) { + ivec.ll[0] = cpu_to_le64(sector_num); + ivec.ll[1] = 0; + AES_cbc_encrypt(in_buf, out_buf, 512, key, + ivec.b, enc); + sector_num++; + in_buf += 512; + out_buf += 512; + } +} + +static int copy_sectors(BlockDriverState *bs, uint64_t start_sect, + uint64_t cluster_offset, int n_start, int n_end) +{ + BDRVQcowState *s = bs->opaque; + int n, ret; + + n = n_end - n_start; + if (n <= 0) + return 0; + ret = qcow_read(bs, start_sect + n_start, s->cluster_data, n); + if (ret < 0) + return ret; + if (s->crypt_method) { + encrypt_sectors(s, start_sect + n_start, + s->cluster_data, + s->cluster_data, n, 1, + &s->aes_encrypt_key); + } + ret = bdrv_write(s->hd, (cluster_offset >> 9) + n_start, + s->cluster_data, n); + if (ret < 0) + return ret; + return 0; +} + +static void l2_cache_reset(BlockDriverState *bs) +{ + BDRVQcowState *s = bs->opaque; + + memset(s->l2_cache, 0, s->l2_size * L2_CACHE_SIZE * sizeof(uint64_t)); + memset(s->l2_cache_offsets, 0, L2_CACHE_SIZE * sizeof(uint64_t)); + memset(s->l2_cache_counts, 0, L2_CACHE_SIZE * sizeof(uint32_t)); +} + +static inline int l2_cache_new_entry(BlockDriverState *bs) +{ + BDRVQcowState *s = bs->opaque; + uint32_t min_count; + int min_index, i; + + /* find a new entry in the least used one */ + min_index = 0; + min_count = 0xffffffff; + for(i = 0; i < L2_CACHE_SIZE; i++) { + if (s->l2_cache_counts[i] < min_count) { + min_count = s->l2_cache_counts[i]; + min_index = i; + } + } + return min_index; +} + +static int64_t align_offset(int64_t offset, int n) +{ + offset = (offset + n - 1) & ~(n - 1); + return offset; +} + +static int grow_l1_table(BlockDriverState *bs, int min_size) +{ + BDRVQcowState *s = bs->opaque; + int new_l1_size, new_l1_size2, ret, i; + uint64_t *new_l1_table; + uint64_t new_l1_table_offset; + uint64_t data64; + uint32_t data32; + + new_l1_size = s->l1_size; + if (min_size <= new_l1_size) + return 0; + while (min_size > new_l1_size) { + new_l1_size = (new_l1_size * 3 + 1) / 2; + } +#ifdef DEBUG_ALLOC2 + printf("grow l1_table from %d to %d\n", s->l1_size, new_l1_size); +#endif + + new_l1_size2 = sizeof(uint64_t) * new_l1_size; + new_l1_table = qemu_mallocz(new_l1_size2); + if (!new_l1_table) + return -ENOMEM; + memcpy(new_l1_table, s->l1_table, s->l1_size * sizeof(uint64_t)); + + /* write new table (align to cluster) */ + new_l1_table_offset = alloc_clusters(bs, new_l1_size2); + + for(i = 0; i < s->l1_size; i++) + new_l1_table[i] = cpu_to_be64(new_l1_table[i]); + ret = bdrv_pwrite(s->hd, new_l1_table_offset, new_l1_table, new_l1_size2); + if (ret != new_l1_size2) + goto fail; + for(i = 0; i < s->l1_size; i++) + new_l1_table[i] = be64_to_cpu(new_l1_table[i]); + + /* set new table */ + data64 = cpu_to_be64(new_l1_table_offset); + if (bdrv_pwrite(s->hd, offsetof(QCowHeader, l1_table_offset), + &data64, sizeof(data64)) != sizeof(data64)) + goto fail; + data32 = cpu_to_be32(new_l1_size); + if (bdrv_pwrite(s->hd, offsetof(QCowHeader, l1_size), + &data32, sizeof(data32)) != sizeof(data32)) + goto fail; + qemu_free(s->l1_table); + free_clusters(bs, s->l1_table_offset, s->l1_size * sizeof(uint64_t)); + s->l1_table_offset = new_l1_table_offset; + s->l1_table = new_l1_table; + s->l1_size = new_l1_size; + return 0; + fail: + qemu_free(s->l1_table); + return -EIO; +} + +/* 'allocate' is: + * + * 0 not to allocate. + * + * 1 to allocate a normal cluster (for sector indexes 'n_start' to + * 'n_end') + * + * 2 to allocate a compressed cluster of size + * 'compressed_size'. 'compressed_size' must be > 0 and < + * cluster_size + * + * return 0 if not allocated. + */ +static uint64_t get_cluster_offset(BlockDriverState *bs, + uint64_t offset, int allocate, + int compressed_size, + int n_start, int n_end) +{ + BDRVQcowState *s = bs->opaque; + int min_index, i, j, l1_index, l2_index, ret; + uint64_t l2_offset, *l2_table, cluster_offset, tmp, old_l2_offset; + + l1_index = offset >> (s->l2_bits + s->cluster_bits); + if (l1_index >= s->l1_size) { + /* outside l1 table is allowed: we grow the table if needed */ + if (!allocate) + return 0; + if (grow_l1_table(bs, l1_index + 1) < 0) + return 0; + } + l2_offset = s->l1_table[l1_index]; + if (!l2_offset) { + if (!allocate) + return 0; + l2_allocate: + old_l2_offset = l2_offset; + /* allocate a new l2 entry */ + l2_offset = alloc_clusters(bs, s->l2_size * sizeof(uint64_t)); + /* update the L1 entry */ + s->l1_table[l1_index] = l2_offset | QCOW_OFLAG_COPIED; + tmp = cpu_to_be64(l2_offset | QCOW_OFLAG_COPIED); + if (bdrv_pwrite(s->hd, s->l1_table_offset + l1_index * sizeof(tmp), + &tmp, sizeof(tmp)) != sizeof(tmp)) + return 0; + min_index = l2_cache_new_entry(bs); + l2_table = s->l2_cache + (min_index << s->l2_bits); + + if (old_l2_offset == 0) { + memset(l2_table, 0, s->l2_size * sizeof(uint64_t)); + } else { + if (bdrv_pread(s->hd, old_l2_offset, + l2_table, s->l2_size * sizeof(uint64_t)) != + s->l2_size * sizeof(uint64_t)) + return 0; + } + if (bdrv_pwrite(s->hd, l2_offset, + l2_table, s->l2_size * sizeof(uint64_t)) != + s->l2_size * sizeof(uint64_t)) + return 0; + } else { + if (!(l2_offset & QCOW_OFLAG_COPIED)) { + if (allocate) { + free_clusters(bs, l2_offset, s->l2_size * sizeof(uint64_t)); + goto l2_allocate; + } + } else { + l2_offset &= ~QCOW_OFLAG_COPIED; + } + for(i = 0; i < L2_CACHE_SIZE; i++) { + if (l2_offset == s->l2_cache_offsets[i]) { + /* increment the hit count */ + if (++s->l2_cache_counts[i] == 0xffffffff) { + for(j = 0; j < L2_CACHE_SIZE; j++) { + s->l2_cache_counts[j] >>= 1; + } + } + l2_table = s->l2_cache + (i << s->l2_bits); + goto found; + } + } + /* not found: load a new entry in the least used one */ + min_index = l2_cache_new_entry(bs); + l2_table = s->l2_cache + (min_index << s->l2_bits); + if (bdrv_pread(s->hd, l2_offset, l2_table, s->l2_size * sizeof(uint64_t)) != + s->l2_size * sizeof(uint64_t)) + return 0; + } + s->l2_cache_offsets[min_index] = l2_offset; + s->l2_cache_counts[min_index] = 1; + found: + l2_index = (offset >> s->cluster_bits) & (s->l2_size - 1); + cluster_offset = be64_to_cpu(l2_table[l2_index]); + if (!cluster_offset) { + if (!allocate) + return cluster_offset; + } else if (!(cluster_offset & QCOW_OFLAG_COPIED)) { + if (!allocate) + return cluster_offset; + /* free the cluster */ + if (cluster_offset & QCOW_OFLAG_COMPRESSED) { + int nb_csectors; + nb_csectors = ((cluster_offset >> s->csize_shift) & + s->csize_mask) + 1; + free_clusters(bs, (cluster_offset & s->cluster_offset_mask) & ~511, + nb_csectors * 512); + } else { + free_clusters(bs, cluster_offset, s->cluster_size); + } + } else { + cluster_offset &= ~QCOW_OFLAG_COPIED; + return cluster_offset; + } + if (allocate == 1) { + /* allocate a new cluster */ + cluster_offset = alloc_clusters(bs, s->cluster_size); + + /* we must initialize the cluster content which won't be + written */ + if ((n_end - n_start) < s->cluster_sectors) { + uint64_t start_sect; + + start_sect = (offset & ~(s->cluster_size - 1)) >> 9; + ret = copy_sectors(bs, start_sect, + cluster_offset, 0, n_start); + if (ret < 0) + return 0; + ret = copy_sectors(bs, start_sect, + cluster_offset, n_end, s->cluster_sectors); + if (ret < 0) + return 0; + } + tmp = cpu_to_be64(cluster_offset | QCOW_OFLAG_COPIED); + } else { + int nb_csectors; + cluster_offset = alloc_bytes(bs, compressed_size); + nb_csectors = ((cluster_offset + compressed_size - 1) >> 9) - + (cluster_offset >> 9); + cluster_offset |= QCOW_OFLAG_COMPRESSED | + ((uint64_t)nb_csectors << s->csize_shift); + /* compressed clusters never have the copied flag */ + tmp = cpu_to_be64(cluster_offset); + } + /* update L2 table */ + l2_table[l2_index] = tmp; + if (bdrv_pwrite(s->hd, + l2_offset + l2_index * sizeof(tmp), &tmp, sizeof(tmp)) != sizeof(tmp)) + return 0; + return cluster_offset; +} + +static int qcow_is_allocated(BlockDriverState *bs, int64_t sector_num, + int nb_sectors, int *pnum) +{ + BDRVQcowState *s = bs->opaque; + int index_in_cluster, n; + uint64_t cluster_offset; + + cluster_offset = get_cluster_offset(bs, sector_num << 9, 0, 0, 0, 0); + index_in_cluster = sector_num & (s->cluster_sectors - 1); + n = s->cluster_sectors - index_in_cluster; + if (n > nb_sectors) + n = nb_sectors; + *pnum = n; + return (cluster_offset != 0); +} + +static int decompress_buffer(uint8_t *out_buf, int out_buf_size, + const uint8_t *buf, int buf_size) +{ + z_stream strm1, *strm = &strm1; + int ret, out_len; + + memset(strm, 0, sizeof(*strm)); + + strm->next_in = (uint8_t *)buf; + strm->avail_in = buf_size; + strm->next_out = out_buf; + strm->avail_out = out_buf_size; + + ret = inflateInit2(strm, -12); + if (ret != Z_OK) + return -1; + ret = inflate(strm, Z_FINISH); + out_len = strm->next_out - out_buf; + if ((ret != Z_STREAM_END && ret != Z_BUF_ERROR) || + out_len != out_buf_size) { + inflateEnd(strm); + return -1; + } + inflateEnd(strm); + return 0; +} + +static int decompress_cluster(BDRVQcowState *s, uint64_t cluster_offset) +{ + int ret, csize, nb_csectors, sector_offset; + uint64_t coffset; + + coffset = cluster_offset & s->cluster_offset_mask; + if (s->cluster_cache_offset != coffset) { + nb_csectors = ((cluster_offset >> s->csize_shift) & s->csize_mask) + 1; + sector_offset = coffset & 511; + csize = nb_csectors * 512 - sector_offset; + ret = bdrv_read(s->hd, coffset >> 9, s->cluster_data, nb_csectors); + if (ret < 0) { + return -1; + } + if (decompress_buffer(s->cluster_cache, s->cluster_size, + s->cluster_data + sector_offset, csize) < 0) { + return -1; + } + s->cluster_cache_offset = coffset; + } + return 0; +} + +/* handle reading after the end of the backing file */ +static int backing_read1(BlockDriverState *bs, + int64_t sector_num, uint8_t *buf, int nb_sectors) +{ + int n1; + if ((sector_num + nb_sectors) <= bs->total_sectors) + return nb_sectors; + if (sector_num >= bs->total_sectors) + n1 = 0; + else + n1 = bs->total_sectors - sector_num; + memset(buf + n1 * 512, 0, 512 * (nb_sectors - n1)); + return n1; +} + +static int qcow_read(BlockDriverState *bs, int64_t sector_num, + uint8_t *buf, int nb_sectors) +{ + BDRVQcowState *s = bs->opaque; + int ret, index_in_cluster, n, n1; + uint64_t cluster_offset; + + while (nb_sectors > 0) { + cluster_offset = get_cluster_offset(bs, sector_num << 9, 0, 0, 0, 0); + index_in_cluster = sector_num & (s->cluster_sectors - 1); + n = s->cluster_sectors - index_in_cluster; + if (n > nb_sectors) + n = nb_sectors; + if (!cluster_offset) { + if (bs->backing_hd) { + /* read from the base image */ + n1 = backing_read1(bs->backing_hd, sector_num, buf, n); + if (n1 > 0) { + ret = bdrv_read(bs->backing_hd, sector_num, buf, n1); + if (ret < 0) + return -1; + } + } else { + memset(buf, 0, 512 * n); + } + } else if (cluster_offset & QCOW_OFLAG_COMPRESSED) { + if (decompress_cluster(s, cluster_offset) < 0) + return -1; + memcpy(buf, s->cluster_cache + index_in_cluster * 512, 512 * n); + } else { + ret = bdrv_pread(s->hd, cluster_offset + index_in_cluster * 512, buf, n * 512); + if (ret != n * 512) + return -1; + if (s->crypt_method) { + encrypt_sectors(s, sector_num, buf, buf, n, 0, + &s->aes_decrypt_key); + } + } + nb_sectors -= n; + sector_num += n; + buf += n * 512; + } + return 0; +} + +static int qcow_write(BlockDriverState *bs, int64_t sector_num, + const uint8_t *buf, int nb_sectors) +{ + BDRVQcowState *s = bs->opaque; + int ret, index_in_cluster, n; + uint64_t cluster_offset; + + while (nb_sectors > 0) { + index_in_cluster = sector_num & (s->cluster_sectors - 1); + n = s->cluster_sectors - index_in_cluster; + if (n > nb_sectors) + n = nb_sectors; + cluster_offset = get_cluster_offset(bs, sector_num << 9, 1, 0, + index_in_cluster, + index_in_cluster + n); + if (!cluster_offset) + return -1; + if (s->crypt_method) { + encrypt_sectors(s, sector_num, s->cluster_data, buf, n, 1, + &s->aes_encrypt_key); + ret = bdrv_pwrite(s->hd, cluster_offset + index_in_cluster * 512, + s->cluster_data, n * 512); + } else { + ret = bdrv_pwrite(s->hd, cluster_offset + index_in_cluster * 512, buf, n * 512); + } + if (ret != n * 512) + return -1; + nb_sectors -= n; + sector_num += n; + buf += n * 512; + } + s->cluster_cache_offset = -1; /* disable compressed cache */ + return 0; +} + +typedef struct QCowAIOCB { + BlockDriverAIOCB common; + int64_t sector_num; + uint8_t *buf; + int nb_sectors; + int n; + uint64_t cluster_offset; + uint8_t *cluster_data; + BlockDriverAIOCB *hd_aiocb; +} QCowAIOCB; + +static void qcow_aio_read_cb(void *opaque, int ret) +{ + QCowAIOCB *acb = opaque; + BlockDriverState *bs = acb->common.bs; + BDRVQcowState *s = bs->opaque; + int index_in_cluster, n1; + + acb->hd_aiocb = NULL; + if (ret < 0) { + fail: + acb->common.cb(acb->common.opaque, ret); + qemu_aio_release(acb); + return; + } + + redo: + /* post process the read buffer */ + if (!acb->cluster_offset) { + /* nothing to do */ + } else if (acb->cluster_offset & QCOW_OFLAG_COMPRESSED) { + /* nothing to do */ + } else { + if (s->crypt_method) { + encrypt_sectors(s, acb->sector_num, acb->buf, acb->buf, + acb->n, 0, + &s->aes_decrypt_key); + } + } + + acb->nb_sectors -= acb->n; + acb->sector_num += acb->n; + acb->buf += acb->n * 512; + + if (acb->nb_sectors == 0) { + /* request completed */ + acb->common.cb(acb->common.opaque, 0); + qemu_aio_release(acb); + return; + } + + /* prepare next AIO request */ + acb->cluster_offset = get_cluster_offset(bs, acb->sector_num << 9, + 0, 0, 0, 0); + index_in_cluster = acb->sector_num & (s->cluster_sectors - 1); + acb->n = s->cluster_sectors - index_in_cluster; + if (acb->n > acb->nb_sectors) + acb->n = acb->nb_sectors; + + if (!acb->cluster_offset) { + if (bs->backing_hd) { + /* read from the base image */ + n1 = backing_read1(bs->backing_hd, acb->sector_num, + acb->buf, acb->n); + if (n1 > 0) { + acb->hd_aiocb = bdrv_aio_read(bs->backing_hd, acb->sector_num, + acb->buf, acb->n, qcow_aio_read_cb, acb); + if (acb->hd_aiocb == NULL) + goto fail; + } else { + goto redo; + } + } else { + /* Note: in this case, no need to wait */ + memset(acb->buf, 0, 512 * acb->n); + goto redo; + } + } else if (acb->cluster_offset & QCOW_OFLAG_COMPRESSED) { + /* add AIO support for compressed blocks ? */ + if (decompress_cluster(s, acb->cluster_offset) < 0) + goto fail; + memcpy(acb->buf, + s->cluster_cache + index_in_cluster * 512, 512 * acb->n); + goto redo; + } else { + if ((acb->cluster_offset & 511) != 0) { + ret = -EIO; + goto fail; + } + acb->hd_aiocb = bdrv_aio_read(s->hd, + (acb->cluster_offset >> 9) + index_in_cluster, + acb->buf, acb->n, qcow_aio_read_cb, acb); + if (acb->hd_aiocb == NULL) + goto fail; + } +} + +static QCowAIOCB *qcow_aio_setup(BlockDriverState *bs, + int64_t sector_num, uint8_t *buf, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque) +{ + QCowAIOCB *acb; + + acb = qemu_aio_get(bs, cb, opaque); + if (!acb) + return NULL; + acb->hd_aiocb = NULL; + acb->sector_num = sector_num; + acb->buf = buf; + acb->nb_sectors = nb_sectors; + acb->n = 0; + acb->cluster_offset = 0; + return acb; +} + +static BlockDriverAIOCB *qcow_aio_read(BlockDriverState *bs, + int64_t sector_num, uint8_t *buf, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque) +{ + QCowAIOCB *acb; + + acb = qcow_aio_setup(bs, sector_num, buf, nb_sectors, cb, opaque); + if (!acb) + return NULL; + + qcow_aio_read_cb(acb, 0); + return &acb->common; +} + +static void qcow_aio_write_cb(void *opaque, int ret) +{ + QCowAIOCB *acb = opaque; + BlockDriverState *bs = acb->common.bs; + BDRVQcowState *s = bs->opaque; + int index_in_cluster; + uint64_t cluster_offset; + const uint8_t *src_buf; + + acb->hd_aiocb = NULL; + + if (ret < 0) { + fail: + acb->common.cb(acb->common.opaque, ret); + qemu_aio_release(acb); + return; + } + + acb->nb_sectors -= acb->n; + acb->sector_num += acb->n; + acb->buf += acb->n * 512; + + if (acb->nb_sectors == 0) { + /* request completed */ + acb->common.cb(acb->common.opaque, 0); + qemu_aio_release(acb); + return; + } + + index_in_cluster = acb->sector_num & (s->cluster_sectors - 1); + acb->n = s->cluster_sectors - index_in_cluster; + if (acb->n > acb->nb_sectors) + acb->n = acb->nb_sectors; + cluster_offset = get_cluster_offset(bs, acb->sector_num << 9, 1, 0, + index_in_cluster, + index_in_cluster + acb->n); + if (!cluster_offset || (cluster_offset & 511) != 0) { + ret = -EIO; + goto fail; + } + if (s->crypt_method) { + if (!acb->cluster_data) { + acb->cluster_data = qemu_mallocz(s->cluster_size); + if (!acb->cluster_data) { + ret = -ENOMEM; + goto fail; + } + } + encrypt_sectors(s, acb->sector_num, acb->cluster_data, acb->buf, + acb->n, 1, &s->aes_encrypt_key); + src_buf = acb->cluster_data; + } else { + src_buf = acb->buf; + } + acb->hd_aiocb = bdrv_aio_write(s->hd, + (cluster_offset >> 9) + index_in_cluster, + src_buf, acb->n, + qcow_aio_write_cb, acb); + if (acb->hd_aiocb == NULL) + goto fail; +} + +static BlockDriverAIOCB *qcow_aio_write(BlockDriverState *bs, + int64_t sector_num, const uint8_t *buf, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque) +{ + BDRVQcowState *s = bs->opaque; + QCowAIOCB *acb; + + s->cluster_cache_offset = -1; /* disable compressed cache */ + + acb = qcow_aio_setup(bs, sector_num, (uint8_t*)buf, nb_sectors, cb, opaque); + if (!acb) + return NULL; + + qcow_aio_write_cb(acb, 0); + return &acb->common; +} + +static void qcow_aio_cancel(BlockDriverAIOCB *blockacb) +{ + QCowAIOCB *acb = (QCowAIOCB *)blockacb; + if (acb->hd_aiocb) + bdrv_aio_cancel(acb->hd_aiocb); + qemu_aio_release(acb); +} + +static void qcow_close(BlockDriverState *bs) +{ + BDRVQcowState *s = bs->opaque; + qemu_free(s->l1_table); + qemu_free(s->l2_cache); + qemu_free(s->cluster_cache); + qemu_free(s->cluster_data); + refcount_close(bs); + bdrv_delete(s->hd); +} + +/* XXX: use std qcow open function ? */ +typedef struct QCowCreateState { + int cluster_size; + int cluster_bits; + uint16_t *refcount_block; + uint64_t *refcount_table; + int64_t l1_table_offset; + int64_t refcount_table_offset; + int64_t refcount_block_offset; +} QCowCreateState; + +static void create_refcount_update(QCowCreateState *s, + int64_t offset, int64_t size) +{ + int refcount; + int64_t start, last, cluster_offset; + uint16_t *p; + + start = offset & ~(s->cluster_size - 1); + last = (offset + size - 1) & ~(s->cluster_size - 1); + for(cluster_offset = start; cluster_offset <= last; + cluster_offset += s->cluster_size) { + p = &s->refcount_block[cluster_offset >> s->cluster_bits]; + refcount = be16_to_cpu(*p); + refcount++; + *p = cpu_to_be16(refcount); + } +} + +static int qcow_create(const char *filename, int64_t total_size, + const char *backing_file, int flags) +{ + int fd, header_size, backing_filename_len, l1_size, i, shift, l2_bits; + QCowHeader header; + uint64_t tmp, offset; + QCowCreateState s1, *s = &s1; + + memset(s, 0, sizeof(*s)); + + fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0644); + if (fd < 0) + return -1; + memset(&header, 0, sizeof(header)); + header.magic = cpu_to_be32(QCOW_MAGIC); + header.version = cpu_to_be32(QCOW_VERSION); + header.size = cpu_to_be64(total_size * 512); + header_size = sizeof(header); + backing_filename_len = 0; + if (backing_file) { + header.backing_file_offset = cpu_to_be64(header_size); + backing_filename_len = strlen(backing_file); + header.backing_file_size = cpu_to_be32(backing_filename_len); + header_size += backing_filename_len; + } + s->cluster_bits = 12; /* 4 KB clusters */ + s->cluster_size = 1 << s->cluster_bits; + header.cluster_bits = cpu_to_be32(s->cluster_bits); + header_size = (header_size + 7) & ~7; + if (flags & BLOCK_FLAG_ENCRYPT) { + header.crypt_method = cpu_to_be32(QCOW_CRYPT_AES); + } else { + header.crypt_method = cpu_to_be32(QCOW_CRYPT_NONE); + } + l2_bits = s->cluster_bits - 3; + shift = s->cluster_bits + l2_bits; + l1_size = (((total_size * 512) + (1LL << shift) - 1) >> shift); + offset = align_offset(header_size, s->cluster_size); + s->l1_table_offset = offset; + header.l1_table_offset = cpu_to_be64(s->l1_table_offset); + header.l1_size = cpu_to_be32(l1_size); + offset += align_offset(l1_size * sizeof(uint64_t), s->cluster_size); + + s->refcount_table = qemu_mallocz(s->cluster_size); + if (!s->refcount_table) + goto fail; + s->refcount_block = qemu_mallocz(s->cluster_size); + if (!s->refcount_block) + goto fail; + + s->refcount_table_offset = offset; + header.refcount_table_offset = cpu_to_be64(offset); + header.refcount_table_clusters = cpu_to_be32(1); + offset += s->cluster_size; + + s->refcount_table[0] = cpu_to_be64(offset); + s->refcount_block_offset = offset; + offset += s->cluster_size; + + /* update refcounts */ + create_refcount_update(s, 0, header_size); + create_refcount_update(s, s->l1_table_offset, l1_size * sizeof(uint64_t)); + create_refcount_update(s, s->refcount_table_offset, s->cluster_size); + create_refcount_update(s, s->refcount_block_offset, s->cluster_size); + + /* write all the data */ + if(write(fd, &header, sizeof(header))!=sizeof(header)) return -1; + if (backing_file) { + if(write(fd, backing_file, backing_filename_len)!=backing_filename_len) return -1; + } + lseek(fd, s->l1_table_offset, SEEK_SET); + tmp = 0; + for(i = 0;i < l1_size; i++) { + if(write(fd, &tmp, sizeof(tmp))!=sizeof(tmp)) return -1; + } + lseek(fd, s->refcount_table_offset, SEEK_SET); + if(write(fd, s->refcount_table, s->cluster_size)!=s->cluster_size) return -1; + + lseek(fd, s->refcount_block_offset, SEEK_SET); + if(write(fd, s->refcount_block, s->cluster_size)!=s->cluster_size) return -1; + + qemu_free(s->refcount_table); + qemu_free(s->refcount_block); + close(fd); + return 0; + fail: + qemu_free(s->refcount_table); + qemu_free(s->refcount_block); + close(fd); + return -ENOMEM; +} + +static int qcow_make_empty(BlockDriverState *bs) +{ +#if 0 + /* XXX: not correct */ + BDRVQcowState *s = bs->opaque; + uint32_t l1_length = s->l1_size * sizeof(uint64_t); + int ret; + + memset(s->l1_table, 0, l1_length); + if (bdrv_pwrite(s->hd, s->l1_table_offset, s->l1_table, l1_length) < 0) + return -1; + ret = bdrv_truncate(s->hd, s->l1_table_offset + l1_length); + if (ret < 0) + return ret; + + l2_cache_reset(bs); +#endif + return 0; +} + +/* XXX: put compressed sectors first, then all the cluster aligned + tables to avoid losing bytes in alignment */ +static int qcow_write_compressed(BlockDriverState *bs, int64_t sector_num, + const uint8_t *buf, int nb_sectors) +{ + BDRVQcowState *s = bs->opaque; + z_stream strm; + int ret, out_len; + uint8_t *out_buf; + uint64_t cluster_offset; + + if (nb_sectors == 0) { + /* align end of file to a sector boundary to ease reading with + sector based I/Os */ + cluster_offset = bdrv_getlength(s->hd); + cluster_offset = (cluster_offset + 511) & ~511; + bdrv_truncate(s->hd, cluster_offset); + return 0; + } + + if (nb_sectors != s->cluster_sectors) + return -EINVAL; + + out_buf = qemu_malloc(s->cluster_size + (s->cluster_size / 1000) + 128); + if (!out_buf) + return -ENOMEM; + + /* best compression, small window, no zlib header */ + memset(&strm, 0, sizeof(strm)); + ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, + Z_DEFLATED, -12, + 9, Z_DEFAULT_STRATEGY); + if (ret != 0) { + qemu_free(out_buf); + return -1; + } + + strm.avail_in = s->cluster_size; + strm.next_in = (uint8_t *)buf; + strm.avail_out = s->cluster_size; + strm.next_out = out_buf; + + ret = deflate(&strm, Z_FINISH); + if (ret != Z_STREAM_END && ret != Z_OK) { + qemu_free(out_buf); + deflateEnd(&strm); + return -1; + } + out_len = strm.next_out - out_buf; + + deflateEnd(&strm); + + if (ret != Z_STREAM_END || out_len >= s->cluster_size) { + /* could not compress: write normal cluster */ + qcow_write(bs, sector_num, buf, s->cluster_sectors); + } else { + cluster_offset = get_cluster_offset(bs, sector_num << 9, 2, + out_len, 0, 0); + cluster_offset &= s->cluster_offset_mask; + if (bdrv_pwrite(s->hd, cluster_offset, out_buf, out_len) != out_len) { + qemu_free(out_buf); + return -1; + } + } + + qemu_free(out_buf); + return 0; +} + +static void qcow_flush(BlockDriverState *bs) +{ + BDRVQcowState *s = bs->opaque; + bdrv_flush(s->hd); +} + +static int qcow_get_info(BlockDriverState *bs, BlockDriverInfo *bdi) +{ + BDRVQcowState *s = bs->opaque; + bdi->cluster_size = s->cluster_size; + bdi->vm_state_offset = (int64_t)s->l1_vm_state_index << + (s->cluster_bits + s->l2_bits); + return 0; +} + +/*********************************************************/ +/* snapshot support */ + +/* update the refcounts of snapshots and the copied flag */ +static int update_snapshot_refcount(BlockDriverState *bs, + int64_t l1_table_offset, + int l1_size, + int addend) +{ + BDRVQcowState *s = bs->opaque; + uint64_t *l1_table, *l2_table, l2_offset, offset, l1_size2, l1_allocated; + int64_t old_offset, old_l2_offset; + int l2_size, i, j, l1_modified, l2_modified, nb_csectors, refcount; + + l2_cache_reset(bs); + + l2_table = NULL; + l1_table = NULL; + l1_size2 = l1_size * sizeof(uint64_t); + l1_allocated = 0; + if (l1_table_offset != s->l1_table_offset) { + l1_table = qemu_malloc(l1_size2); + if (!l1_table) + goto fail; + l1_allocated = 1; + if (bdrv_pread(s->hd, l1_table_offset, + l1_table, l1_size2) != l1_size2) + goto fail; + for(i = 0;i < l1_size; i++) + be64_to_cpus(&l1_table[i]); + } else { + assert(l1_size == s->l1_size); + l1_table = s->l1_table; + l1_allocated = 0; + } + + l2_size = s->l2_size * sizeof(uint64_t); + l2_table = qemu_malloc(l2_size); + if (!l2_table) + goto fail; + l1_modified = 0; + for(i = 0; i < l1_size; i++) { + l2_offset = l1_table[i]; + if (l2_offset) { + old_l2_offset = l2_offset; + l2_offset &= ~QCOW_OFLAG_COPIED; + l2_modified = 0; + if (bdrv_pread(s->hd, l2_offset, l2_table, l2_size) != l2_size) + goto fail; + for(j = 0; j < s->l2_size; j++) { + offset = be64_to_cpu(l2_table[j]); + if (offset != 0) { + old_offset = offset; + offset &= ~QCOW_OFLAG_COPIED; + if (offset & QCOW_OFLAG_COMPRESSED) { + nb_csectors = ((offset >> s->csize_shift) & + s->csize_mask) + 1; + if (addend != 0) + update_refcount(bs, (offset & s->cluster_offset_mask) & ~511, + nb_csectors * 512, addend); + /* compressed clusters are never modified */ + refcount = 2; + } else { + if (addend != 0) { + refcount = update_cluster_refcount(bs, offset >> s->cluster_bits, addend); + } else { + refcount = get_refcount(bs, offset >> s->cluster_bits); + } + } + + if (refcount == 1) { + offset |= QCOW_OFLAG_COPIED; + } + if (offset != old_offset) { + l2_table[j] = cpu_to_be64(offset); + l2_modified = 1; + } + } + } + if (l2_modified) { + if (bdrv_pwrite(s->hd, + l2_offset, l2_table, l2_size) != l2_size) + goto fail; + } + + if (addend != 0) { + refcount = update_cluster_refcount(bs, l2_offset >> s->cluster_bits, addend); + } else { + refcount = get_refcount(bs, l2_offset >> s->cluster_bits); + } + if (refcount == 1) { + l2_offset |= QCOW_OFLAG_COPIED; + } + if (l2_offset != old_l2_offset) { + l1_table[i] = l2_offset; + l1_modified = 1; + } + } + } + if (l1_modified) { + for(i = 0; i < l1_size; i++) + cpu_to_be64s(&l1_table[i]); + if (bdrv_pwrite(s->hd, l1_table_offset, l1_table, + l1_size2) != l1_size2) + goto fail; + for(i = 0; i < l1_size; i++) + be64_to_cpus(&l1_table[i]); + } + if (l1_allocated) + qemu_free(l1_table); + qemu_free(l2_table); + return 0; + fail: + if (l1_allocated) + qemu_free(l1_table); + qemu_free(l2_table); + return -EIO; +} + +static void qcow_free_snapshots(BlockDriverState *bs) +{ + BDRVQcowState *s = bs->opaque; + int i; + + for(i = 0; i < s->nb_snapshots; i++) { + qemu_free(s->snapshots[i].name); + qemu_free(s->snapshots[i].id_str); + } + qemu_free(s->snapshots); + s->snapshots = NULL; + s->nb_snapshots = 0; +} + +static int qcow_read_snapshots(BlockDriverState *bs) +{ + BDRVQcowState *s = bs->opaque; + QCowSnapshotHeader h; + QCowSnapshot *sn; + int i, id_str_size, name_size; + int64_t offset; + uint32_t extra_data_size; + + offset = s->snapshots_offset; + s->snapshots = qemu_mallocz(s->nb_snapshots * sizeof(QCowSnapshot)); + if (!s->snapshots) + goto fail; + for(i = 0; i < s->nb_snapshots; i++) { + offset = align_offset(offset, 8); + if (bdrv_pread(s->hd, offset, &h, sizeof(h)) != sizeof(h)) + goto fail; + offset += sizeof(h); + sn = s->snapshots + i; + sn->l1_table_offset = be64_to_cpu(h.l1_table_offset); + sn->l1_size = be32_to_cpu(h.l1_size); + sn->vm_state_size = be32_to_cpu(h.vm_state_size); + sn->date_sec = be32_to_cpu(h.date_sec); + sn->date_nsec = be32_to_cpu(h.date_nsec); + sn->vm_clock_nsec = be64_to_cpu(h.vm_clock_nsec); + extra_data_size = be32_to_cpu(h.extra_data_size); + + id_str_size = be16_to_cpu(h.id_str_size); + name_size = be16_to_cpu(h.name_size); + + offset += extra_data_size; + + sn->id_str = qemu_malloc(id_str_size + 1); + if (!sn->id_str) + goto fail; + if (bdrv_pread(s->hd, offset, sn->id_str, id_str_size) != id_str_size) + goto fail; + offset += id_str_size; + sn->id_str[id_str_size] = '\0'; + + sn->name = qemu_malloc(name_size + 1); + if (!sn->name) + goto fail; + if (bdrv_pread(s->hd, offset, sn->name, name_size) != name_size) + goto fail; + offset += name_size; + sn->name[name_size] = '\0'; + } + s->snapshots_size = offset - s->snapshots_offset; + return 0; + fail: + qcow_free_snapshots(bs); + return -1; +} + +/* add at the end of the file a new list of snapshots */ +static int qcow_write_snapshots(BlockDriverState *bs) +{ + BDRVQcowState *s = bs->opaque; + QCowSnapshot *sn; + QCowSnapshotHeader h; + int i, name_size, id_str_size, snapshots_size; + uint64_t data64; + uint32_t data32; + int64_t offset, snapshots_offset; + + /* compute the size of the snapshots */ + offset = 0; + for(i = 0; i < s->nb_snapshots; i++) { + sn = s->snapshots + i; + offset = align_offset(offset, 8); + offset += sizeof(h); + offset += strlen(sn->id_str); + offset += strlen(sn->name); + } + snapshots_size = offset; + + snapshots_offset = alloc_clusters(bs, snapshots_size); + offset = snapshots_offset; + + for(i = 0; i < s->nb_snapshots; i++) { + sn = s->snapshots + i; + memset(&h, 0, sizeof(h)); + h.l1_table_offset = cpu_to_be64(sn->l1_table_offset); + h.l1_size = cpu_to_be32(sn->l1_size); + h.vm_state_size = cpu_to_be32(sn->vm_state_size); + h.date_sec = cpu_to_be32(sn->date_sec); + h.date_nsec = cpu_to_be32(sn->date_nsec); + h.vm_clock_nsec = cpu_to_be64(sn->vm_clock_nsec); + + id_str_size = strlen(sn->id_str); + name_size = strlen(sn->name); + h.id_str_size = cpu_to_be16(id_str_size); + h.name_size = cpu_to_be16(name_size); + offset = align_offset(offset, 8); + if (bdrv_pwrite(s->hd, offset, &h, sizeof(h)) != sizeof(h)) + goto fail; + offset += sizeof(h); + if (bdrv_pwrite(s->hd, offset, sn->id_str, id_str_size) != id_str_size) + goto fail; + offset += id_str_size; + if (bdrv_pwrite(s->hd, offset, sn->name, name_size) != name_size) + goto fail; + offset += name_size; + } + + /* update the various header fields */ + data64 = cpu_to_be64(snapshots_offset); + if (bdrv_pwrite(s->hd, offsetof(QCowHeader, snapshots_offset), + &data64, sizeof(data64)) != sizeof(data64)) + goto fail; + data32 = cpu_to_be32(s->nb_snapshots); + if (bdrv_pwrite(s->hd, offsetof(QCowHeader, nb_snapshots), + &data32, sizeof(data32)) != sizeof(data32)) + goto fail; + + /* free the old snapshot table */ + free_clusters(bs, s->snapshots_offset, s->snapshots_size); + s->snapshots_offset = snapshots_offset; + s->snapshots_size = snapshots_size; + return 0; + fail: + return -1; +} + +static void find_new_snapshot_id(BlockDriverState *bs, + char *id_str, int id_str_size) +{ + BDRVQcowState *s = bs->opaque; + QCowSnapshot *sn; + int i, id, id_max = 0; + + for(i = 0; i < s->nb_snapshots; i++) { + sn = s->snapshots + i; + id = strtoul(sn->id_str, NULL, 10); + if (id > id_max) + id_max = id; + } + snprintf(id_str, id_str_size, "%d", id_max + 1); +} + +static int find_snapshot_by_id(BlockDriverState *bs, const char *id_str) +{ + BDRVQcowState *s = bs->opaque; + int i; + + for(i = 0; i < s->nb_snapshots; i++) { + if (!strcmp(s->snapshots[i].id_str, id_str)) + return i; + } + return -1; +} + +static int find_snapshot_by_id_or_name(BlockDriverState *bs, const char *name) +{ + BDRVQcowState *s = bs->opaque; + int i, ret; + + ret = find_snapshot_by_id(bs, name); + if (ret >= 0) + return ret; + for(i = 0; i < s->nb_snapshots; i++) { + if (!strcmp(s->snapshots[i].name, name)) + return i; + } + return -1; +} + +/* if no id is provided, a new one is constructed */ +static int qcow_snapshot_create(BlockDriverState *bs, + QEMUSnapshotInfo *sn_info) +{ + BDRVQcowState *s = bs->opaque; + QCowSnapshot *snapshots1, sn1, *sn = &sn1; + int i, ret; + uint64_t *l1_table = NULL; + + memset(sn, 0, sizeof(*sn)); + + if (sn_info->id_str[0] == '\0') { + /* compute a new id */ + find_new_snapshot_id(bs, sn_info->id_str, sizeof(sn_info->id_str)); + } + + /* check that the ID is unique */ + if (find_snapshot_by_id(bs, sn_info->id_str) >= 0) + return -ENOENT; + + sn->id_str = qemu_strdup(sn_info->id_str); + if (!sn->id_str) + goto fail; + sn->name = qemu_strdup(sn_info->name); + if (!sn->name) + goto fail; + sn->vm_state_size = sn_info->vm_state_size; + sn->date_sec = sn_info->date_sec; + sn->date_nsec = sn_info->date_nsec; + sn->vm_clock_nsec = sn_info->vm_clock_nsec; + + ret = update_snapshot_refcount(bs, s->l1_table_offset, s->l1_size, 1); + if (ret < 0) + goto fail; + + /* create the L1 table of the snapshot */ + sn->l1_table_offset = alloc_clusters(bs, s->l1_size * sizeof(uint64_t)); + sn->l1_size = s->l1_size; + + l1_table = qemu_malloc(s->l1_size * sizeof(uint64_t)); + if (!l1_table) + goto fail; + for(i = 0; i < s->l1_size; i++) { + l1_table[i] = cpu_to_be64(s->l1_table[i]); + } + if (bdrv_pwrite(s->hd, sn->l1_table_offset, + l1_table, s->l1_size * sizeof(uint64_t)) != + (s->l1_size * sizeof(uint64_t))) + goto fail; + qemu_free(l1_table); + l1_table = NULL; + + snapshots1 = qemu_malloc((s->nb_snapshots + 1) * sizeof(QCowSnapshot)); + if (!snapshots1) + goto fail; + memcpy(snapshots1, s->snapshots, s->nb_snapshots * sizeof(QCowSnapshot)); + s->snapshots = snapshots1; + s->snapshots[s->nb_snapshots++] = *sn; + + if (qcow_write_snapshots(bs) < 0) + goto fail; +#ifdef DEBUG_ALLOC + check_refcounts(bs); +#endif + return 0; + fail: + qemu_free(sn->name); + qemu_free(l1_table); + return -1; +} + +/* copy the snapshot 'snapshot_name' into the current disk image */ +static int qcow_snapshot_goto(BlockDriverState *bs, + const char *snapshot_id) +{ + BDRVQcowState *s = bs->opaque; + QCowSnapshot *sn; + int i, snapshot_index, l1_size2; + + snapshot_index = find_snapshot_by_id_or_name(bs, snapshot_id); + if (snapshot_index < 0) + return -ENOENT; + sn = &s->snapshots[snapshot_index]; + + if (update_snapshot_refcount(bs, s->l1_table_offset, s->l1_size, -1) < 0) + goto fail; + + if (grow_l1_table(bs, sn->l1_size) < 0) + goto fail; + + s->l1_size = sn->l1_size; + l1_size2 = s->l1_size * sizeof(uint64_t); + /* copy the snapshot l1 table to the current l1 table */ + if (bdrv_pread(s->hd, sn->l1_table_offset, + s->l1_table, l1_size2) != l1_size2) + goto fail; + if (bdrv_pwrite(s->hd, s->l1_table_offset, + s->l1_table, l1_size2) != l1_size2) + goto fail; + for(i = 0;i < s->l1_size; i++) { + be64_to_cpus(&s->l1_table[i]); + } + + if (update_snapshot_refcount(bs, s->l1_table_offset, s->l1_size, 1) < 0) + goto fail; + +#ifdef DEBUG_ALLOC + check_refcounts(bs); +#endif + return 0; + fail: + return -EIO; +} + +static int qcow_snapshot_delete(BlockDriverState *bs, const char *snapshot_id) +{ + BDRVQcowState *s = bs->opaque; + QCowSnapshot *sn; + int snapshot_index, ret; + + snapshot_index = find_snapshot_by_id_or_name(bs, snapshot_id); + if (snapshot_index < 0) + return -ENOENT; + sn = &s->snapshots[snapshot_index]; + + ret = update_snapshot_refcount(bs, sn->l1_table_offset, sn->l1_size, -1); + if (ret < 0) + return ret; + /* must update the copied flag on the current cluster offsets */ + ret = update_snapshot_refcount(bs, s->l1_table_offset, s->l1_size, 0); + if (ret < 0) + return ret; + free_clusters(bs, sn->l1_table_offset, sn->l1_size * sizeof(uint64_t)); + + qemu_free(sn->id_str); + qemu_free(sn->name); + memmove(sn, sn + 1, (s->nb_snapshots - snapshot_index - 1) * sizeof(*sn)); + s->nb_snapshots--; + ret = qcow_write_snapshots(bs); + if (ret < 0) { + /* XXX: restore snapshot if error ? */ + return ret; + } +#ifdef DEBUG_ALLOC + check_refcounts(bs); +#endif + return 0; +} + +static int qcow_snapshot_list(BlockDriverState *bs, + QEMUSnapshotInfo **psn_tab) +{ + BDRVQcowState *s = bs->opaque; + QEMUSnapshotInfo *sn_tab, *sn_info; + QCowSnapshot *sn; + int i; + + sn_tab = qemu_mallocz(s->nb_snapshots * sizeof(QEMUSnapshotInfo)); + if (!sn_tab) + goto fail; + for(i = 0; i < s->nb_snapshots; i++) { + sn_info = sn_tab + i; + sn = s->snapshots + i; + pstrcpy(sn_info->id_str, sizeof(sn_info->id_str), + sn->id_str); + pstrcpy(sn_info->name, sizeof(sn_info->name), + sn->name); + sn_info->vm_state_size = sn->vm_state_size; + sn_info->date_sec = sn->date_sec; + sn_info->date_nsec = sn->date_nsec; + sn_info->vm_clock_nsec = sn->vm_clock_nsec; + } + *psn_tab = sn_tab; + return s->nb_snapshots; + fail: + qemu_free(sn_tab); + *psn_tab = NULL; + return -ENOMEM; +} + +/*********************************************************/ +/* refcount handling */ + +static int refcount_init(BlockDriverState *bs) +{ + BDRVQcowState *s = bs->opaque; + int ret, refcount_table_size2, i; + + s->refcount_block_cache = qemu_malloc(s->cluster_size); + if (!s->refcount_block_cache) + goto fail; + refcount_table_size2 = s->refcount_table_size * sizeof(uint64_t); + s->refcount_table = qemu_malloc(refcount_table_size2); + if (!s->refcount_table) + goto fail; + if (s->refcount_table_size > 0) { + ret = bdrv_pread(s->hd, s->refcount_table_offset, + s->refcount_table, refcount_table_size2); + if (ret != refcount_table_size2) + goto fail; + for(i = 0; i < s->refcount_table_size; i++) + be64_to_cpus(&s->refcount_table[i]); + } + return 0; + fail: + return -ENOMEM; +} + +static void refcount_close(BlockDriverState *bs) +{ + BDRVQcowState *s = bs->opaque; + qemu_free(s->refcount_block_cache); + qemu_free(s->refcount_table); +} + + +static int load_refcount_block(BlockDriverState *bs, + int64_t refcount_block_offset) +{ + BDRVQcowState *s = bs->opaque; + int ret; + ret = bdrv_pread(s->hd, refcount_block_offset, s->refcount_block_cache, + s->cluster_size); + if (ret != s->cluster_size) + return -EIO; + s->refcount_block_cache_offset = refcount_block_offset; + return 0; +} + +static int get_refcount(BlockDriverState *bs, int64_t cluster_index) +{ + BDRVQcowState *s = bs->opaque; + int refcount_table_index, block_index; + int64_t refcount_block_offset; + + refcount_table_index = cluster_index >> (s->cluster_bits - REFCOUNT_SHIFT); + if (refcount_table_index >= s->refcount_table_size) + return 0; + refcount_block_offset = s->refcount_table[refcount_table_index]; + if (!refcount_block_offset) + return 0; + if (refcount_block_offset != s->refcount_block_cache_offset) { + /* better than nothing: return allocated if read error */ + if (load_refcount_block(bs, refcount_block_offset) < 0) + return 1; + } + block_index = cluster_index & + ((1 << (s->cluster_bits - REFCOUNT_SHIFT)) - 1); + return be16_to_cpu(s->refcount_block_cache[block_index]); +} + +/* return < 0 if error */ +static int64_t alloc_clusters_noref(BlockDriverState *bs, int64_t size) +{ + BDRVQcowState *s = bs->opaque; + int i, nb_clusters; + + nb_clusters = (size + s->cluster_size - 1) >> s->cluster_bits; + for(;;) { + if (get_refcount(bs, s->free_cluster_index) == 0) { + s->free_cluster_index++; + for(i = 1; i < nb_clusters; i++) { + if (get_refcount(bs, s->free_cluster_index) != 0) + goto not_found; + s->free_cluster_index++; + } +#ifdef DEBUG_ALLOC2 + printf("alloc_clusters: size=%lld -> %lld\n", + size, + (s->free_cluster_index - nb_clusters) << s->cluster_bits); +#endif + return (s->free_cluster_index - nb_clusters) << s->cluster_bits; + } else { + not_found: + s->free_cluster_index++; + } + } +} + +static int64_t alloc_clusters(BlockDriverState *bs, int64_t size) +{ + int64_t offset; + + offset = alloc_clusters_noref(bs, size); + update_refcount(bs, offset, size, 1); + return offset; +} + +/* only used to allocate compressed sectors. We try to allocate + contiguous sectors. size must be <= cluster_size */ +static int64_t alloc_bytes(BlockDriverState *bs, int size) +{ + BDRVQcowState *s = bs->opaque; + int64_t offset, cluster_offset; + int free_in_cluster; + + assert(size > 0 && size <= s->cluster_size); + if (s->free_byte_offset == 0) { + s->free_byte_offset = alloc_clusters(bs, s->cluster_size); + } + redo: + free_in_cluster = s->cluster_size - + (s->free_byte_offset & (s->cluster_size - 1)); + if (size <= free_in_cluster) { + /* enough space in current cluster */ + offset = s->free_byte_offset; + s->free_byte_offset += size; + free_in_cluster -= size; + if (free_in_cluster == 0) + s->free_byte_offset = 0; + if ((offset & (s->cluster_size - 1)) != 0) + update_cluster_refcount(bs, offset >> s->cluster_bits, 1); + } else { + offset = alloc_clusters(bs, s->cluster_size); + cluster_offset = s->free_byte_offset & ~(s->cluster_size - 1); + if ((cluster_offset + s->cluster_size) == offset) { + /* we are lucky: contiguous data */ + offset = s->free_byte_offset; + update_cluster_refcount(bs, offset >> s->cluster_bits, 1); + s->free_byte_offset += size; + } else { + s->free_byte_offset = offset; + goto redo; + } + } + return offset; +} + +static void free_clusters(BlockDriverState *bs, + int64_t offset, int64_t size) +{ + update_refcount(bs, offset, size, -1); +} + +static int grow_refcount_table(BlockDriverState *bs, int min_size) +{ + BDRVQcowState *s = bs->opaque; + int new_table_size, new_table_size2, refcount_table_clusters, i, ret; + uint64_t *new_table; + int64_t table_offset; + uint64_t data64; + uint32_t data32; + int old_table_size; + int64_t old_table_offset; + + if (min_size <= s->refcount_table_size) + return 0; + /* compute new table size */ + refcount_table_clusters = s->refcount_table_size >> (s->cluster_bits - 3); + for(;;) { + if (refcount_table_clusters == 0) { + refcount_table_clusters = 1; + } else { + refcount_table_clusters = (refcount_table_clusters * 3 + 1) / 2; + } + new_table_size = refcount_table_clusters << (s->cluster_bits - 3); + if (min_size <= new_table_size) + break; + } +#ifdef DEBUG_ALLOC2 + printf("grow_refcount_table from %d to %d\n", + s->refcount_table_size, + new_table_size); +#endif + new_table_size2 = new_table_size * sizeof(uint64_t); + new_table = qemu_mallocz(new_table_size2); + if (!new_table) + return -ENOMEM; + memcpy(new_table, s->refcount_table, + s->refcount_table_size * sizeof(uint64_t)); + for(i = 0; i < s->refcount_table_size; i++) + cpu_to_be64s(&new_table[i]); + /* Note: we cannot update the refcount now to avoid recursion */ + table_offset = alloc_clusters_noref(bs, new_table_size2); + ret = bdrv_pwrite(s->hd, table_offset, new_table, new_table_size2); + if (ret != new_table_size2) + goto fail; + for(i = 0; i < s->refcount_table_size; i++) + be64_to_cpus(&new_table[i]); + + data64 = cpu_to_be64(table_offset); + if (bdrv_pwrite(s->hd, offsetof(QCowHeader, refcount_table_offset), + &data64, sizeof(data64)) != sizeof(data64)) + goto fail; + data32 = cpu_to_be32(refcount_table_clusters); + if (bdrv_pwrite(s->hd, offsetof(QCowHeader, refcount_table_clusters), + &data32, sizeof(data32)) != sizeof(data32)) + goto fail; + qemu_free(s->refcount_table); + old_table_offset = s->refcount_table_offset; + old_table_size = s->refcount_table_size; + s->refcount_table = new_table; + s->refcount_table_size = new_table_size; + s->refcount_table_offset = table_offset; + + update_refcount(bs, table_offset, new_table_size2, 1); + free_clusters(bs, old_table_offset, old_table_size * sizeof(uint64_t)); + return 0; + fail: + free_clusters(bs, table_offset, new_table_size2); + qemu_free(new_table); + return -EIO; +} + +/* addend must be 1 or -1 */ +/* XXX: cache several refcount block clusters ? */ +static int update_cluster_refcount(BlockDriverState *bs, + int64_t cluster_index, + int addend) +{ + BDRVQcowState *s = bs->opaque; + int64_t offset, refcount_block_offset; + int ret, refcount_table_index, block_index, refcount; + uint64_t data64; + + refcount_table_index = cluster_index >> (s->cluster_bits - REFCOUNT_SHIFT); + if (refcount_table_index >= s->refcount_table_size) { + if (addend < 0) + return -EINVAL; + ret = grow_refcount_table(bs, refcount_table_index + 1); + if (ret < 0) + return ret; + } + refcount_block_offset = s->refcount_table[refcount_table_index]; + if (!refcount_block_offset) { + if (addend < 0) + return -EINVAL; + /* create a new refcount block */ + /* Note: we cannot update the refcount now to avoid recursion */ + offset = alloc_clusters_noref(bs, s->cluster_size); + memset(s->refcount_block_cache, 0, s->cluster_size); + ret = bdrv_pwrite(s->hd, offset, s->refcount_block_cache, s->cluster_size); + if (ret != s->cluster_size) + return -EINVAL; + s->refcount_table[refcount_table_index] = offset; + data64 = cpu_to_be64(offset); + ret = bdrv_pwrite(s->hd, s->refcount_table_offset + + refcount_table_index * sizeof(uint64_t), + &data64, sizeof(data64)); + if (ret != sizeof(data64)) + return -EINVAL; + + refcount_block_offset = offset; + s->refcount_block_cache_offset = offset; + update_refcount(bs, offset, s->cluster_size, 1); + } else { + if (refcount_block_offset != s->refcount_block_cache_offset) { + if (load_refcount_block(bs, refcount_block_offset) < 0) + return -EIO; + } + } + /* we can update the count and save it */ + block_index = cluster_index & + ((1 << (s->cluster_bits - REFCOUNT_SHIFT)) - 1); + refcount = be16_to_cpu(s->refcount_block_cache[block_index]); + refcount += addend; + if (refcount < 0 || refcount > 0xffff) + return -EINVAL; + if (refcount == 0 && cluster_index < s->free_cluster_index) { + s->free_cluster_index = cluster_index; + } + s->refcount_block_cache[block_index] = cpu_to_be16(refcount); + if (bdrv_pwrite(s->hd, + refcount_block_offset + (block_index << REFCOUNT_SHIFT), + &s->refcount_block_cache[block_index], 2) != 2) + return -EIO; + return refcount; +} + +static void update_refcount(BlockDriverState *bs, + int64_t offset, int64_t length, + int addend) +{ + BDRVQcowState *s = bs->opaque; + int64_t start, last, cluster_offset; + +#ifdef DEBUG_ALLOC2 + printf("update_refcount: offset=%lld size=%lld addend=%d\n", + offset, length, addend); +#endif + if (length <= 0) + return; + start = offset & ~(s->cluster_size - 1); + last = (offset + length - 1) & ~(s->cluster_size - 1); + for(cluster_offset = start; cluster_offset <= last; + cluster_offset += s->cluster_size) { + update_cluster_refcount(bs, cluster_offset >> s->cluster_bits, addend); + } +} + +#ifdef DEBUG_ALLOC +static void inc_refcounts(BlockDriverState *bs, + uint16_t *refcount_table, + int refcount_table_size, + int64_t offset, int64_t size) +{ + BDRVQcowState *s = bs->opaque; + int64_t start, last, cluster_offset; + int k; + + if (size <= 0) + return; + + start = offset & ~(s->cluster_size - 1); + last = (offset + size - 1) & ~(s->cluster_size - 1); + for(cluster_offset = start; cluster_offset <= last; + cluster_offset += s->cluster_size) { + k = cluster_offset >> s->cluster_bits; + if (k < 0 || k >= refcount_table_size) { + printf("ERROR: invalid cluster offset=0x%llx\n", cluster_offset); + } else { + if (++refcount_table[k] == 0) { + printf("ERROR: overflow cluster offset=0x%llx\n", cluster_offset); + } + } + } +} + +static int check_refcounts_l1(BlockDriverState *bs, + uint16_t *refcount_table, + int refcount_table_size, + int64_t l1_table_offset, int l1_size, + int check_copied) +{ + BDRVQcowState *s = bs->opaque; + uint64_t *l1_table, *l2_table, l2_offset, offset, l1_size2; + int l2_size, i, j, nb_csectors, refcount; + + l2_table = NULL; + l1_size2 = l1_size * sizeof(uint64_t); + + inc_refcounts(bs, refcount_table, refcount_table_size, + l1_table_offset, l1_size2); + + l1_table = qemu_malloc(l1_size2); + if (!l1_table) + goto fail; + if (bdrv_pread(s->hd, l1_table_offset, + l1_table, l1_size2) != l1_size2) + goto fail; + for(i = 0;i < l1_size; i++) + be64_to_cpus(&l1_table[i]); + + l2_size = s->l2_size * sizeof(uint64_t); + l2_table = qemu_malloc(l2_size); + if (!l2_table) + goto fail; + for(i = 0; i < l1_size; i++) { + l2_offset = l1_table[i]; + if (l2_offset) { + if (check_copied) { + refcount = get_refcount(bs, (l2_offset & ~QCOW_OFLAG_COPIED) >> s->cluster_bits); + if ((refcount == 1) != ((l2_offset & QCOW_OFLAG_COPIED) != 0)) { + printf("ERROR OFLAG_COPIED: l2_offset=%llx refcount=%d\n", + l2_offset, refcount); + } + } + l2_offset &= ~QCOW_OFLAG_COPIED; + if (bdrv_pread(s->hd, l2_offset, l2_table, l2_size) != l2_size) + goto fail; + for(j = 0; j < s->l2_size; j++) { + offset = be64_to_cpu(l2_table[j]); + if (offset != 0) { + if (offset & QCOW_OFLAG_COMPRESSED) { + if (offset & QCOW_OFLAG_COPIED) { + printf("ERROR: cluster %lld: copied flag must never be set for compressed clusters\n", + offset >> s->cluster_bits); + offset &= ~QCOW_OFLAG_COPIED; + } + nb_csectors = ((offset >> s->csize_shift) & + s->csize_mask) + 1; + offset &= s->cluster_offset_mask; + inc_refcounts(bs, refcount_table, + refcount_table_size, + offset & ~511, nb_csectors * 512); + } else { + if (check_copied) { + refcount = get_refcount(bs, (offset & ~QCOW_OFLAG_COPIED) >> s->cluster_bits); + if ((refcount == 1) != ((offset & QCOW_OFLAG_COPIED) != 0)) { + printf("ERROR OFLAG_COPIED: offset=%llx refcount=%d\n", + offset, refcount); + } + } + offset &= ~QCOW_OFLAG_COPIED; + inc_refcounts(bs, refcount_table, + refcount_table_size, + offset, s->cluster_size); + } + } + } + inc_refcounts(bs, refcount_table, + refcount_table_size, + l2_offset, + s->cluster_size); + } + } + qemu_free(l1_table); + qemu_free(l2_table); + return 0; + fail: + printf("ERROR: I/O error in check_refcounts_l1\n"); + qemu_free(l1_table); + qemu_free(l2_table); + return -EIO; +} + +static void check_refcounts(BlockDriverState *bs) +{ + BDRVQcowState *s = bs->opaque; + int64_t size; + int nb_clusters, refcount1, refcount2, i; + QCowSnapshot *sn; + uint16_t *refcount_table; + + size = bdrv_getlength(s->hd); + nb_clusters = (size + s->cluster_size - 1) >> s->cluster_bits; + refcount_table = qemu_mallocz(nb_clusters * sizeof(uint16_t)); + + /* header */ + inc_refcounts(bs, refcount_table, nb_clusters, + 0, s->cluster_size); + + check_refcounts_l1(bs, refcount_table, nb_clusters, + s->l1_table_offset, s->l1_size, 1); + + /* snapshots */ + for(i = 0; i < s->nb_snapshots; i++) { + sn = s->snapshots + i; + check_refcounts_l1(bs, refcount_table, nb_clusters, + sn->l1_table_offset, sn->l1_size, 0); + } + inc_refcounts(bs, refcount_table, nb_clusters, + s->snapshots_offset, s->snapshots_size); + + /* refcount data */ + inc_refcounts(bs, refcount_table, nb_clusters, + s->refcount_table_offset, + s->refcount_table_size * sizeof(uint64_t)); + for(i = 0; i < s->refcount_table_size; i++) { + int64_t offset; + offset = s->refcount_table[i]; + if (offset != 0) { + inc_refcounts(bs, refcount_table, nb_clusters, + offset, s->cluster_size); + } + } + + /* compare ref counts */ + for(i = 0; i < nb_clusters; i++) { + refcount1 = get_refcount(bs, i); + refcount2 = refcount_table[i]; + if (refcount1 != refcount2) + printf("ERROR cluster %d refcount=%d reference=%d\n", + i, refcount1, refcount2); + } + + qemu_free(refcount_table); +} + +#if 0 +static void dump_refcounts(BlockDriverState *bs) +{ + BDRVQcowState *s = bs->opaque; + int64_t nb_clusters, k, k1, size; + int refcount; + + size = bdrv_getlength(s->hd); + nb_clusters = (size + s->cluster_size - 1) >> s->cluster_bits; + for(k = 0; k < nb_clusters;) { + k1 = k; + refcount = get_refcount(bs, k); + k++; + while (k < nb_clusters && get_refcount(bs, k) == refcount) + k++; + printf("%lld: refcount=%d nb=%lld\n", k, refcount, k - k1); + } +} +#endif +#endif + +BlockDriver bdrv_qcow2 = { + "qcow2", + sizeof(BDRVQcowState), + qcow_probe, + qcow_open, + NULL, + NULL, + qcow_close, + qcow_create, + qcow_flush, + qcow_is_allocated, + qcow_set_key, + qcow_make_empty, + + .bdrv_aio_read = qcow_aio_read, + .bdrv_aio_write = qcow_aio_write, + .bdrv_aio_cancel = qcow_aio_cancel, + .aiocb_size = sizeof(QCowAIOCB), + .bdrv_write_compressed = qcow_write_compressed, + + .bdrv_snapshot_create = qcow_snapshot_create, + .bdrv_snapshot_goto = qcow_snapshot_goto, + .bdrv_snapshot_delete = qcow_snapshot_delete, + .bdrv_snapshot_list = qcow_snapshot_list, + .bdrv_get_info = qcow_get_info, +}; diff --git a/lib/qemu/block-raw-posix.c b/lib/qemu/block-raw-posix.c new file mode 100644 index 0000000..4a430dc --- /dev/null +++ b/lib/qemu/block-raw-posix.c @@ -0,0 +1,928 @@ +/* + * Block driver for RAW files (posix) + * + * Copyright (c) 2006 Fabrice Bellard + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include + +#include "qemu-common.h" +#ifndef QEMU_IMG +#include "qemu-timer.h" +#include "exec-all.h" +#endif +#include "block_int.h" +#include +#include + +#ifdef CONFIG_COCOA +#include +#include +#include +#include +#include +#include +#include +//#include +#include +#endif + +#ifdef __sun__ +#define _POSIX_PTHREAD_SEMANTICS 1 +#include +#include +#endif +#ifdef __linux__ +#include +#include +#include +#endif +#ifdef __FreeBSD__ +#include +#endif + +//#define DEBUG_FLOPPY + +//#define DEBUG_BLOCK +#if defined(DEBUG_BLOCK) && !defined(QEMU_IMG) +#define DEBUG_BLOCK_PRINT(formatCstr, args...) do { if (loglevel != 0) \ + { fprintf(logfile, formatCstr, ##args); fflush(logfile); } } while (0) +#else +#define DEBUG_BLOCK_PRINT(formatCstr, args...) +#endif + +#define FTYPE_FILE 0 +#define FTYPE_CD 1 +#define FTYPE_FD 2 + +/* if the FD is not accessed during that time (in ms), we try to + reopen it to see if the disk has been changed */ +#define FD_OPEN_TIMEOUT 1000 + +typedef struct BDRVRawState { + int fd; + int type; + unsigned int lseek_err_cnt; +#if defined(__linux__) + /* linux floppy specific */ + int fd_open_flags; + int64_t fd_open_time; + int64_t fd_error_time; + int fd_got_error; + int fd_media_changed; +#endif +} BDRVRawState; + +static int fd_open(BlockDriverState *bs); + +static int raw_open(BlockDriverState *bs, const char *filename, int flags) +{ + BDRVRawState *s = bs->opaque; + int fd, open_flags, ret; + + s->lseek_err_cnt = 0; + + open_flags = O_BINARY; + if ((flags & BDRV_O_ACCESS) == O_RDWR) { + open_flags |= O_RDWR; + } else { + open_flags |= O_RDONLY; + bs->read_only = 1; + } + if (flags & BDRV_O_CREAT) + open_flags |= O_CREAT | O_TRUNC; +#ifdef O_DIRECT + if (flags & BDRV_O_DIRECT) + open_flags |= O_DIRECT; +#endif + + s->type = FTYPE_FILE; + + fd = open(filename, open_flags, 0644); + if (fd < 0) { + ret = -errno; + if (ret == -EROFS) + ret = -EACCES; + return ret; + } + s->fd = fd; + return 0; +} + +/* XXX: use host sector size if necessary with: +#ifdef DIOCGSECTORSIZE + { + unsigned int sectorsize = 512; + if (!ioctl(fd, DIOCGSECTORSIZE, §orsize) && + sectorsize > bufsize) + bufsize = sectorsize; + } +#endif +#ifdef CONFIG_COCOA + u_int32_t blockSize = 512; + if ( !ioctl( fd, DKIOCGETBLOCKSIZE, &blockSize ) && blockSize > bufsize) { + bufsize = blockSize; + } +#endif +*/ + +static int raw_pread(BlockDriverState *bs, int64_t offset, + uint8_t *buf, int count) +{ + BDRVRawState *s = bs->opaque; + int ret; + + ret = fd_open(bs); + if (ret < 0) + return ret; + + if (offset >= 0 && lseek(s->fd, offset, SEEK_SET) == (off_t)-1) { + ++(s->lseek_err_cnt); + if(s->lseek_err_cnt <= 10) { + DEBUG_BLOCK_PRINT("raw_pread(%d:%s, %" PRId64 ", %p, %d) [%" PRId64 + "] lseek failed : %d = %s\n", + s->fd, bs->filename, offset, buf, count, + bs->total_sectors, errno, strerror(errno)); + } + return -1; + } + s->lseek_err_cnt=0; + + ret = read(s->fd, buf, count); + if (ret == count) + goto label__raw_read__success; + + DEBUG_BLOCK_PRINT("raw_pread(%d:%s, %" PRId64 ", %p, %d) [%" PRId64 + "] read failed %d : %d = %s\n", + s->fd, bs->filename, offset, buf, count, + bs->total_sectors, ret, errno, strerror(errno)); + + /* Try harder for CDrom. */ + if (bs->type == BDRV_TYPE_CDROM) { + lseek(s->fd, offset, SEEK_SET); + ret = read(s->fd, buf, count); + if (ret == count) + goto label__raw_read__success; + lseek(s->fd, offset, SEEK_SET); + ret = read(s->fd, buf, count); + if (ret == count) + goto label__raw_read__success; + + DEBUG_BLOCK_PRINT("raw_pread(%d:%s, %" PRId64 ", %p, %d) [%" PRId64 + "] retry read failed %d : %d = %s\n", + s->fd, bs->filename, offset, buf, count, + bs->total_sectors, ret, errno, strerror(errno)); + } + +label__raw_read__success: + + return ret; +} + +static int raw_pwrite(BlockDriverState *bs, int64_t offset, + const uint8_t *buf, int count) +{ + BDRVRawState *s = bs->opaque; + int ret; + + ret = fd_open(bs); + if (ret < 0) + return ret; + + if (offset >= 0 && lseek(s->fd, offset, SEEK_SET) == (off_t)-1) { + ++(s->lseek_err_cnt); + if(s->lseek_err_cnt) { + DEBUG_BLOCK_PRINT("raw_pwrite(%d:%s, %" PRId64 ", %p, %d) [%" + PRId64 "] lseek failed : %d = %s\n", + s->fd, bs->filename, offset, buf, count, + bs->total_sectors, errno, strerror(errno)); + } + return -1; + } + s->lseek_err_cnt = 0; + + ret = write(s->fd, buf, count); + if (ret == count) + goto label__raw_write__success; + + DEBUG_BLOCK_PRINT("raw_pwrite(%d:%s, %" PRId64 ", %p, %d) [%" PRId64 + "] write failed %d : %d = %s\n", + s->fd, bs->filename, offset, buf, count, + bs->total_sectors, ret, errno, strerror(errno)); + +label__raw_write__success: + + return ret; +} + +/***********************************************************/ +/* Unix AIO using POSIX AIO */ + +typedef struct RawAIOCB { + BlockDriverAIOCB common; + struct aiocb aiocb; + struct RawAIOCB *next; +} RawAIOCB; + +static int aio_sig_num = SIGUSR2; +static RawAIOCB *first_aio; /* AIO issued */ +static int aio_initialized = 0; + +static void aio_signal_handler(int signum) +{ +#ifndef QEMU_IMG + CPUState *env = cpu_single_env; + if (env) { + /* stop the currently executing cpu because a timer occured */ + cpu_interrupt(env, CPU_INTERRUPT_EXIT); +#ifdef USE_KQEMU + if (env->kqemu_enabled) { + kqemu_cpu_interrupt(env); + } +#endif + } +#endif +} + +void qemu_aio_init(void) +{ + struct sigaction act; + + aio_initialized = 1; + + sigfillset(&act.sa_mask); + act.sa_flags = 0; /* do not restart syscalls to interrupt select() */ + act.sa_handler = aio_signal_handler; + sigaction(aio_sig_num, &act, NULL); + + //#if defined(__GLIBC__) && defined(__linux__) + // { + // /* XXX: aio thread exit seems to hang on RedHat 9 and this init + // seems to fix the problem. */ + // struct aioinit ai; + // memset(&ai, 0, sizeof(ai)); + // ai.aio_threads = 1; + // ai.aio_num = 1; + // ai.aio_idle_time = 365 * 100000; + // aio_init(&ai); + // } + //#endif +} + +void qemu_aio_poll(void) +{ + RawAIOCB *acb, **pacb; + int ret; + + for(;;) { + pacb = &first_aio; + for(;;) { + acb = *pacb; + if (!acb) + goto the_end; + ret = aio_error(&acb->aiocb); + if (ret == ECANCELED) { + /* remove the request */ + *pacb = acb->next; + qemu_aio_release(acb); + } else if (ret != EINPROGRESS) { + /* end of aio */ + if (ret == 0) { + ret = aio_return(&acb->aiocb); + if (ret == acb->aiocb.aio_nbytes) + ret = 0; + else + ret = -EINVAL; + } else { + ret = -ret; + } + /* remove the request */ + *pacb = acb->next; + /* call the callback */ + acb->common.cb(acb->common.opaque, ret); + qemu_aio_release(acb); + break; + } else { + pacb = &acb->next; + } + } + } + the_end: ; +} + +/* Wait for all IO requests to complete. */ +void qemu_aio_flush(void) +{ + qemu_aio_wait_start(); + qemu_aio_poll(); + while (first_aio) { + qemu_aio_wait(); + } + qemu_aio_wait_end(); +} + +/* wait until at least one AIO was handled */ +static sigset_t wait_oset; + +void qemu_aio_wait_start(void) +{ + sigset_t set; + + if (!aio_initialized) + qemu_aio_init(); + sigemptyset(&set); + sigaddset(&set, aio_sig_num); + sigprocmask(SIG_BLOCK, &set, &wait_oset); +} + +void qemu_aio_wait(void) +{ + sigset_t set; + int nb_sigs; + +#ifndef QEMU_IMG + if (qemu_bh_poll()) + return; +#endif + sigemptyset(&set); + sigaddset(&set, aio_sig_num); + sigwait(&set, &nb_sigs); + qemu_aio_poll(); +} + +void qemu_aio_wait_end(void) +{ + sigprocmask(SIG_SETMASK, &wait_oset, NULL); +} + +static RawAIOCB *raw_aio_setup(BlockDriverState *bs, + int64_t sector_num, uint8_t *buf, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque) +{ + BDRVRawState *s = bs->opaque; + RawAIOCB *acb; + + if (fd_open(bs) < 0) + return NULL; + + acb = qemu_aio_get(bs, cb, opaque); + if (!acb) + return NULL; + acb->aiocb.aio_fildes = s->fd; + acb->aiocb.aio_sigevent.sigev_signo = aio_sig_num; + acb->aiocb.aio_sigevent.sigev_notify = SIGEV_SIGNAL; + acb->aiocb.aio_buf = buf; + if (nb_sectors < 0) + acb->aiocb.aio_nbytes = -nb_sectors; + else + acb->aiocb.aio_nbytes = nb_sectors * 512; + acb->aiocb.aio_offset = sector_num * 512; + acb->next = first_aio; + first_aio = acb; + return acb; +} + +static BlockDriverAIOCB *raw_aio_read(BlockDriverState *bs, + int64_t sector_num, uint8_t *buf, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque) +{ + RawAIOCB *acb; + + acb = raw_aio_setup(bs, sector_num, buf, nb_sectors, cb, opaque); + if (!acb) + return NULL; + if (aio_read(&acb->aiocb) < 0) { + qemu_aio_release(acb); + return NULL; + } + return &acb->common; +} + +static BlockDriverAIOCB *raw_aio_write(BlockDriverState *bs, + int64_t sector_num, const uint8_t *buf, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque) +{ + RawAIOCB *acb; + + acb = raw_aio_setup(bs, sector_num, (uint8_t*)buf, nb_sectors, cb, opaque); + if (!acb) + return NULL; + if (aio_write(&acb->aiocb) < 0) { + qemu_aio_release(acb); + return NULL; + } + return &acb->common; +} + +static void raw_aio_cancel(BlockDriverAIOCB *blockacb) +{ + int ret; + RawAIOCB *acb = (RawAIOCB *)blockacb; + RawAIOCB **pacb; + + ret = aio_cancel(acb->aiocb.aio_fildes, &acb->aiocb); + if (ret == AIO_NOTCANCELED) { + /* fail safe: if the aio could not be canceled, we wait for + it */ + while (aio_error(&acb->aiocb) == EINPROGRESS); + } + + /* remove the callback from the queue */ + pacb = &first_aio; + for(;;) { + if (*pacb == NULL) { + break; + } else if (*pacb == acb) { + *pacb = acb->next; + qemu_aio_release(acb); + break; + } + pacb = &acb->next; + } +} + +static void raw_close(BlockDriverState *bs) +{ + BDRVRawState *s = bs->opaque; + if (s->fd >= 0) { + close(s->fd); + s->fd = -1; + } +} + +static int raw_truncate(BlockDriverState *bs, int64_t offset) +{ + BDRVRawState *s = bs->opaque; + if (s->type != FTYPE_FILE) + return -ENOTSUP; + if (ftruncate(s->fd, offset) < 0) + return -errno; + return 0; +} + +static int64_t raw_getlength(BlockDriverState *bs) +{ + BDRVRawState *s = bs->opaque; + int fd = s->fd; + int64_t size; +#ifdef _BSD + struct stat sb; +#endif +#ifdef __sun__ + struct dk_minfo minfo; + int rv; +#endif + int ret; + + ret = fd_open(bs); + if (ret < 0) + return ret; + +#ifdef _BSD + if (!fstat(fd, &sb) && (S_IFCHR & sb.st_mode)) { +#ifdef DIOCGMEDIASIZE + if (ioctl(fd, DIOCGMEDIASIZE, (off_t *)&size)) +#endif +#ifdef CONFIG_COCOA + size = LONG_LONG_MAX; +#else + size = lseek(fd, 0LL, SEEK_END); +#endif + } else +#endif +#ifdef __sun__ + /* + * use the DKIOCGMEDIAINFO ioctl to read the size. + */ + rv = ioctl ( fd, DKIOCGMEDIAINFO, &minfo ); + if ( rv != -1 ) { + size = minfo.dki_lbsize * minfo.dki_capacity; + } else /* there are reports that lseek on some devices + fails, but irc discussion said that contingency + on contingency was overkill */ +#endif + { + size = lseek(fd, 0, SEEK_END); + } + return size; +} + +static int raw_create(const char *filename, int64_t total_size, + const char *backing_file, int flags) +{ + int fd; + + if (flags || backing_file) + return -ENOTSUP; + + fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, + 0644); + if (fd < 0) + return -EIO; + if(ftruncate(fd, total_size * 512)<0) return -1; + close(fd); + return 0; +} + +static void raw_flush(BlockDriverState *bs) +{ + BDRVRawState *s = bs->opaque; + fsync(s->fd); +} + +BlockDriver bdrv_raw = { + "raw", + sizeof(BDRVRawState), + NULL, /* no probe for protocols */ + raw_open, + NULL, + NULL, + raw_close, + raw_create, + raw_flush, + + .bdrv_aio_read = raw_aio_read, + .bdrv_aio_write = raw_aio_write, + .bdrv_aio_cancel = raw_aio_cancel, + .aiocb_size = sizeof(RawAIOCB), + .protocol_name = "file", + .bdrv_pread = raw_pread, + .bdrv_pwrite = raw_pwrite, + .bdrv_truncate = raw_truncate, + .bdrv_getlength = raw_getlength, +}; + +/***********************************************/ +/* host device */ + +#ifdef CONFIG_COCOA +static kern_return_t FindEjectableCDMedia( io_iterator_t *mediaIterator ); +static kern_return_t GetBSDPath( io_iterator_t mediaIterator, char *bsdPath, CFIndex maxPathSize ); + +kern_return_t FindEjectableCDMedia( io_iterator_t *mediaIterator ) +{ + kern_return_t kernResult; + mach_port_t masterPort; + CFMutableDictionaryRef classesToMatch; + + kernResult = IOMasterPort( MACH_PORT_NULL, &masterPort ); + if ( KERN_SUCCESS != kernResult ) { + printf( "IOMasterPort returned %d\n", kernResult ); + } + + classesToMatch = IOServiceMatching( kIOCDMediaClass ); + if ( classesToMatch == NULL ) { + printf( "IOServiceMatching returned a NULL dictionary.\n" ); + } else { + CFDictionarySetValue( classesToMatch, CFSTR( kIOMediaEjectableKey ), kCFBooleanTrue ); + } + kernResult = IOServiceGetMatchingServices( masterPort, classesToMatch, mediaIterator ); + if ( KERN_SUCCESS != kernResult ) + { + printf( "IOServiceGetMatchingServices returned %d\n", kernResult ); + } + + return kernResult; +} + +kern_return_t GetBSDPath( io_iterator_t mediaIterator, char *bsdPath, CFIndex maxPathSize ) +{ + io_object_t nextMedia; + kern_return_t kernResult = KERN_FAILURE; + *bsdPath = '\0'; + nextMedia = IOIteratorNext( mediaIterator ); + if ( nextMedia ) + { + CFTypeRef bsdPathAsCFString; + bsdPathAsCFString = IORegistryEntryCreateCFProperty( nextMedia, CFSTR( kIOBSDNameKey ), kCFAllocatorDefault, 0 ); + if ( bsdPathAsCFString ) { + size_t devPathLength; + strcpy( bsdPath, _PATH_DEV ); + strcat( bsdPath, "r" ); + devPathLength = strlen( bsdPath ); + if ( CFStringGetCString( bsdPathAsCFString, bsdPath + devPathLength, maxPathSize - devPathLength, kCFStringEncodingASCII ) ) { + kernResult = KERN_SUCCESS; + } + CFRelease( bsdPathAsCFString ); + } + IOObjectRelease( nextMedia ); + } + + return kernResult; +} + +#endif + +static int hdev_open(BlockDriverState *bs, const char *filename, int flags) +{ + BDRVRawState *s = bs->opaque; + int fd, open_flags, ret; + +#ifdef CONFIG_COCOA + if (strstart(filename, "/dev/cdrom", NULL)) { + kern_return_t kernResult; + io_iterator_t mediaIterator; + char bsdPath[ MAXPATHLEN ]; + int fd; + + kernResult = FindEjectableCDMedia( &mediaIterator ); + kernResult = GetBSDPath( mediaIterator, bsdPath, sizeof( bsdPath ) ); + + if ( bsdPath[ 0 ] != '\0' ) { + strcat(bsdPath,"s0"); + /* some CDs don't have a partition 0 */ + fd = open(bsdPath, O_RDONLY | O_BINARY | O_LARGEFILE); + if (fd < 0) { + bsdPath[strlen(bsdPath)-1] = '1'; + } else { + close(fd); + } + filename = bsdPath; + } + + if ( mediaIterator ) + IOObjectRelease( mediaIterator ); + } +#endif + open_flags = O_BINARY; + if ((flags & BDRV_O_ACCESS) == O_RDWR) { + open_flags |= O_RDWR; + } else { + open_flags |= O_RDONLY; + bs->read_only = 1; + } +#ifdef O_DIRECT + if (flags & BDRV_O_DIRECT) + open_flags |= O_DIRECT; +#endif + + s->type = FTYPE_FILE; +#if defined(__linux__) + if (strstart(filename, "/dev/cd", NULL)) { + /* open will not fail even if no CD is inserted */ + open_flags |= O_NONBLOCK; + s->type = FTYPE_CD; + } else if (strstart(filename, "/dev/fd", NULL)) { + s->type = FTYPE_FD; + s->fd_open_flags = open_flags; + /* open will not fail even if no floppy is inserted */ + open_flags |= O_NONBLOCK; + } else if (strstart(filename, "/dev/sg", NULL)) { + bs->sg = 1; + } +#endif + fd = open(filename, open_flags, 0644); + if (fd < 0) { + ret = -errno; + if (ret == -EROFS) + ret = -EACCES; + return ret; + } + s->fd = fd; +#if defined(__linux__) + /* close fd so that we can reopen it as needed */ + if (s->type == FTYPE_FD) { + close(s->fd); + s->fd = -1; + s->fd_media_changed = 1; + } +#endif + return 0; +} + +#if defined(__linux__) && !defined(QEMU_IMG) + +/* Note: we do not have a reliable method to detect if the floppy is + present. The current method is to try to open the floppy at every + I/O and to keep it opened during a few hundreds of ms. */ +static int fd_open(BlockDriverState *bs) +{ + BDRVRawState *s = bs->opaque; + int last_media_present; + + if (s->type != FTYPE_FD) + return 0; + last_media_present = (s->fd >= 0); + if (s->fd >= 0 && + (qemu_get_clock(rt_clock) - s->fd_open_time) >= FD_OPEN_TIMEOUT) { + close(s->fd); + s->fd = -1; +#ifdef DEBUG_FLOPPY + printf("Floppy closed\n"); +#endif + } + if (s->fd < 0) { + if (s->fd_got_error && + (qemu_get_clock(rt_clock) - s->fd_error_time) < FD_OPEN_TIMEOUT) { +#ifdef DEBUG_FLOPPY + printf("No floppy (open delayed)\n"); +#endif + return -EIO; + } + s->fd = open(bs->filename, s->fd_open_flags); + if (s->fd < 0) { + s->fd_error_time = qemu_get_clock(rt_clock); + s->fd_got_error = 1; + if (last_media_present) + s->fd_media_changed = 1; +#ifdef DEBUG_FLOPPY + printf("No floppy\n"); +#endif + return -EIO; + } +#ifdef DEBUG_FLOPPY + printf("Floppy opened\n"); +#endif + } + if (!last_media_present) + s->fd_media_changed = 1; + s->fd_open_time = qemu_get_clock(rt_clock); + s->fd_got_error = 0; + return 0; +} +#else +static int fd_open(BlockDriverState *bs) +{ + return 0; +} +#endif + +#if defined(__linux__) + +static int raw_is_inserted(BlockDriverState *bs) +{ + BDRVRawState *s = bs->opaque; + int ret; + + switch(s->type) { + case FTYPE_CD: + ret = ioctl(s->fd, CDROM_DRIVE_STATUS, CDSL_CURRENT); + if (ret == CDS_DISC_OK) + return 1; + else + return 0; + break; + case FTYPE_FD: + ret = fd_open(bs); + return (ret >= 0); + default: + return 1; + } +} + +/* currently only used by fdc.c, but a CD version would be good too */ +static int raw_media_changed(BlockDriverState *bs) +{ + BDRVRawState *s = bs->opaque; + + switch(s->type) { + case FTYPE_FD: + { + int ret; + /* XXX: we do not have a true media changed indication. It + does not work if the floppy is changed without trying + to read it */ + fd_open(bs); + ret = s->fd_media_changed; + s->fd_media_changed = 0; +#ifdef DEBUG_FLOPPY + printf("Floppy changed=%d\n", ret); +#endif + return ret; + } + default: + return -ENOTSUP; + } +} + +static int raw_eject(BlockDriverState *bs, int eject_flag) +{ + BDRVRawState *s = bs->opaque; + + switch(s->type) { + case FTYPE_CD: + if (eject_flag) { + if (ioctl (s->fd, CDROMEJECT, NULL) < 0) + perror("CDROMEJECT"); + } else { + if (ioctl (s->fd, CDROMCLOSETRAY, NULL) < 0) + perror("CDROMEJECT"); + } + break; + case FTYPE_FD: + { + int fd; + if (s->fd >= 0) { + close(s->fd); + s->fd = -1; + } + fd = open(bs->filename, s->fd_open_flags | O_NONBLOCK); + if (fd >= 0) { + if (ioctl(fd, FDEJECT, 0) < 0) + perror("FDEJECT"); + close(fd); + } + } + break; + default: + return -ENOTSUP; + } + return 0; +} + +static int raw_set_locked(BlockDriverState *bs, int locked) +{ + BDRVRawState *s = bs->opaque; + + switch(s->type) { + case FTYPE_CD: + if (ioctl (s->fd, CDROM_LOCKDOOR, locked) < 0) { + /* Note: an error can happen if the distribution automatically + mounts the CD-ROM */ + // perror("CDROM_LOCKDOOR"); + } + break; + default: + return -ENOTSUP; + } + return 0; +} + +static int raw_ioctl(BlockDriverState *bs, unsigned long int req, void *buf) +{ + BDRVRawState *s = bs->opaque; + + return ioctl(s->fd, req, buf); +} +#else + +static int raw_is_inserted(BlockDriverState *bs) +{ + return 1; +} + +static int raw_media_changed(BlockDriverState *bs) +{ + return -ENOTSUP; +} + +static int raw_eject(BlockDriverState *bs, int eject_flag) +{ + return -ENOTSUP; +} + +static int raw_set_locked(BlockDriverState *bs, int locked) +{ + return -ENOTSUP; +} + +static int raw_ioctl(BlockDriverState *bs, unsigned long int req, void *buf) +{ + return -ENOTSUP; +} +#endif /* !linux */ + +BlockDriver bdrv_host_device = { + "host_device", + sizeof(BDRVRawState), + NULL, /* no probe for protocols */ + hdev_open, + NULL, + NULL, + raw_close, + NULL, + raw_flush, + + .bdrv_aio_read = raw_aio_read, + .bdrv_aio_write = raw_aio_write, + .bdrv_aio_cancel = raw_aio_cancel, + .aiocb_size = sizeof(RawAIOCB), + .bdrv_pread = raw_pread, + .bdrv_pwrite = raw_pwrite, + .bdrv_getlength = raw_getlength, + + /* removable device support */ + .bdrv_is_inserted = raw_is_inserted, + .bdrv_media_changed = raw_media_changed, + .bdrv_eject = raw_eject, + .bdrv_set_locked = raw_set_locked, + /* generic scsi device */ + .bdrv_ioctl = raw_ioctl, +}; diff --git a/lib/qemu/block-vmdk.c b/lib/qemu/block-vmdk.c new file mode 100644 index 0000000..6e6ffe4 --- /dev/null +++ b/lib/qemu/block-vmdk.c @@ -0,0 +1,835 @@ +/* + * Block driver for the VMDK format + * + * Copyright (c) 2004 Fabrice Bellard + * Copyright (c) 2005 Filip Navara + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "aff-block-vmdk.h" /* added for AFF */ + +#include "qemu-common.h" +#include "block_int.h" + +#define VMDK3_MAGIC (('C' << 24) | ('O' << 16) | ('W' << 8) | 'D') +#define VMDK4_MAGIC (('K' << 24) | ('D' << 16) | ('M' << 8) | 'V') + +typedef struct { + uint32_t version; + uint32_t flags; + uint32_t disk_sectors; + uint32_t granularity; + uint32_t l1dir_offset; + uint32_t l1dir_size; + uint32_t file_sectors; + uint32_t cylinders; + uint32_t heads; + uint32_t sectors_per_track; +} VMDK3Header; + +typedef struct { + uint32_t version; + uint32_t flags; + int64_t capacity; + int64_t granularity; + int64_t desc_offset; + int64_t desc_size; + int32_t num_gtes_per_gte; + int64_t rgd_offset; + int64_t gd_offset; + int64_t grain_offset; + char filler[1]; + char check_bytes[4]; +} __attribute__((packed)) VMDK4Header; + +#define L2_CACHE_SIZE 16 + +typedef struct BDRVVmdkState { + BlockDriverState *hd; + int64_t l1_table_offset; + int64_t l1_backup_table_offset; + uint32_t *l1_table; + uint32_t *l1_backup_table; + unsigned int l1_size; + uint32_t l1_entry_sectors; + + unsigned int l2_size; + uint32_t *l2_cache; + uint32_t l2_cache_offsets[L2_CACHE_SIZE]; + uint32_t l2_cache_counts[L2_CACHE_SIZE]; + + unsigned int cluster_sectors; + uint32_t parent_cid; + int is_parent; +} BDRVVmdkState; + +typedef struct VmdkMetaData { + uint32_t offset; + unsigned int l1_index; + unsigned int l2_index; + unsigned int l2_offset; + int valid; +} VmdkMetaData; + +typedef struct ActiveBDRVState{ + BlockDriverState *hd; // active image handler + uint64_t cluster_offset; // current write offset +}ActiveBDRVState; + +static ActiveBDRVState activeBDRV; + + +static int vmdk_probe(const uint8_t *buf, int buf_size, const char *filename) +{ + uint32_t magic; + + if (buf_size < 4) + return 0; + magic = be32_to_cpu(*(uint32_t *)buf); + if (magic == VMDK3_MAGIC || + magic == VMDK4_MAGIC) + return 100; + else + return 0; +} + +#define CHECK_CID 1 + +#define SECTOR_SIZE 512 +#define DESC_SIZE 20*SECTOR_SIZE // 20 sectors of 512 bytes each +#define HEADER_SIZE 512 // first sector of 512 bytes + +static uint32_t vmdk_read_cid(BlockDriverState *bs, int parent) +{ + BDRVVmdkState *s = bs->opaque; + char desc[DESC_SIZE]; + uint32_t cid; + char *p_name, *cid_str; + size_t cid_str_size; + + /* the descriptor offset = 0x200 */ + if (bdrv_pread(s->hd, 0x200, desc, DESC_SIZE) != DESC_SIZE) + return 0; + + if (parent) { + cid_str = "parentCID"; + cid_str_size = sizeof("parentCID"); + } else { + cid_str = "CID"; + cid_str_size = sizeof("CID"); + } + + if ((p_name = strstr(desc,cid_str)) != 0) { + p_name += cid_str_size; + sscanf(p_name,"%x",&cid); + } + + return cid; +} + +static int vmdk_write_cid(BlockDriverState *bs, uint32_t cid) +{ + BDRVVmdkState *s = bs->opaque; + char desc[DESC_SIZE], tmp_desc[DESC_SIZE]; + char *p_name, *tmp_str; + + /* the descriptor offset = 0x200 */ + if (bdrv_pread(s->hd, 0x200, desc, DESC_SIZE) != DESC_SIZE) + return -1; + + tmp_str = strstr(desc,"parentCID"); + strcpy(tmp_desc, tmp_str); + if ((p_name = strstr(desc,"CID")) != 0) { + p_name += sizeof("CID"); + sprintf(p_name,"%x\n",cid); + strcat(desc,tmp_desc); + } + + if (bdrv_pwrite(s->hd, 0x200, desc, DESC_SIZE) != DESC_SIZE) + return -1; + return 0; +} + +static int vmdk_is_cid_valid(BlockDriverState *bs) +{ +#ifdef CHECK_CID + BDRVVmdkState *s = bs->opaque; + BlockDriverState *p_bs = s->hd->backing_hd; + uint32_t cur_pcid; + + if (p_bs) { + cur_pcid = vmdk_read_cid(p_bs,0); + if (s->parent_cid != cur_pcid) + // CID not valid + return 0; + } +#endif + // CID valid + return 1; +} + +static int vmdk_snapshot_create(const char *filename, const char *backing_file) +{ + int snp_fd, p_fd; + uint32_t p_cid; + char *p_name, *gd_buf, *rgd_buf; + const char *real_filename, *temp_str; + VMDK4Header header; + uint32_t gde_entries, gd_size; + int64_t gd_offset, rgd_offset, capacity, gt_size; + char p_desc[DESC_SIZE], s_desc[DESC_SIZE], hdr[HEADER_SIZE]; + char *desc_template = + "# Disk DescriptorFile\n" + "version=1\n" + "CID=%x\n" + "parentCID=%x\n" + "createType=\"monolithicSparse\"\n" + "parentFileNameHint=\"%s\"\n" + "\n" + "# Extent description\n" + "RW %lu SPARSE \"%s\"\n" + "\n" + "# The Disk Data Base \n" + "#DDB\n" + "\n"; + + snp_fd = open(filename, O_RDWR | O_CREAT | O_TRUNC | O_BINARY | O_LARGEFILE, 0644); + if (snp_fd < 0) + return -1; + p_fd = open(backing_file, O_RDONLY | O_BINARY | O_LARGEFILE); + if (p_fd < 0) { + close(snp_fd); + return -1; + } + + /* read the header */ + if (lseek(p_fd, 0x0, SEEK_SET) == -1) + goto fail; + if (read(p_fd, hdr, HEADER_SIZE) != HEADER_SIZE) + goto fail; + + /* write the header */ + if (lseek(snp_fd, 0x0, SEEK_SET) == -1) + goto fail; + if (write(snp_fd, hdr, HEADER_SIZE) == -1) + goto fail; + + memset(&header, 0, sizeof(header)); + memcpy(&header,&hdr[4], sizeof(header)); // skip the VMDK4_MAGIC + + if(ftruncate(snp_fd, header.grain_offset << 9)<0) return -1; + /* the descriptor offset = 0x200 */ + if (lseek(p_fd, 0x200, SEEK_SET) == -1) + goto fail; + if (read(p_fd, p_desc, DESC_SIZE) != DESC_SIZE) + goto fail; + + if ((p_name = strstr(p_desc,"CID")) != 0) { + p_name += sizeof("CID"); + sscanf(p_name,"%x",&p_cid); + } + + real_filename = filename; + if ((temp_str = strrchr(real_filename, '\\')) != NULL) + real_filename = temp_str + 1; + if ((temp_str = strrchr(real_filename, '/')) != NULL) + real_filename = temp_str + 1; + if ((temp_str = strrchr(real_filename, ':')) != NULL) + real_filename = temp_str + 1; + + sprintf(s_desc, desc_template, p_cid, p_cid, backing_file + , (uint32_t)header.capacity, real_filename); + + /* write the descriptor */ + if (lseek(snp_fd, 0x200, SEEK_SET) == -1) + goto fail; + if (write(snp_fd, s_desc, strlen(s_desc)) == -1) + goto fail; + + gd_offset = header.gd_offset * SECTOR_SIZE; // offset of GD table + rgd_offset = header.rgd_offset * SECTOR_SIZE; // offset of RGD table + capacity = header.capacity * SECTOR_SIZE; // Extent size + /* + * Each GDE span 32M disk, means: + * 512 GTE per GT, each GTE points to grain + */ + gt_size = (int64_t)header.num_gtes_per_gte * header.granularity * SECTOR_SIZE; + if (!gt_size) + goto fail; + gde_entries = (uint32_t)(capacity / gt_size); // number of gde/rgde + gd_size = gde_entries * sizeof(uint32_t); + + /* write RGD */ + rgd_buf = qemu_malloc(gd_size); + if (!rgd_buf) + goto fail; + if (lseek(p_fd, rgd_offset, SEEK_SET) == -1) + goto fail_rgd; + if (read(p_fd, rgd_buf, gd_size) != gd_size) + goto fail_rgd; + if (lseek(snp_fd, rgd_offset, SEEK_SET) == -1) + goto fail_rgd; + if (write(snp_fd, rgd_buf, gd_size) == -1) + goto fail_rgd; + qemu_free(rgd_buf); + + /* write GD */ + gd_buf = qemu_malloc(gd_size); + if (!gd_buf) + goto fail_rgd; + if (lseek(p_fd, gd_offset, SEEK_SET) == -1) + goto fail_gd; + if (read(p_fd, gd_buf, gd_size) != gd_size) + goto fail_gd; + if (lseek(snp_fd, gd_offset, SEEK_SET) == -1) + goto fail_gd; + if (write(snp_fd, gd_buf, gd_size) == -1) + goto fail_gd; + qemu_free(gd_buf); + + close(p_fd); + close(snp_fd); + return 0; + + fail_gd: + qemu_free(gd_buf); + fail_rgd: + qemu_free(rgd_buf); + fail: + close(p_fd); + close(snp_fd); + return -1; +} + +static void vmdk_parent_close(BlockDriverState *bs) +{ + if (bs->backing_hd) + bdrv_close(bs->backing_hd); +} + +int parent_open = 0; +static int vmdk_parent_open(BlockDriverState *bs, const char * filename) +{ + BDRVVmdkState *s = bs->opaque; + char *p_name; + char desc[DESC_SIZE]; + char parent_img_name[1024]; + + /* the descriptor offset = 0x200 */ + if (bdrv_pread(s->hd, 0x200, desc, DESC_SIZE) != DESC_SIZE) + return -1; + + if ((p_name = strstr(desc,"parentFileNameHint")) != 0) { + char *end_name; + struct stat file_buf; + + p_name += sizeof("parentFileNameHint") + 1; + if ((end_name = strchr(p_name,'\"')) == 0) + return -1; + + strncpy(s->hd->backing_file, p_name, end_name - p_name); + if (stat(s->hd->backing_file, &file_buf) != 0) { + path_combine(parent_img_name, sizeof(parent_img_name), + filename, s->hd->backing_file); + } else { + strcpy(parent_img_name, s->hd->backing_file); + } + + s->hd->backing_hd = bdrv_new(""); + if (!s->hd->backing_hd) { + failure: + bdrv_close(s->hd); + return -1; + } + parent_open = 1; + if (bdrv_open(s->hd->backing_hd, parent_img_name, BDRV_O_RDONLY) < 0) + goto failure; + parent_open = 0; + } + + return 0; +} + +static int vmdk_open(BlockDriverState *bs, const char *filename, int flags) +{ + BDRVVmdkState *s = bs->opaque; + uint32_t magic; + int l1_size, i, ret; + + if (parent_open) + // Parent must be opened as RO. + flags = BDRV_O_RDONLY; + // SLG AFFLIB MOD START + //fprintf(stderr, "(VMDK) image open: flags=0x%x filename=%s\n", flags, bs->filename); + // SLG AFFLIB MOD END + + ret = bdrv_file_open(&s->hd, filename, flags); + if (ret < 0) + return ret; + if (bdrv_pread(s->hd, 0, &magic, sizeof(magic)) != sizeof(magic)) + goto fail; + + magic = be32_to_cpu(magic); + if (magic == VMDK3_MAGIC) { + VMDK3Header header; + + if (bdrv_pread(s->hd, sizeof(magic), &header, sizeof(header)) != sizeof(header)) + goto fail; + s->cluster_sectors = le32_to_cpu(header.granularity); + s->l2_size = 1 << 9; + s->l1_size = 1 << 6; + bs->total_sectors = le32_to_cpu(header.disk_sectors); + s->l1_table_offset = le32_to_cpu(header.l1dir_offset) << 9; + s->l1_backup_table_offset = 0; + s->l1_entry_sectors = s->l2_size * s->cluster_sectors; + } else if (magic == VMDK4_MAGIC) { + VMDK4Header header; + + if (bdrv_pread(s->hd, sizeof(magic), &header, sizeof(header)) != sizeof(header)) + goto fail; + bs->total_sectors = le64_to_cpu(header.capacity); + s->cluster_sectors = le64_to_cpu(header.granularity); + s->l2_size = le32_to_cpu(header.num_gtes_per_gte); + s->l1_entry_sectors = s->l2_size * s->cluster_sectors; + if (s->l1_entry_sectors <= 0) + goto fail; + s->l1_size = (bs->total_sectors + s->l1_entry_sectors - 1) + / s->l1_entry_sectors; + s->l1_table_offset = le64_to_cpu(header.rgd_offset) << 9; + s->l1_backup_table_offset = le64_to_cpu(header.gd_offset) << 9; + + if (parent_open) + s->is_parent = 1; + else + s->is_parent = 0; + + // try to open parent images, if exist + if (vmdk_parent_open(bs, filename) != 0) + goto fail; + // write the CID once after the image creation + s->parent_cid = vmdk_read_cid(bs,1); + } else { + goto fail; + } + + /* read the L1 table */ + l1_size = s->l1_size * sizeof(uint32_t); + s->l1_table = qemu_malloc(l1_size); + if (!s->l1_table) + goto fail; + if (bdrv_pread(s->hd, s->l1_table_offset, s->l1_table, l1_size) != l1_size) + goto fail; + for(i = 0; i < s->l1_size; i++) { + le32_to_cpus(&s->l1_table[i]); + } + + if (s->l1_backup_table_offset) { + s->l1_backup_table = qemu_malloc(l1_size); + if (!s->l1_backup_table) + goto fail; + if (bdrv_pread(s->hd, s->l1_backup_table_offset, s->l1_backup_table, l1_size) != l1_size) + goto fail; + for(i = 0; i < s->l1_size; i++) { + le32_to_cpus(&s->l1_backup_table[i]); + } + } + + s->l2_cache = qemu_malloc(s->l2_size * L2_CACHE_SIZE * sizeof(uint32_t)); + if (!s->l2_cache) + goto fail; + return 0; + fail: + qemu_free(s->l1_backup_table); + qemu_free(s->l1_table); + qemu_free(s->l2_cache); + bdrv_delete(s->hd); + return -1; +} + +static uint64_t get_cluster_offset(BlockDriverState *bs, VmdkMetaData *m_data, + uint64_t offset, int allocate); + +static int get_whole_cluster(BlockDriverState *bs, uint64_t cluster_offset, + uint64_t offset, int allocate) +{ + uint64_t parent_cluster_offset; + BDRVVmdkState *s = bs->opaque; + uint8_t whole_grain[s->cluster_sectors*512]; // 128 sectors * 512 bytes each = grain size 64KB + + // we will be here if it's first write on non-exist grain(cluster). + // try to read from parent image, if exist + if (s->hd->backing_hd) { + BDRVVmdkState *ps = s->hd->backing_hd->opaque; + + if (!vmdk_is_cid_valid(bs)) + return -1; + + parent_cluster_offset = get_cluster_offset(s->hd->backing_hd, NULL, offset, allocate); + + if (parent_cluster_offset) { + BDRVVmdkState *act_s = activeBDRV.hd->opaque; + + if (bdrv_pread(ps->hd, parent_cluster_offset, whole_grain, ps->cluster_sectors*512) != ps->cluster_sectors*512) + return -1; + + //Write grain only into the active image + if (bdrv_pwrite(act_s->hd, activeBDRV.cluster_offset << 9, whole_grain, sizeof(whole_grain)) != sizeof(whole_grain)) + return -1; + } + } + return 0; +} + +static int vmdk_L2update(BlockDriverState *bs, VmdkMetaData *m_data) +{ + BDRVVmdkState *s = bs->opaque; + + /* update L2 table */ + if (bdrv_pwrite(s->hd, ((int64_t)m_data->l2_offset * 512) + (m_data->l2_index * sizeof(m_data->offset)), + &(m_data->offset), sizeof(m_data->offset)) != sizeof(m_data->offset)) + return -1; + /* update backup L2 table */ + if (s->l1_backup_table_offset != 0) { + m_data->l2_offset = s->l1_backup_table[m_data->l1_index]; + if (bdrv_pwrite(s->hd, ((int64_t)m_data->l2_offset * 512) + (m_data->l2_index * sizeof(m_data->offset)), + &(m_data->offset), sizeof(m_data->offset)) != sizeof(m_data->offset)) + return -1; + } + + return 0; +} + +static uint64_t get_cluster_offset(BlockDriverState *bs, VmdkMetaData *m_data, + uint64_t offset, int allocate) +{ + BDRVVmdkState *s = bs->opaque; + unsigned int l1_index, l2_offset, l2_index; + int min_index, i, j; + uint32_t min_count, *l2_table, tmp = 0; + uint64_t cluster_offset; + + if (m_data) + m_data->valid = 0; + + l1_index = (offset >> 9) / s->l1_entry_sectors; + if (l1_index >= s->l1_size) + return 0; + l2_offset = s->l1_table[l1_index]; + if (!l2_offset) + return 0; + for(i = 0; i < L2_CACHE_SIZE; i++) { + if (l2_offset == s->l2_cache_offsets[i]) { + /* increment the hit count */ + if (++s->l2_cache_counts[i] == 0xffffffff) { + for(j = 0; j < L2_CACHE_SIZE; j++) { + s->l2_cache_counts[j] >>= 1; + } + } + l2_table = s->l2_cache + (i * s->l2_size); + goto found; + } + } + /* not found: load a new entry in the least used one */ + min_index = 0; + min_count = 0xffffffff; + for(i = 0; i < L2_CACHE_SIZE; i++) { + if (s->l2_cache_counts[i] < min_count) { + min_count = s->l2_cache_counts[i]; + min_index = i; + } + } + l2_table = s->l2_cache + (min_index * s->l2_size); + if (bdrv_pread(s->hd, (int64_t)l2_offset * 512, l2_table, s->l2_size * sizeof(uint32_t)) != + s->l2_size * sizeof(uint32_t)) + return 0; + + s->l2_cache_offsets[min_index] = l2_offset; + s->l2_cache_counts[min_index] = 1; + found: + l2_index = ((offset >> 9) / s->cluster_sectors) % s->l2_size; + cluster_offset = le32_to_cpu(l2_table[l2_index]); + + if (!cluster_offset) { + if (!allocate) + return 0; + // Avoid the L2 tables update for the images that have snapshots. + if (!s->is_parent) { + cluster_offset = bdrv_getlength(s->hd); + bdrv_truncate(s->hd, cluster_offset + (s->cluster_sectors << 9)); + + cluster_offset >>= 9; + tmp = cpu_to_le32(cluster_offset); + l2_table[l2_index] = tmp; + // Save the active image state + activeBDRV.cluster_offset = cluster_offset; + activeBDRV.hd = bs; + } + /* First of all we write grain itself, to avoid race condition + * that may to corrupt the image. + * This problem may occur because of insufficient space on host disk + * or inappropriate VM shutdown. + */ + if (get_whole_cluster(bs, cluster_offset, offset, allocate) == -1) + return 0; + + if (m_data) { + m_data->offset = tmp; + m_data->l1_index = l1_index; + m_data->l2_index = l2_index; + m_data->l2_offset = l2_offset; + m_data->valid = 1; + } + } + cluster_offset <<= 9; + return cluster_offset; +} + +static int vmdk_is_allocated(BlockDriverState *bs, int64_t sector_num, + int nb_sectors, int *pnum) +{ + BDRVVmdkState *s = bs->opaque; + int index_in_cluster, n; + uint64_t cluster_offset; + + cluster_offset = get_cluster_offset(bs, NULL, sector_num << 9, 0); + index_in_cluster = sector_num % s->cluster_sectors; + n = s->cluster_sectors - index_in_cluster; + if (n > nb_sectors) + n = nb_sectors; + *pnum = n; + return (cluster_offset != 0); +} + +static int vmdk_read(BlockDriverState *bs, int64_t sector_num, + uint8_t *buf, int nb_sectors) +{ + BDRVVmdkState *s = bs->opaque; + int index_in_cluster, n, ret; + uint64_t cluster_offset; + + while (nb_sectors > 0) { + cluster_offset = get_cluster_offset(bs, NULL, sector_num << 9, 0); + index_in_cluster = sector_num % s->cluster_sectors; + n = s->cluster_sectors - index_in_cluster; + if (n > nb_sectors) + n = nb_sectors; + if (!cluster_offset) { + // try to read from parent image, if exist + if (s->hd->backing_hd) { + if (!vmdk_is_cid_valid(bs)) + return -1; + ret = bdrv_read(s->hd->backing_hd, sector_num, buf, n); + if (ret < 0) + return -1; + } else { + memset(buf, 0, 512 * n); + } + } else { + if(bdrv_pread(s->hd, cluster_offset + index_in_cluster * 512, buf, n * 512) != n * 512) + return -1; + } + nb_sectors -= n; + sector_num += n; + buf += n * 512; + } + return 0; +} + +static int vmdk_write(BlockDriverState *bs, int64_t sector_num, + const uint8_t *buf, int nb_sectors) +{ + BDRVVmdkState *s = bs->opaque; + VmdkMetaData m_data; + int index_in_cluster, n; + uint64_t cluster_offset; + static int cid_update = 0; + + if (sector_num > bs->total_sectors) { + fprintf(stderr, + "(VMDK) Wrong offset: sector_num=0x%" PRIx64 + " total_sectors=0x%" PRIx64 "\n", + sector_num, bs->total_sectors); + return -1; + } + + while (nb_sectors > 0) { + index_in_cluster = sector_num & (s->cluster_sectors - 1); + n = s->cluster_sectors - index_in_cluster; + if (n > nb_sectors) + n = nb_sectors; + cluster_offset = get_cluster_offset(bs, &m_data, sector_num << 9, 1); + if (!cluster_offset) + return -1; + + if (bdrv_pwrite(s->hd, cluster_offset + index_in_cluster * 512, buf, n * 512) != n * 512) + return -1; + if (m_data.valid) { + /* update L2 tables */ + if (vmdk_L2update(bs, &m_data) == -1) + return -1; + } + nb_sectors -= n; + sector_num += n; + buf += n * 512; + + // update CID on the first write every time the virtual disk is opened + if (!cid_update) { + vmdk_write_cid(bs, time(NULL)); + cid_update++; + } + } + return 0; +} + +static int vmdk_create(const char *filename, int64_t total_size, + const char *backing_file, int flags) +{ + int fd, i; + VMDK4Header header; + uint32_t tmp, magic, grains, gd_size, gt_size, gt_count; + char *desc_template = + "# Disk DescriptorFile\n" + "version=1\n" + "CID=%x\n" + "parentCID=ffffffff\n" + "createType=\"monolithicSparse\"\n" + "\n" + "# Extent description\n" + "RW %lu SPARSE \"%s\"\n" + "\n" + "# The Disk Data Base \n" + "#DDB\n" + "\n" + "ddb.virtualHWVersion = \"%d\"\n" + "ddb.geometry.cylinders = \"%lu\"\n" + "ddb.geometry.heads = \"16\"\n" + "ddb.geometry.sectors = \"63\"\n" + "ddb.adapterType = \"ide\"\n"; + char desc[1024]; + const char *real_filename, *temp_str; + + /* XXX: add support for backing file */ + if (backing_file) { + return vmdk_snapshot_create(filename, backing_file); + } + + fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY | O_LARGEFILE, + 0644); + if (fd < 0) + return -1; + magic = cpu_to_be32(VMDK4_MAGIC); + memset(&header, 0, sizeof(header)); + header.version = cpu_to_le32(1); + header.flags = cpu_to_le32(3); /* ?? */ + header.capacity = cpu_to_le64(total_size); + header.granularity = cpu_to_le64(128); + header.num_gtes_per_gte = cpu_to_le32(512); + + grains = (total_size + header.granularity - 1) / header.granularity; + gt_size = ((header.num_gtes_per_gte * sizeof(uint32_t)) + 511) >> 9; + gt_count = (grains + header.num_gtes_per_gte - 1) / header.num_gtes_per_gte; + gd_size = (gt_count * sizeof(uint32_t) + 511) >> 9; + + header.desc_offset = 1; + header.desc_size = 20; + header.rgd_offset = header.desc_offset + header.desc_size; + header.gd_offset = header.rgd_offset + gd_size + (gt_size * gt_count); + header.grain_offset = + ((header.gd_offset + gd_size + (gt_size * gt_count) + + header.granularity - 1) / header.granularity) * + header.granularity; + + header.desc_offset = cpu_to_le64(header.desc_offset); + header.desc_size = cpu_to_le64(header.desc_size); + header.rgd_offset = cpu_to_le64(header.rgd_offset); + header.gd_offset = cpu_to_le64(header.gd_offset); + header.grain_offset = cpu_to_le64(header.grain_offset); + + header.check_bytes[0] = 0xa; + header.check_bytes[1] = 0x20; + header.check_bytes[2] = 0xd; + header.check_bytes[3] = 0xa; + + /* write all the data */ + if(write(fd, &magic, sizeof(magic))!=sizeof(magic)) return -1; + if(write(fd, &header, sizeof(header))!=sizeof(header)) return -1; + + if(ftruncate(fd, header.grain_offset << 9)<0) return -1; + + /* write grain directory */ + lseek(fd, le64_to_cpu(header.rgd_offset) << 9, SEEK_SET); + for (i = 0, tmp = header.rgd_offset + gd_size; + i < gt_count; i++, tmp += gt_size) + if(write(fd, &tmp, sizeof(tmp))!=sizeof(tmp)) return -1; + + /* write backup grain directory */ + lseek(fd, le64_to_cpu(header.gd_offset) << 9, SEEK_SET); + for (i = 0, tmp = header.gd_offset + gd_size; + i < gt_count; i++, tmp += gt_size) + if(write(fd, &tmp, sizeof(tmp))!=sizeof(tmp)) return -1; + + /* compose the descriptor */ + real_filename = filename; + if ((temp_str = strrchr(real_filename, '\\')) != NULL) + real_filename = temp_str + 1; + if ((temp_str = strrchr(real_filename, '/')) != NULL) + real_filename = temp_str + 1; + if ((temp_str = strrchr(real_filename, ':')) != NULL) + real_filename = temp_str + 1; + sprintf(desc, desc_template, time(NULL), (unsigned long)total_size, + real_filename, (flags & BLOCK_FLAG_COMPAT6 ? 6 : 4), total_size / (63 * 16)); + + /* write the descriptor */ + lseek(fd, le64_to_cpu(header.desc_offset) << 9, SEEK_SET); + if(write(fd, desc, strlen(desc))!=strlen(desc)) return -1; + + close(fd); + return 0; +} + +static void vmdk_close(BlockDriverState *bs) +{ + BDRVVmdkState *s = bs->opaque; + + qemu_free(s->l1_table); + qemu_free(s->l2_cache); + bdrv_delete(s->hd); + // try to close parent image, if exist + vmdk_parent_close(s->hd); +} + +static void vmdk_flush(BlockDriverState *bs) +{ + BDRVVmdkState *s = bs->opaque; + bdrv_flush(s->hd); +} + +BlockDriver bdrv_vmdk = { + "vmdk", + sizeof(BDRVVmdkState), + vmdk_probe, + vmdk_open, + vmdk_read, + vmdk_write, + vmdk_close, + vmdk_create, + vmdk_flush, + vmdk_is_allocated, +}; diff --git a/lib/qemu/block-vpc.c b/lib/qemu/block-vpc.c new file mode 100644 index 0000000..f76c451 --- /dev/null +++ b/lib/qemu/block-vpc.c @@ -0,0 +1,239 @@ +/* + * Block driver for Conectix/Microsoft Virtual PC images + * + * Copyright (c) 2005 Alex Beregszaszi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include "qemu-common.h" +#include "block_int.h" + +/**************************************************************/ + +#define HEADER_SIZE 512 + +//#define CACHE + +// always big-endian +struct vpc_subheader { + char magic[8]; // "conectix" / "cxsparse" + union { + struct { + uint32_t unk1[2]; + uint32_t unk2; // always zero? + uint32_t subheader_offset; + uint32_t unk3; // some size? + char creator[4]; // "vpc " + uint16_t major; + uint16_t minor; + char guest[4]; // "Wi2k" + uint32_t unk4[7]; + uint8_t vnet_id[16]; // virtual network id, purpose unknown + // next 16 longs are used, but dunno the purpose + // next 6 longs unknown, following 7 long maybe a serial + char padding[HEADER_SIZE - 84]; + } main; + struct { + uint32_t unk1[2]; // all bits set + uint32_t unk2; // always zero? + uint32_t pagetable_offset; + uint32_t unk3; + uint32_t pagetable_entries; // 32bit/entry + uint32_t pageentry_size; // 512*8*512 + uint32_t nb_sectors; + char padding[HEADER_SIZE - 40]; + } sparse; + char padding[HEADER_SIZE - 8]; + } type; +}; + +typedef struct BDRVVPCState { + int fd; + + int pagetable_entries; + uint32_t *pagetable; + + uint32_t pageentry_size; +#ifdef CACHE + uint8_t *pageentry_u8; + uint32_t *pageentry_u32; + uint16_t *pageentry_u16; + + uint64_t last_bitmap; +#endif +} BDRVVPCState; + +static int vpc_probe(const uint8_t *buf, int buf_size, const char *filename) +{ + if (buf_size >= 8 && !strncmp((char *)buf, "conectix", 8)) + return 100; + return 0; +} + +static int vpc_open(BlockDriverState *bs, const char *filename, int flags) +{ + BDRVVPCState *s = bs->opaque; + int fd, i; + struct vpc_subheader header; + + fd = open(filename, O_RDONLY | O_BINARY); + if (fd < 0) + return -1; + + bs->read_only = 1; // no write support yet + + s->fd = fd; + + if (read(fd, &header, HEADER_SIZE) != HEADER_SIZE) + goto fail; + + if (strncmp(header.magic, "conectix", 8)) + goto fail; + lseek(s->fd, be32_to_cpu(header.type.main.subheader_offset), SEEK_SET); + + if (read(fd, &header, HEADER_SIZE) != HEADER_SIZE) + goto fail; + + if (strncmp(header.magic, "cxsparse", 8)) + goto fail; + + bs->total_sectors = ((uint64_t)be32_to_cpu(header.type.sparse.pagetable_entries) * + be32_to_cpu(header.type.sparse.pageentry_size)) / 512; + + lseek(s->fd, be32_to_cpu(header.type.sparse.pagetable_offset), SEEK_SET); + + s->pagetable_entries = be32_to_cpu(header.type.sparse.pagetable_entries); + s->pagetable = qemu_malloc(s->pagetable_entries * 4); + if (!s->pagetable) + goto fail; + if (read(s->fd, s->pagetable, s->pagetable_entries * 4) != + s->pagetable_entries * 4) + goto fail; + for (i = 0; i < s->pagetable_entries; i++) + be32_to_cpus(&s->pagetable[i]); + + s->pageentry_size = be32_to_cpu(header.type.sparse.pageentry_size); +#ifdef CACHE + s->pageentry_u8 = qemu_malloc(512); + if (!s->pageentry_u8) + goto fail; + s->pageentry_u32 = s->pageentry_u8; + s->pageentry_u16 = s->pageentry_u8; + s->last_pagetable = -1; +#endif + + return 0; + fail: + close(fd); + return -1; +} + +static inline int seek_to_sector(BlockDriverState *bs, int64_t sector_num) +{ + BDRVVPCState *s = bs->opaque; + uint64_t offset = sector_num * 512; + uint64_t bitmap_offset, block_offset; + uint32_t pagetable_index, pageentry_index; + + pagetable_index = offset / s->pageentry_size; + pageentry_index = (offset % s->pageentry_size) / 512; + + if (pagetable_index > s->pagetable_entries || s->pagetable[pagetable_index] == 0xffffffff) + return -1; // not allocated + + bitmap_offset = 512 * s->pagetable[pagetable_index]; + block_offset = bitmap_offset + 512 + (512 * pageentry_index); + +// printf("sector: %" PRIx64 ", index: %x, offset: %x, bioff: %" PRIx64 ", bloff: %" PRIx64 "\n", +// sector_num, pagetable_index, pageentry_index, +// bitmap_offset, block_offset); + +// disabled by reason +#if 0 +#ifdef CACHE + if (bitmap_offset != s->last_bitmap) + { + lseek(s->fd, bitmap_offset, SEEK_SET); + + s->last_bitmap = bitmap_offset; + + // Scary! Bitmap is stored as big endian 32bit entries, + // while we used to look it up byte by byte + read(s->fd, s->pageentry_u8, 512); + for (i = 0; i < 128; i++) + be32_to_cpus(&s->pageentry_u32[i]); + } + + if ((s->pageentry_u8[pageentry_index / 8] >> (pageentry_index % 8)) & 1) + return -1; +#else + lseek(s->fd, bitmap_offset + (pageentry_index / 8), SEEK_SET); + + read(s->fd, &bitmap_entry, 1); + + if ((bitmap_entry >> (pageentry_index % 8)) & 1) + return -1; // not allocated +#endif +#endif + lseek(s->fd, block_offset, SEEK_SET); + + return 0; +} + +static int vpc_read(BlockDriverState *bs, int64_t sector_num, + uint8_t *buf, int nb_sectors) +{ + BDRVVPCState *s = bs->opaque; + int ret; + + while (nb_sectors > 0) { + if (!seek_to_sector(bs, sector_num)) + { + ret = read(s->fd, buf, 512); + if (ret != 512) + return -1; + } + else + memset(buf, 0, 512); + nb_sectors--; + sector_num++; + buf += 512; + } + return 0; +} + +static void vpc_close(BlockDriverState *bs) +{ + BDRVVPCState *s = bs->opaque; + qemu_free(s->pagetable); +#ifdef CACHE + qemu_free(s->pageentry_u8); +#endif + close(s->fd); +} + +BlockDriver bdrv_vpc = { + "vpc", + sizeof(BDRVVPCState), + vpc_probe, + vpc_open, + vpc_read, + NULL, + vpc_close, +}; diff --git a/lib/qemu/block-vvfat.c b/lib/qemu/block-vvfat.c new file mode 100644 index 0000000..d33b5a1 --- /dev/null +++ b/lib/qemu/block-vvfat.c @@ -0,0 +1,2843 @@ +/* vim:set shiftwidth=4 ts=8: */ +/* + * QEMU Block driver for virtual VFAT (shadows a local directory) + * + * Copyright (c) 2004,2005 Johannes E. Schindelin + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include +#include +#include +#include "qemu-common.h" +#include "block_int.h" + +#ifndef S_IWGRP +#define S_IWGRP 0 +#endif +#ifndef S_IWOTH +#define S_IWOTH 0 +#endif + +/* TODO: add ":bootsector=blabla.img:" */ +/* LATER TODO: add automatic boot sector generation from + BOOTEASY.ASM and Ranish Partition Manager + Note that DOS assumes the system files to be the first files in the + file system (test if the boot sector still relies on that fact)! */ +/* MAYBE TODO: write block-visofs.c */ +/* TODO: call try_commit() only after a timeout */ + +/* #define DEBUG */ + +#ifdef DEBUG + +#define DLOG(a) a + +#undef stderr +#define stderr STDERR +FILE* stderr = NULL; + +static void checkpoint(); + +#ifdef __MINGW32__ +void nonono(const char* file, int line, const char* msg) { + fprintf(stderr, "Nonono! %s:%d %s\n", file, line, msg); + exit(-5); +} +#undef assert +#define assert(a) do {if (!(a)) nonono(__FILE__, __LINE__, #a);}while(0) +#endif + +#else + +#define DLOG(a) + +#endif + +/* dynamic array functions */ +typedef struct array_t { + char* pointer; + unsigned int size,next,item_size; +} array_t; + +static inline void array_init(array_t* array,unsigned int item_size) +{ + array->pointer=0; + array->size=0; + array->next=0; + array->item_size=item_size; +} + +static inline void array_free(array_t* array) +{ + if(array->pointer) + free(array->pointer); + array->size=array->next=0; +} + +/* does not automatically grow */ +static inline void* array_get(array_t* array,unsigned int index) { + assert(index >= 0); + assert(index < array->next); + return array->pointer + index * array->item_size; +} + +static inline int array_ensure_allocated(array_t* array, int index) +{ + if((index + 1) * array->item_size > array->size) { + int new_size = (index + 32) * array->item_size; + array->pointer = realloc(array->pointer, new_size); + if (!array->pointer) + return -1; + array->size = new_size; + array->next = index + 1; + } + + return 0; +} + +static inline void* array_get_next(array_t* array) { + unsigned int next = array->next; + void* result; + + if (array_ensure_allocated(array, next) < 0) + return NULL; + + array->next = next + 1; + result = array_get(array, next); + + return result; +} + +static inline void* array_insert(array_t* array,unsigned int index,unsigned int count) { + if((array->next+count)*array->item_size>array->size) { + int increment=count*array->item_size; + array->pointer=realloc(array->pointer,array->size+increment); + if(!array->pointer) + return 0; + array->size+=increment; + } + memmove(array->pointer+(index+count)*array->item_size, + array->pointer+index*array->item_size, + (array->next-index)*array->item_size); + array->next+=count; + return array->pointer+index*array->item_size; +} + +/* this performs a "roll", so that the element which was at index_from becomes + * index_to, but the order of all other elements is preserved. */ +static inline int array_roll(array_t* array,int index_to,int index_from,int count) +{ + char* buf; + char* from; + char* to; + int is; + + if(!array || + index_to<0 || index_to>=array->next || + index_from<0 || index_from>=array->next) + return -1; + + if(index_to==index_from) + return 0; + + is=array->item_size; + from=array->pointer+index_from*is; + to=array->pointer+index_to*is; + buf=malloc(is*count); + memcpy(buf,from,is*count); + + if(index_to=0); + assert(count > 0); + assert(index + count <= array->next); + if(array_roll(array,array->next-1,index,count)) + return -1; + array->next -= count; + return 0; +} + +static int array_remove(array_t* array,int index) +{ + return array_remove_slice(array, index, 1); +} + +/* return the index for a given member */ +static int array_index(array_t* array, void* pointer) +{ + size_t offset = (char*)pointer - array->pointer; + assert(offset >= 0); + assert((offset % array->item_size) == 0); + assert(offset/array->item_size < array->next); + return offset/array->item_size; +} + +/* These structures are used to fake a disk and the VFAT filesystem. + * For this reason we need to use __attribute__((packed)). */ + +typedef struct bootsector_t { + uint8_t jump[3]; + uint8_t name[8]; + uint16_t sector_size; + uint8_t sectors_per_cluster; + uint16_t reserved_sectors; + uint8_t number_of_fats; + uint16_t root_entries; + uint16_t total_sectors16; + uint8_t media_type; + uint16_t sectors_per_fat; + uint16_t sectors_per_track; + uint16_t number_of_heads; + uint32_t hidden_sectors; + uint32_t total_sectors; + union { + struct { + uint8_t drive_number; + uint8_t current_head; + uint8_t signature; + uint32_t id; + uint8_t volume_label[11]; + } __attribute__((packed)) fat16; + struct { + uint32_t sectors_per_fat; + uint16_t flags; + uint8_t major,minor; + uint32_t first_cluster_of_root_directory; + uint16_t info_sector; + uint16_t backup_boot_sector; + uint16_t ignored; + } __attribute__((packed)) fat32; + } u; + uint8_t fat_type[8]; + uint8_t ignored[0x1c0]; + uint8_t magic[2]; +} __attribute__((packed)) bootsector_t; + +typedef struct { + uint8_t head; + uint8_t sector; + uint8_t cylinder; +} mbr_chs_t; + +typedef struct partition_t { + uint8_t attributes; /* 0x80 = bootable */ + mbr_chs_t start_CHS; + uint8_t fs_type; /* 0x1 = FAT12, 0x6 = FAT16, 0xe = FAT16_LBA, 0xb = FAT32, 0xc = FAT32_LBA */ + mbr_chs_t end_CHS; + uint32_t start_sector_long; + uint32_t length_sector_long; +} __attribute__((packed)) partition_t; + +typedef struct mbr_t { + uint8_t ignored[0x1b8]; + uint32_t nt_id; + uint8_t ignored2[2]; + partition_t partition[4]; + uint8_t magic[2]; +} __attribute__((packed)) mbr_t; + +typedef struct direntry_t { + uint8_t name[8]; + uint8_t extension[3]; + uint8_t attributes; + uint8_t reserved[2]; + uint16_t ctime; + uint16_t cdate; + uint16_t adate; + uint16_t begin_hi; + uint16_t mtime; + uint16_t mdate; + uint16_t begin; + uint32_t size; +} __attribute__((packed)) direntry_t; + +/* this structure are used to transparently access the files */ + +typedef struct mapping_t { + /* begin is the first cluster, end is the last+1 */ + uint32_t begin,end; + /* as s->directory is growable, no pointer may be used here */ + unsigned int dir_index; + /* the clusters of a file may be in any order; this points to the first */ + int first_mapping_index; + union { + /* offset is + * - the offset in the file (in clusters) for a file, or + * - the next cluster of the directory for a directory, and + * - the address of the buffer for a faked entry + */ + struct { + uint32_t offset; + } file; + struct { + int parent_mapping_index; + int first_dir_index; + } dir; + } info; + /* path contains the full path, i.e. it always starts with s->path */ + char* path; + + enum { MODE_UNDEFINED = 0, MODE_NORMAL = 1, MODE_MODIFIED = 2, + MODE_DIRECTORY = 4, MODE_FAKED = 8, + MODE_DELETED = 16, MODE_RENAMED = 32 } mode; + int read_only; +} mapping_t; + +#ifdef DEBUG +static void print_direntry(const struct direntry_t*); +static void print_mapping(const struct mapping_t* mapping); +#endif + +/* here begins the real VVFAT driver */ + +typedef struct BDRVVVFATState { + BlockDriverState* bs; /* pointer to parent */ + unsigned int first_sectors_number; /* 1 for a single partition, 0x40 for a disk with partition table */ + unsigned char first_sectors[0x40*0x200]; + + int fat_type; /* 16 or 32 */ + array_t fat,directory,mapping; + + unsigned int cluster_size; + unsigned int sectors_per_cluster; + unsigned int sectors_per_fat; + unsigned int sectors_of_root_directory; + uint32_t last_cluster_of_root_directory; + unsigned int faked_sectors; /* how many sectors are faked before file data */ + uint32_t sector_count; /* total number of sectors of the partition */ + uint32_t cluster_count; /* total number of clusters of this partition */ + uint32_t max_fat_value; + + int current_fd; + mapping_t* current_mapping; + unsigned char* cluster; /* points to current cluster */ + unsigned char* cluster_buffer; /* points to a buffer to hold temp data */ + unsigned int current_cluster; + + /* write support */ + BlockDriverState* write_target; + char* qcow_filename; + BlockDriverState* qcow; + void* fat2; + char* used_clusters; + array_t commits; + const char* path; + int downcase_short_names; +} BDRVVVFATState; + +/* take the sector position spos and convert it to Cylinder/Head/Sector position + * if the position is outside the specified geometry, fill maximum value for CHS + * and return 1 to signal overflow. + */ +static int sector2CHS(BlockDriverState* bs, mbr_chs_t * chs, int spos){ + int head,sector; + sector = spos % (bs->secs); spos/= bs->secs; + head = spos % (bs->heads); spos/= bs->heads; + if(spos >= bs->cyls){ + /* Overflow, + it happens if 32bit sector positions are used, while CHS is only 24bit. + Windows/Dos is said to take 1023/255/63 as nonrepresentable CHS */ + chs->head = 0xFF; + chs->sector = 0xFF; + chs->cylinder = 0xFF; + return 1; + } + chs->head = (uint8_t)head; + chs->sector = (uint8_t)( (sector+1) | ((spos>>8)<<6) ); + chs->cylinder = (uint8_t)spos; + return 0; +} + +static void init_mbr(BDRVVVFATState* s) +{ + /* TODO: if the files mbr.img and bootsect.img exist, use them */ + mbr_t* real_mbr=(mbr_t*)s->first_sectors; + partition_t* partition=&(real_mbr->partition[0]); + int lba; + + memset(s->first_sectors,0,512); + + /* Win NT Disk Signature */ + real_mbr->nt_id= cpu_to_le32(0xbe1afdfa); + + partition->attributes=0x80; /* bootable */ + + /* LBA is used when partition is outside the CHS geometry */ + lba = sector2CHS(s->bs, &partition->start_CHS, s->first_sectors_number-1); + lba|= sector2CHS(s->bs, &partition->end_CHS, s->sector_count); + + /*LBA partitions are identified only by start/length_sector_long not by CHS*/ + partition->start_sector_long =cpu_to_le32(s->first_sectors_number-1); + partition->length_sector_long=cpu_to_le32(s->sector_count - s->first_sectors_number+1); + + /* FAT12/FAT16/FAT32 */ + /* DOS uses different types when partition is LBA, + probably to prevent older versions from using CHS on them */ + partition->fs_type= s->fat_type==12 ? 0x1: + s->fat_type==16 ? (lba?0xe:0x06): + /*fat_tyoe==32*/ (lba?0xc:0x0b); + + real_mbr->magic[0]=0x55; real_mbr->magic[1]=0xaa; +} + +/* direntry functions */ + +/* dest is assumed to hold 258 bytes, and pads with 0xffff up to next multiple of 26 */ +static inline int short2long_name(char* dest,const char* src) +{ + int i; + int len; + for(i=0;i<129 && src[i];i++) { + dest[2*i]=src[i]; + dest[2*i+1]=0; + } + len=2*i; + dest[2*i]=dest[2*i+1]=0; + for(i=2*i+2;(i%26);i++) + dest[i]=0xff; + return len; +} + +static inline direntry_t* create_long_filename(BDRVVVFATState* s,const char* filename) +{ + char buffer[258]; + int length=short2long_name(buffer,filename), + number_of_entries=(length+25)/26,i; + direntry_t* entry; + + for(i=0;idirectory)); + entry->attributes=0xf; + entry->reserved[0]=0; + entry->begin=0; + entry->name[0]=(number_of_entries-i)|(i==0?0x40:0); + } + for(i=0;i<26*number_of_entries;i++) { + int offset=(i%26); + if(offset<10) offset=1+offset; + else if(offset<22) offset=14+offset-10; + else offset=28+offset-22; + entry=array_get(&(s->directory),s->directory.next-1-(i/26)); + entry->name[offset]=buffer[i]; + } + return array_get(&(s->directory),s->directory.next-number_of_entries); +} + +static char is_free(const direntry_t* direntry) +{ + /* return direntry->name[0]==0 ; */ + return direntry->attributes == 0 || direntry->name[0]==0xe5; +} + +static char is_volume_label(const direntry_t* direntry) +{ + return direntry->attributes == 0x28; +} + +static char is_long_name(const direntry_t* direntry) +{ + return direntry->attributes == 0xf; +} + +static char is_short_name(const direntry_t* direntry) +{ + return !is_volume_label(direntry) && !is_long_name(direntry) + && !is_free(direntry); +} + +static char is_directory(const direntry_t* direntry) +{ + return direntry->attributes & 0x10 && direntry->name[0] != 0xe5; +} + +static inline char is_dot(const direntry_t* direntry) +{ + return is_short_name(direntry) && direntry->name[0] == '.'; +} + +static char is_file(const direntry_t* direntry) +{ + return is_short_name(direntry) && !is_directory(direntry); +} + +static inline uint32_t begin_of_direntry(const direntry_t* direntry) +{ + return le16_to_cpu(direntry->begin)|(le16_to_cpu(direntry->begin_hi)<<16); +} + +static inline uint32_t filesize_of_direntry(const direntry_t* direntry) +{ + return le32_to_cpu(direntry->size); +} + +static void set_begin_of_direntry(direntry_t* direntry, uint32_t begin) +{ + direntry->begin = cpu_to_le16(begin & 0xffff); + direntry->begin_hi = cpu_to_le16((begin >> 16) & 0xffff); +} + +/* fat functions */ + +static inline uint8_t fat_chksum(const direntry_t* entry) +{ + uint8_t chksum=0; + int i; + + for(i=0;i<11;i++) + chksum=(((chksum&0xfe)>>1)|((chksum&0x01)?0x80:0)) + +(unsigned char)entry->name[i]; + + return chksum; +} + +/* if return_time==0, this returns the fat_date, else the fat_time */ +static uint16_t fat_datetime(time_t time,int return_time) { + struct tm* t; +#ifdef _WIN32 + t=localtime(&time); /* this is not thread safe */ +#else + struct tm t1; + t=&t1; + localtime_r(&time,t); +#endif + if(return_time) + return cpu_to_le16((t->tm_sec/2)|(t->tm_min<<5)|(t->tm_hour<<11)); + return cpu_to_le16((t->tm_mday)|((t->tm_mon+1)<<5)|((t->tm_year-80)<<9)); +} + +static inline void fat_set(BDRVVVFATState* s,unsigned int cluster,uint32_t value) +{ + if(s->fat_type==32) { + uint32_t* entry=array_get(&(s->fat),cluster); + *entry=cpu_to_le32(value); + } else if(s->fat_type==16) { + uint16_t* entry=array_get(&(s->fat),cluster); + *entry=cpu_to_le16(value&0xffff); + } else { + int offset = (cluster*3/2); + unsigned char* p = array_get(&(s->fat), offset); + switch (cluster&1) { + case 0: + p[0] = value&0xff; + p[1] = (p[1]&0xf0) | ((value>>8)&0xf); + break; + case 1: + p[0] = (p[0]&0xf) | ((value&0xf)<<4); + p[1] = (value>>4); + break; + } + } +} + +static inline uint32_t fat_get(BDRVVVFATState* s,unsigned int cluster) +{ + if(s->fat_type==32) { + uint32_t* entry=array_get(&(s->fat),cluster); + return le32_to_cpu(*entry); + } else if(s->fat_type==16) { + uint16_t* entry=array_get(&(s->fat),cluster); + return le16_to_cpu(*entry); + } else { + const uint8_t* x=(uint8_t*)(s->fat.pointer)+cluster*3/2; + return ((x[0]|(x[1]<<8))>>(cluster&1?4:0))&0x0fff; + } +} + +static inline int fat_eof(BDRVVVFATState* s,uint32_t fat_entry) +{ + if(fat_entry>s->max_fat_value-8) + return -1; + return 0; +} + +static inline void init_fat(BDRVVVFATState* s) +{ + if (s->fat_type == 12) { + array_init(&(s->fat),1); + array_ensure_allocated(&(s->fat), + s->sectors_per_fat * 0x200 * 3 / 2 - 1); + } else { + array_init(&(s->fat),(s->fat_type==32?4:2)); + array_ensure_allocated(&(s->fat), + s->sectors_per_fat * 0x200 / s->fat.item_size - 1); + } + memset(s->fat.pointer,0,s->fat.size); + + switch(s->fat_type) { + case 12: s->max_fat_value=0xfff; break; + case 16: s->max_fat_value=0xffff; break; + case 32: s->max_fat_value=0x0fffffff; break; + default: s->max_fat_value=0; /* error... */ + } + +} + +/* TODO: in create_short_filename, 0xe5->0x05 is not yet handled! */ +/* TODO: in parse_short_filename, 0x05->0xe5 is not yet handled! */ +static inline direntry_t* create_short_and_long_name(BDRVVVFATState* s, + unsigned int directory_start, const char* filename, int is_dot) +{ + int i,j,long_index=s->directory.next; + direntry_t* entry=0; + direntry_t* entry_long=0; + + if(is_dot) { + entry=array_get_next(&(s->directory)); + memset(entry->name,0x20,11); + memcpy(entry->name,filename,strlen(filename)); + return entry; + } + + entry_long=create_long_filename(s,filename); + + i = strlen(filename); + for(j = i - 1; j>0 && filename[j]!='.';j--); + if (j > 0) + i = (j > 8 ? 8 : j); + else if (i > 8) + i = 8; + + entry=array_get_next(&(s->directory)); + memset(entry->name,0x20,11); + strncpy((char*)entry->name,filename,i); + + if(j > 0) + for (i = 0; i < 3 && filename[j+1+i]; i++) + entry->extension[i] = filename[j+1+i]; + + /* upcase & remove unwanted characters */ + for(i=10;i>=0;i--) { + if(i==10 || i==7) for(;i>0 && entry->name[i]==' ';i--); + if(entry->name[i]<=' ' || entry->name[i]>0x7f + || strchr(".*?<>|\":/\\[];,+='",entry->name[i])) + entry->name[i]='_'; + else if(entry->name[i]>='a' && entry->name[i]<='z') + entry->name[i]+='A'-'a'; + } + + /* mangle duplicates */ + while(1) { + direntry_t* entry1=array_get(&(s->directory),directory_start); + int j; + + for(;entry1name,entry->name,11)) + break; /* found dupe */ + if(entry1==entry) /* no dupe found */ + break; + + /* use all 8 characters of name */ + if(entry->name[7]==' ') { + int j; + for(j=6;j>0 && entry->name[j]==' ';j--) + entry->name[j]='~'; + } + + /* increment number */ + for(j=7;j>0 && entry->name[j]=='9';j--) + entry->name[j]='0'; + if(j>0) { + if(entry->name[j]<'0' || entry->name[j]>'9') + entry->name[j]='0'; + else + entry->name[j]++; + } + } + + /* calculate checksum; propagate to long name */ + if(entry_long) { + uint8_t chksum=fat_chksum(entry); + + /* calculate anew, because realloc could have taken place */ + entry_long=array_get(&(s->directory),long_index); + while(entry_longreserved[1]=chksum; + entry_long++; + } + } + + return entry; +} + +/* + * Read a directory. (the index of the corresponding mapping must be passed). + */ +static int read_directory(BDRVVVFATState* s, int mapping_index) +{ + mapping_t* mapping = array_get(&(s->mapping), mapping_index); + direntry_t* direntry; + const char* dirname = mapping->path; + int first_cluster = mapping->begin; + int parent_index = mapping->info.dir.parent_mapping_index; + mapping_t* parent_mapping = (mapping_t*) + (parent_index >= 0 ? array_get(&(s->mapping), parent_index) : 0); + int first_cluster_of_parent = parent_mapping ? parent_mapping->begin : -1; + + DIR* dir=opendir(dirname); + struct dirent* entry; + int i; + + assert(mapping->mode & MODE_DIRECTORY); + + if(!dir) { + mapping->end = mapping->begin; + return -1; + } + + i = mapping->info.dir.first_dir_index = + first_cluster == 0 ? 0 : s->directory.next; + + /* actually read the directory, and allocate the mappings */ + while((entry=readdir(dir))) { + unsigned int length=strlen(dirname)+2+strlen(entry->d_name); + char* buffer; + direntry_t* direntry; + struct stat st; + int is_dot=!strcmp(entry->d_name,"."); + int is_dotdot=!strcmp(entry->d_name,".."); + + if(first_cluster == 0 && (is_dotdot || is_dot)) + continue; + + buffer=(char*)malloc(length); + assert(buffer); + snprintf(buffer,length,"%s/%s",dirname,entry->d_name); + + if(stat(buffer,&st)<0) { + free(buffer); + continue; + } + + /* create directory entry for this file */ + direntry=create_short_and_long_name(s, i, entry->d_name, + is_dot || is_dotdot); + direntry->attributes=(S_ISDIR(st.st_mode)?0x10:0x20); + direntry->reserved[0]=direntry->reserved[1]=0; + direntry->ctime=fat_datetime(st.st_ctime,1); + direntry->cdate=fat_datetime(st.st_ctime,0); + direntry->adate=fat_datetime(st.st_atime,0); + direntry->begin_hi=0; + direntry->mtime=fat_datetime(st.st_mtime,1); + direntry->mdate=fat_datetime(st.st_mtime,0); + if(is_dotdot) + set_begin_of_direntry(direntry, first_cluster_of_parent); + else if(is_dot) + set_begin_of_direntry(direntry, first_cluster); + else + direntry->begin=0; /* do that later */ + if (st.st_size > 0x7fffffff) { + fprintf(stderr, "File %s is larger than 2GB\n", buffer); + free(buffer); + return -2; + } + direntry->size=cpu_to_le32(S_ISDIR(st.st_mode)?0:st.st_size); + + /* create mapping for this file */ + if(!is_dot && !is_dotdot && (S_ISDIR(st.st_mode) || st.st_size)) { + s->current_mapping=(mapping_t*)array_get_next(&(s->mapping)); + s->current_mapping->begin=0; + s->current_mapping->end=st.st_size; + /* + * we get the direntry of the most recent direntry, which + * contains the short name and all the relevant information. + */ + s->current_mapping->dir_index=s->directory.next-1; + s->current_mapping->first_mapping_index = -1; + if (S_ISDIR(st.st_mode)) { + s->current_mapping->mode = MODE_DIRECTORY; + s->current_mapping->info.dir.parent_mapping_index = + mapping_index; + } else { + s->current_mapping->mode = MODE_UNDEFINED; + s->current_mapping->info.file.offset = 0; + } + s->current_mapping->path=buffer; + s->current_mapping->read_only = + (st.st_mode & (S_IWUSR | S_IWGRP | S_IWOTH)) == 0; + } + } + closedir(dir); + + /* fill with zeroes up to the end of the cluster */ + while(s->directory.next%(0x10*s->sectors_per_cluster)) { + direntry_t* direntry=array_get_next(&(s->directory)); + memset(direntry,0,sizeof(direntry_t)); + } + +/* TODO: if there are more entries, bootsector has to be adjusted! */ +#define ROOT_ENTRIES (0x02 * 0x10 * s->sectors_per_cluster) + if (mapping_index == 0 && s->directory.next < ROOT_ENTRIES) { + /* root directory */ + int cur = s->directory.next; + array_ensure_allocated(&(s->directory), ROOT_ENTRIES - 1); + memset(array_get(&(s->directory), cur), 0, + (ROOT_ENTRIES - cur) * sizeof(direntry_t)); + } + + /* reget the mapping, since s->mapping was possibly realloc()ed */ + mapping = (mapping_t*)array_get(&(s->mapping), mapping_index); + first_cluster += (s->directory.next - mapping->info.dir.first_dir_index) + * 0x20 / s->cluster_size; + mapping->end = first_cluster; + + direntry = (direntry_t*)array_get(&(s->directory), mapping->dir_index); + set_begin_of_direntry(direntry, mapping->begin); + + return 0; +} + +static inline uint32_t sector2cluster(BDRVVVFATState* s,off_t sector_num) +{ + return (sector_num-s->faked_sectors)/s->sectors_per_cluster; +} + +static inline off_t cluster2sector(BDRVVVFATState* s, uint32_t cluster_num) +{ + return s->faked_sectors + s->sectors_per_cluster * cluster_num; +} + +static inline uint32_t sector_offset_in_cluster(BDRVVVFATState* s,off_t sector_num) +{ + return (sector_num-s->first_sectors_number-2*s->sectors_per_fat)%s->sectors_per_cluster; +} + +#ifdef DBG +static direntry_t* get_direntry_for_mapping(BDRVVVFATState* s,mapping_t* mapping) +{ + if(mapping->mode==MODE_UNDEFINED) + return 0; + return (direntry_t*)(s->directory.pointer+sizeof(direntry_t)*mapping->dir_index); +} +#endif + +static int init_directories(BDRVVVFATState* s, + const char* dirname) +{ + bootsector_t* bootsector; + mapping_t* mapping; + unsigned int i; + unsigned int cluster; + + memset(&(s->first_sectors[0]),0,0x40*0x200); + + s->cluster_size=s->sectors_per_cluster*0x200; + s->cluster_buffer=malloc(s->cluster_size); + assert(s->cluster_buffer); + + /* + * The formula: sc = spf+1+spf*spc*(512*8/fat_type), + * where sc is sector_count, + * spf is sectors_per_fat, + * spc is sectors_per_clusters, and + * fat_type = 12, 16 or 32. + */ + i = 1+s->sectors_per_cluster*0x200*8/s->fat_type; + s->sectors_per_fat=(s->sector_count+i)/i; /* round up */ + + array_init(&(s->mapping),sizeof(mapping_t)); + array_init(&(s->directory),sizeof(direntry_t)); + + /* add volume label */ + { + direntry_t* entry=array_get_next(&(s->directory)); + entry->attributes=0x28; /* archive | volume label */ + snprintf((char*)entry->name,11,"QEMU VVFAT"); + } + + /* Now build FAT, and write back information into directory */ + init_fat(s); + + s->faked_sectors=s->first_sectors_number+s->sectors_per_fat*2; + s->cluster_count=sector2cluster(s, s->sector_count); + + mapping = array_get_next(&(s->mapping)); + mapping->begin = 0; + mapping->dir_index = 0; + mapping->info.dir.parent_mapping_index = -1; + mapping->first_mapping_index = -1; + mapping->path = strdup(dirname); + i = strlen(mapping->path); + if (i > 0 && mapping->path[i - 1] == '/') + mapping->path[i - 1] = '\0'; + mapping->mode = MODE_DIRECTORY; + mapping->read_only = 0; + s->path = mapping->path; + + for (i = 0, cluster = 0; i < s->mapping.next; i++) { + int j; + /* MS-DOS expects the FAT to be 0 for the root directory + * (except for the media byte). */ + /* LATER TODO: still true for FAT32? */ + int fix_fat = (i != 0); + mapping = array_get(&(s->mapping), i); + + if (mapping->mode & MODE_DIRECTORY) { + mapping->begin = cluster; + if(read_directory(s, i)) { + fprintf(stderr, "Could not read directory %s\n", + mapping->path); + return -1; + } + mapping = array_get(&(s->mapping), i); + } else { + assert(mapping->mode == MODE_UNDEFINED); + mapping->mode=MODE_NORMAL; + mapping->begin = cluster; + if (mapping->end > 0) { + direntry_t* direntry = array_get(&(s->directory), + mapping->dir_index); + + mapping->end = cluster + 1 + (mapping->end-1)/s->cluster_size; + set_begin_of_direntry(direntry, mapping->begin); + } else { + mapping->end = cluster + 1; + fix_fat = 0; + } + } + + assert(mapping->begin < mapping->end); + + /* fix fat for entry */ + if (fix_fat) { + for(j = mapping->begin; j < mapping->end - 1; j++) + fat_set(s, j, j+1); + fat_set(s, mapping->end - 1, s->max_fat_value); + } + + /* next free cluster */ + cluster = mapping->end; + + if(cluster > s->cluster_count) { + fprintf(stderr,"Directory does not fit in FAT%d\n",s->fat_type); + return -1; + } + } + + mapping = array_get(&(s->mapping), 0); + s->sectors_of_root_directory = mapping->end * s->sectors_per_cluster; + s->last_cluster_of_root_directory = mapping->end; + + /* the FAT signature */ + fat_set(s,0,s->max_fat_value); + fat_set(s,1,s->max_fat_value); + + s->current_mapping = NULL; + + bootsector=(bootsector_t*)(s->first_sectors+(s->first_sectors_number-1)*0x200); + bootsector->jump[0]=0xeb; + bootsector->jump[1]=0x3e; + bootsector->jump[2]=0x90; + memcpy(bootsector->name,"QEMU ",8); + bootsector->sector_size=cpu_to_le16(0x200); + bootsector->sectors_per_cluster=s->sectors_per_cluster; + bootsector->reserved_sectors=cpu_to_le16(1); + bootsector->number_of_fats=0x2; /* number of FATs */ + bootsector->root_entries=cpu_to_le16(s->sectors_of_root_directory*0x10); + bootsector->total_sectors16=s->sector_count>0xffff?0:cpu_to_le16(s->sector_count); + bootsector->media_type=(s->fat_type!=12?0xf8:s->sector_count==5760?0xf9:0xf8); /* media descriptor */ + s->fat.pointer[0] = bootsector->media_type; + bootsector->sectors_per_fat=cpu_to_le16(s->sectors_per_fat); + bootsector->sectors_per_track=cpu_to_le16(s->bs->secs); + bootsector->number_of_heads=cpu_to_le16(s->bs->heads); + bootsector->hidden_sectors=cpu_to_le32(s->first_sectors_number==1?0:0x3f); + bootsector->total_sectors=cpu_to_le32(s->sector_count>0xffff?s->sector_count:0); + + /* LATER TODO: if FAT32, this is wrong */ + bootsector->u.fat16.drive_number=s->fat_type==12?0:0x80; /* assume this is hda (TODO) */ + bootsector->u.fat16.current_head=0; + bootsector->u.fat16.signature=0x29; + bootsector->u.fat16.id=cpu_to_le32(0xfabe1afd); + + memcpy(bootsector->u.fat16.volume_label,"QEMU VVFAT ",11); + memcpy(bootsector->fat_type,(s->fat_type==12?"FAT12 ":s->fat_type==16?"FAT16 ":"FAT32 "),8); + bootsector->magic[0]=0x55; bootsector->magic[1]=0xaa; + + return 0; +} + +#ifdef DEBUG +static BDRVVVFATState *vvv = NULL; +#endif + +static int enable_write_target(BDRVVVFATState *s); +static int is_consistent(BDRVVVFATState *s); + +static int vvfat_open(BlockDriverState *bs, const char* dirname, int flags) +{ + BDRVVVFATState *s = bs->opaque; + int floppy = 0; + int i; + +#ifdef DEBUG + vvv = s; +#endif + +DLOG(if (stderr == NULL) { + stderr = fopen("vvfat.log", "a"); + setbuf(stderr, NULL); +}) + + s->bs = bs; + + s->fat_type=16; + /* LATER TODO: if FAT32, adjust */ + s->sectors_per_cluster=0x10; + /* 504MB disk*/ + bs->cyls=1024; bs->heads=16; bs->secs=63; + + s->current_cluster=0xffffffff; + + s->first_sectors_number=0x40; + /* read only is the default for safety */ + bs->read_only = 1; + s->qcow = s->write_target = NULL; + s->qcow_filename = NULL; + s->fat2 = NULL; + s->downcase_short_names = 1; + + if (!strstart(dirname, "fat:", NULL)) + return -1; + + if (strstr(dirname, ":floppy:")) { + floppy = 1; + s->fat_type = 12; + s->first_sectors_number = 1; + s->sectors_per_cluster=2; + bs->cyls = 80; bs->heads = 2; bs->secs = 36; + } + + s->sector_count=bs->cyls*bs->heads*bs->secs; + + if (strstr(dirname, ":32:")) { + fprintf(stderr, "Big fat greek warning: FAT32 has not been tested. You are welcome to do so!\n"); + s->fat_type = 32; + } else if (strstr(dirname, ":16:")) { + s->fat_type = 16; + } else if (strstr(dirname, ":12:")) { + s->fat_type = 12; + s->sector_count=2880; + } + + if (strstr(dirname, ":rw:")) { + if (enable_write_target(s)) + return -1; + bs->read_only = 0; + } + + i = strrchr(dirname, ':') - dirname; + assert(i >= 3); + if (dirname[i-2] == ':' && isalpha(dirname[i-1])) + /* workaround for DOS drive names */ + dirname += i-1; + else + dirname += i+1; + + bs->total_sectors=bs->cyls*bs->heads*bs->secs; + + if(init_directories(s, dirname)) + return -1; + + s->sector_count = s->faked_sectors + s->sectors_per_cluster*s->cluster_count; + + if(s->first_sectors_number==0x40) + init_mbr(s); + + /* for some reason or other, MS-DOS does not like to know about CHS... */ + if (floppy) + bs->heads = bs->cyls = bs->secs = 0; + + // assert(is_consistent(s)); + return 0; +} + +static inline void vvfat_close_current_file(BDRVVVFATState *s) +{ + if(s->current_mapping) { + s->current_mapping = NULL; + if (s->current_fd) { + close(s->current_fd); + s->current_fd = 0; + } + } + s->current_cluster = -1; +} + +/* mappings between index1 and index2-1 are supposed to be ordered + * return value is the index of the last mapping for which end>cluster_num + */ +static inline int find_mapping_for_cluster_aux(BDRVVVFATState* s,int cluster_num,int index1,int index2) +{ + int index3=index1+1; + while(1) { + mapping_t* mapping; + index3=(index1+index2)/2; + mapping=array_get(&(s->mapping),index3); + assert(mapping->begin < mapping->end); + if(mapping->begin>=cluster_num) { + assert(index2!=index3 || index2==0); + if(index2==index3) + return index1; + index2=index3; + } else { + if(index1==index3) + return mapping->end<=cluster_num ? index2 : index1; + index1=index3; + } + assert(index1<=index2); + DLOG(mapping=array_get(&(s->mapping),index1); + assert(mapping->begin<=cluster_num); + assert(index2 >= s->mapping.next || + ((mapping = array_get(&(s->mapping),index2)) && + mapping->end>cluster_num))); + } +} + +static inline mapping_t* find_mapping_for_cluster(BDRVVVFATState* s,int cluster_num) +{ + int index=find_mapping_for_cluster_aux(s,cluster_num,0,s->mapping.next); + mapping_t* mapping; + if(index>=s->mapping.next) + return 0; + mapping=array_get(&(s->mapping),index); + if(mapping->begin>cluster_num) + return 0; + assert(mapping->begin<=cluster_num && mapping->end>cluster_num); + return mapping; +} + +/* + * This function simply compares path == mapping->path. Since the mappings + * are sorted by cluster, this is expensive: O(n). + */ +static inline mapping_t* find_mapping_for_path(BDRVVVFATState* s, + const char* path) +{ + int i; + + for (i = 0; i < s->mapping.next; i++) { + mapping_t* mapping = array_get(&(s->mapping), i); + if (mapping->first_mapping_index < 0 && + !strcmp(path, mapping->path)) + return mapping; + } + + return NULL; +} + +static int open_file(BDRVVVFATState* s,mapping_t* mapping) +{ + if(!mapping) + return -1; + if(!s->current_mapping || + strcmp(s->current_mapping->path,mapping->path)) { + /* open file */ + int fd = open(mapping->path, O_RDONLY | O_BINARY | O_LARGEFILE); + if(fd<0) + return -1; + vvfat_close_current_file(s); + s->current_fd = fd; + s->current_mapping = mapping; + } + return 0; +} + +static inline int read_cluster(BDRVVVFATState *s,int cluster_num) +{ + if(s->current_cluster != cluster_num) { + int result=0; + off_t offset; + assert(!s->current_mapping || s->current_fd || (s->current_mapping->mode & MODE_DIRECTORY)); + if(!s->current_mapping + || s->current_mapping->begin>cluster_num + || s->current_mapping->end<=cluster_num) { + /* binary search of mappings for file */ + mapping_t* mapping=find_mapping_for_cluster(s,cluster_num); + + assert(!mapping || (cluster_num>=mapping->begin && cluster_numend)); + + if (mapping && mapping->mode & MODE_DIRECTORY) { + vvfat_close_current_file(s); + s->current_mapping = mapping; +read_cluster_directory: + offset = s->cluster_size*(cluster_num-s->current_mapping->begin); + s->cluster = (unsigned char*)s->directory.pointer+offset + + 0x20*s->current_mapping->info.dir.first_dir_index; + assert(((s->cluster-(unsigned char*)s->directory.pointer)%s->cluster_size)==0); + assert((char*)s->cluster+s->cluster_size <= s->directory.pointer+s->directory.next*s->directory.item_size); + s->current_cluster = cluster_num; + return 0; + } + + if(open_file(s,mapping)) + return -2; + } else if (s->current_mapping->mode & MODE_DIRECTORY) + goto read_cluster_directory; + + assert(s->current_fd); + + offset=s->cluster_size*(cluster_num-s->current_mapping->begin)+s->current_mapping->info.file.offset; + if(lseek(s->current_fd, offset, SEEK_SET)!=offset) + return -3; + s->cluster=s->cluster_buffer; + result=read(s->current_fd,s->cluster,s->cluster_size); + if(result<0) { + s->current_cluster = -1; + return -1; + } + s->current_cluster = cluster_num; + } + return 0; +} + +#ifdef DEBUG +static void hexdump(const void* address, uint32_t len) +{ + const unsigned char* p = address; + int i, j; + + for (i = 0; i < len; i += 16) { + for (j = 0; j < 16 && i + j < len; j++) + fprintf(stderr, "%02x ", p[i + j]); + for (; j < 16; j++) + fprintf(stderr, " "); + fprintf(stderr, " "); + for (j = 0; j < 16 && i + j < len; j++) + fprintf(stderr, "%c", (p[i + j] < ' ' || p[i + j] > 0x7f) ? '.' : p[i + j]); + fprintf(stderr, "\n"); + } +} + +static void print_direntry(const direntry_t* direntry) +{ + int j = 0; + char buffer[1024]; + + fprintf(stderr, "direntry 0x%x: ", (int)direntry); + if(!direntry) + return; + if(is_long_name(direntry)) { + unsigned char* c=(unsigned char*)direntry; + int i; + for(i=1;i<11 && c[i] && c[i]!=0xff;i+=2) +#define ADD_CHAR(c) {buffer[j] = (c); if (buffer[j] < ' ') buffer[j] = '°'; j++;} + ADD_CHAR(c[i]); + for(i=14;i<26 && c[i] && c[i]!=0xff;i+=2) + ADD_CHAR(c[i]); + for(i=28;i<32 && c[i] && c[i]!=0xff;i+=2) + ADD_CHAR(c[i]); + buffer[j] = 0; + fprintf(stderr, "%s\n", buffer); + } else { + int i; + for(i=0;i<11;i++) + ADD_CHAR(direntry->name[i]); + buffer[j] = 0; + fprintf(stderr,"%s attributes=0x%02x begin=%d size=%d\n", + buffer, + direntry->attributes, + begin_of_direntry(direntry),le32_to_cpu(direntry->size)); + } +} + +static void print_mapping(const mapping_t* mapping) +{ + fprintf(stderr, "mapping (0x%x): begin, end = %d, %d, dir_index = %d, first_mapping_index = %d, name = %s, mode = 0x%x, " , (int)mapping, mapping->begin, mapping->end, mapping->dir_index, mapping->first_mapping_index, mapping->path, mapping->mode); + if (mapping->mode & MODE_DIRECTORY) + fprintf(stderr, "parent_mapping_index = %d, first_dir_index = %d\n", mapping->info.dir.parent_mapping_index, mapping->info.dir.first_dir_index); + else + fprintf(stderr, "offset = %d\n", mapping->info.file.offset); +} +#endif + +static int vvfat_read(BlockDriverState *bs, int64_t sector_num, + uint8_t *buf, int nb_sectors) +{ + BDRVVVFATState *s = bs->opaque; + int i; + + for(i=0;i= s->sector_count) + return -1; + if (s->qcow) { + int n; + if (s->qcow->drv->bdrv_is_allocated(s->qcow, + sector_num, nb_sectors-i, &n)) { +DLOG(fprintf(stderr, "sectors %d+%d allocated\n", (int)sector_num, n)); + if (s->qcow->drv->bdrv_read(s->qcow, sector_num, buf+i*0x200, n)) + return -1; + i += n - 1; + sector_num += n - 1; + continue; + } +DLOG(fprintf(stderr, "sector %d not allocated\n", (int)sector_num)); + } + if(sector_numfaked_sectors) { + if(sector_numfirst_sectors_number) + memcpy(buf+i*0x200,&(s->first_sectors[sector_num*0x200]),0x200); + else if(sector_num-s->first_sectors_numbersectors_per_fat) + memcpy(buf+i*0x200,&(s->fat.pointer[(sector_num-s->first_sectors_number)*0x200]),0x200); + else if(sector_num-s->first_sectors_number-s->sectors_per_fatsectors_per_fat) + memcpy(buf+i*0x200,&(s->fat.pointer[(sector_num-s->first_sectors_number-s->sectors_per_fat)*0x200]),0x200); + } else { + uint32_t sector=sector_num-s->faked_sectors, + sector_offset_in_cluster=(sector%s->sectors_per_cluster), + cluster_num=sector/s->sectors_per_cluster; + if(read_cluster(s, cluster_num) != 0) { + /* LATER TODO: strict: return -1; */ + memset(buf+i*0x200,0,0x200); + continue; + } + memcpy(buf+i*0x200,s->cluster+sector_offset_in_cluster*0x200,0x200); + } + } + return 0; +} + +/* LATER TODO: statify all functions */ + +/* + * Idea of the write support (use snapshot): + * + * 1. check if all data is consistent, recording renames, modifications, + * new files and directories (in s->commits). + * + * 2. if the data is not consistent, stop committing + * + * 3. handle renames, and create new files and directories (do not yet + * write their contents) + * + * 4. walk the directories, fixing the mapping and direntries, and marking + * the handled mappings as not deleted + * + * 5. commit the contents of the files + * + * 6. handle deleted files and directories + * + */ + +typedef struct commit_t { + char* path; + union { + struct { uint32_t cluster; } rename; + struct { int dir_index; uint32_t modified_offset; } writeout; + struct { uint32_t first_cluster; } new_file; + struct { uint32_t cluster; } mkdir; + } param; + /* DELETEs and RMDIRs are handled differently: see handle_deletes() */ + enum { + ACTION_RENAME, ACTION_WRITEOUT, ACTION_NEW_FILE, ACTION_MKDIR + } action; +} commit_t; + +static void clear_commits(BDRVVVFATState* s) +{ + int i; +DLOG(fprintf(stderr, "clear_commits (%d commits)\n", s->commits.next)); + for (i = 0; i < s->commits.next; i++) { + commit_t* commit = array_get(&(s->commits), i); + assert(commit->path || commit->action == ACTION_WRITEOUT); + if (commit->action != ACTION_WRITEOUT) { + assert(commit->path); + free(commit->path); + } else + assert(commit->path == NULL); + } + s->commits.next = 0; +} + +static void schedule_rename(BDRVVVFATState* s, + uint32_t cluster, char* new_path) +{ + commit_t* commit = array_get_next(&(s->commits)); + commit->path = new_path; + commit->param.rename.cluster = cluster; + commit->action = ACTION_RENAME; +} + +static void schedule_writeout(BDRVVVFATState* s, + int dir_index, uint32_t modified_offset) +{ + commit_t* commit = array_get_next(&(s->commits)); + commit->path = NULL; + commit->param.writeout.dir_index = dir_index; + commit->param.writeout.modified_offset = modified_offset; + commit->action = ACTION_WRITEOUT; +} + +static void schedule_new_file(BDRVVVFATState* s, + char* path, uint32_t first_cluster) +{ + commit_t* commit = array_get_next(&(s->commits)); + commit->path = path; + commit->param.new_file.first_cluster = first_cluster; + commit->action = ACTION_NEW_FILE; +} + +static void schedule_mkdir(BDRVVVFATState* s, uint32_t cluster, char* path) +{ + commit_t* commit = array_get_next(&(s->commits)); + commit->path = path; + commit->param.mkdir.cluster = cluster; + commit->action = ACTION_MKDIR; +} + +typedef struct { + unsigned char name[1024]; + int checksum, len; + int sequence_number; +} long_file_name; + +static void lfn_init(long_file_name* lfn) +{ + lfn->sequence_number = lfn->len = 0; + lfn->checksum = 0x100; +} + +/* return 0 if parsed successfully, > 0 if no long name, < 0 if error */ +static int parse_long_name(long_file_name* lfn, + const direntry_t* direntry) +{ + int i, j, offset; + const unsigned char* pointer = (const unsigned char*)direntry; + + if (!is_long_name(direntry)) + return 1; + + if (pointer[0] & 0x40) { + lfn->sequence_number = pointer[0] & 0x3f; + lfn->checksum = pointer[13]; + lfn->name[0] = 0; + } else if ((pointer[0] & 0x3f) != --lfn->sequence_number) + return -1; + else if (pointer[13] != lfn->checksum) + return -2; + else if (pointer[12] || pointer[26] || pointer[27]) + return -3; + + offset = 13 * (lfn->sequence_number - 1); + for (i = 0, j = 1; i < 13; i++, j+=2) { + if (j == 11) + j = 14; + else if (j == 26) + j = 28; + + if (pointer[j+1] == 0) + lfn->name[offset + i] = pointer[j]; + else if (pointer[j+1] != 0xff || (pointer[0] & 0x40) == 0) + return -4; + else + lfn->name[offset + i] = 0; + } + + if (pointer[0] & 0x40) + lfn->len = offset + strlen((char*)lfn->name + offset); + + return 0; +} + +/* returns 0 if successful, >0 if no short_name, and <0 on error */ +static int parse_short_name(BDRVVVFATState* s, + long_file_name* lfn, direntry_t* direntry) +{ + int i, j; + + if (!is_short_name(direntry)) + return 1; + + for (j = 7; j >= 0 && direntry->name[j] == ' '; j--); + for (i = 0; i <= j; i++) { + if (direntry->name[i] <= ' ' || direntry->name[i] > 0x7f) + return -1; + else if (s->downcase_short_names) + lfn->name[i] = tolower(direntry->name[i]); + else + lfn->name[i] = direntry->name[i]; + } + + for (j = 2; j >= 0 && direntry->extension[j] == ' '; j--); + if (j >= 0) { + lfn->name[i++] = '.'; + lfn->name[i + j + 1] = '\0'; + for (;j >= 0; j--) { + if (direntry->extension[j] <= ' ' || direntry->extension[j] > 0x7f) + return -2; + else if (s->downcase_short_names) + lfn->name[i + j] = tolower(direntry->extension[j]); + else + lfn->name[i + j] = direntry->extension[j]; + } + } else + lfn->name[i + j + 1] = '\0'; + + lfn->len = strlen((char*)lfn->name); + + return 0; +} + +static inline uint32_t modified_fat_get(BDRVVVFATState* s, + unsigned int cluster) +{ + if (cluster < s->last_cluster_of_root_directory) { + if (cluster + 1 == s->last_cluster_of_root_directory) + return s->max_fat_value; + else + return cluster + 1; + } + + if (s->fat_type==32) { + uint32_t* entry=((uint32_t*)s->fat2)+cluster; + return le32_to_cpu(*entry); + } else if (s->fat_type==16) { + uint16_t* entry=((uint16_t*)s->fat2)+cluster; + return le16_to_cpu(*entry); + } else { + const uint8_t* x=s->fat2+cluster*3/2; + return ((x[0]|(x[1]<<8))>>(cluster&1?4:0))&0x0fff; + } +} + +static inline int cluster_was_modified(BDRVVVFATState* s, uint32_t cluster_num) +{ + int was_modified = 0; + int i, dummy; + + if (s->qcow == NULL) + return 0; + + for (i = 0; !was_modified && i < s->sectors_per_cluster; i++) + was_modified = s->qcow->drv->bdrv_is_allocated(s->qcow, + cluster2sector(s, cluster_num) + i, 1, &dummy); + + return was_modified; +} + +static const char* get_basename(const char* path) +{ + char* basename = strrchr(path, '/'); + if (basename == NULL) + return path; + else + return basename + 1; /* strip '/' */ +} + +/* + * The array s->used_clusters holds the states of the clusters. If it is + * part of a file, it has bit 2 set, in case of a directory, bit 1. If it + * was modified, bit 3 is set. + * If any cluster is allocated, but not part of a file or directory, this + * driver refuses to commit. + */ +typedef enum { + USED_DIRECTORY = 1, USED_FILE = 2, USED_ANY = 3, USED_ALLOCATED = 4 +} used_t; + +/* + * get_cluster_count_for_direntry() not only determines how many clusters + * are occupied by direntry, but also if it was renamed or modified. + * + * A file is thought to be renamed *only* if there already was a file with + * exactly the same first cluster, but a different name. + * + * Further, the files/directories handled by this function are + * assumed to be *not* deleted (and *only* those). + */ +static uint32_t get_cluster_count_for_direntry(BDRVVVFATState* s, + direntry_t* direntry, const char* path) +{ + /* + * This is a little bit tricky: + * IF the guest OS just inserts a cluster into the file chain, + * and leaves the rest alone, (i.e. the original file had clusters + * 15 -> 16, but now has 15 -> 32 -> 16), then the following happens: + * + * - do_commit will write the cluster into the file at the given + * offset, but + * + * - the cluster which is overwritten should be moved to a later + * position in the file. + * + * I am not aware that any OS does something as braindead, but this + * situation could happen anyway when not committing for a long time. + * Just to be sure that this does not bite us, detect it, and copy the + * contents of the clusters to-be-overwritten into the qcow. + */ + int copy_it = 0; + int was_modified = 0; + int32_t ret = 0; + + uint32_t cluster_num = begin_of_direntry(direntry); + uint32_t offset = 0; + int first_mapping_index = -1; + mapping_t* mapping = NULL; + const char* basename2 = NULL; + + vvfat_close_current_file(s); + + /* the root directory */ + if (cluster_num == 0) + return 0; + + /* write support */ + if (s->qcow) { + basename2 = get_basename(path); + + mapping = find_mapping_for_cluster(s, cluster_num); + + if (mapping) { + const char* basename; + + assert(mapping->mode & MODE_DELETED); + mapping->mode &= ~MODE_DELETED; + + basename = get_basename(mapping->path); + + assert(mapping->mode & MODE_NORMAL); + + /* rename */ + if (strcmp(basename, basename2)) + schedule_rename(s, cluster_num, strdup(path)); + } else if (is_file(direntry)) + /* new file */ + schedule_new_file(s, strdup(path), cluster_num); + else { + assert(0); + return 0; + } + } + + while(1) { + if (s->qcow) { + if (!copy_it && cluster_was_modified(s, cluster_num)) { + if (mapping == NULL || + mapping->begin > cluster_num || + mapping->end <= cluster_num) + mapping = find_mapping_for_cluster(s, cluster_num); + + + if (mapping && + (mapping->mode & MODE_DIRECTORY) == 0) { + + /* was modified in qcow */ + if (offset != mapping->info.file.offset + s->cluster_size + * (cluster_num - mapping->begin)) { + /* offset of this cluster in file chain has changed */ + assert(0); + copy_it = 1; + } else if (offset == 0) { + const char* basename = get_basename(mapping->path); + + if (strcmp(basename, basename2)) + copy_it = 1; + first_mapping_index = array_index(&(s->mapping), mapping); + } + + if (mapping->first_mapping_index != first_mapping_index + && mapping->info.file.offset > 0) { + assert(0); + copy_it = 1; + } + + /* need to write out? */ + if (!was_modified && is_file(direntry)) { + was_modified = 1; + schedule_writeout(s, mapping->dir_index, offset); + } + } + } + + if (copy_it) { + int i, dummy; + /* + * This is horribly inefficient, but that is okay, since + * it is rarely executed, if at all. + */ + int64_t offset = cluster2sector(s, cluster_num); + + vvfat_close_current_file(s); + for (i = 0; i < s->sectors_per_cluster; i++) + if (!s->qcow->drv->bdrv_is_allocated(s->qcow, + offset + i, 1, &dummy)) { + if (vvfat_read(s->bs, + offset, s->cluster_buffer, 1)) + return -1; + if (s->qcow->drv->bdrv_write(s->qcow, + offset, s->cluster_buffer, 1)) + return -2; + } + } + } + + ret++; + if (s->used_clusters[cluster_num] & USED_ANY) + return 0; + s->used_clusters[cluster_num] = USED_FILE; + + cluster_num = modified_fat_get(s, cluster_num); + + if (fat_eof(s, cluster_num)) + return ret; + else if (cluster_num < 2 || cluster_num > s->max_fat_value - 16) + return -1; + + offset += s->cluster_size; + } +} + +/* + * This function looks at the modified data (qcow). + * It returns 0 upon inconsistency or error, and the number of clusters + * used by the directory, its subdirectories and their files. + */ +static int check_directory_consistency(BDRVVVFATState *s, + int cluster_num, const char* path) +{ + int ret = 0; + unsigned char* cluster = malloc(s->cluster_size); + direntry_t* direntries = (direntry_t*)cluster; + mapping_t* mapping = find_mapping_for_cluster(s, cluster_num); + + long_file_name lfn; + int path_len = strlen(path); + char path2[PATH_MAX]; + + assert(path_len < PATH_MAX); /* len was tested before! */ + strcpy(path2, path); + path2[path_len] = '/'; + path2[path_len + 1] = '\0'; + + if (mapping) { + const char* basename = get_basename(mapping->path); + const char* basename2 = get_basename(path); + + assert(mapping->mode & MODE_DIRECTORY); + + assert(mapping->mode & MODE_DELETED); + mapping->mode &= ~MODE_DELETED; + + if (strcmp(basename, basename2)) + schedule_rename(s, cluster_num, strdup(path)); + } else + /* new directory */ + schedule_mkdir(s, cluster_num, strdup(path)); + + lfn_init(&lfn); + do { + int i; + int subret = 0; + + ret++; + + if (s->used_clusters[cluster_num] & USED_ANY) { + fprintf(stderr, "cluster %d used more than once\n", (int)cluster_num); + return 0; + } + s->used_clusters[cluster_num] = USED_DIRECTORY; + +DLOG(fprintf(stderr, "read cluster %d (sector %d)\n", (int)cluster_num, (int)cluster2sector(s, cluster_num))); + subret = vvfat_read(s->bs, cluster2sector(s, cluster_num), cluster, + s->sectors_per_cluster); + if (subret) { + fprintf(stderr, "Error fetching direntries\n"); + fail: + free(cluster); + return 0; + } + + for (i = 0; i < 0x10 * s->sectors_per_cluster; i++) { + int cluster_count; + +DLOG(fprintf(stderr, "check direntry %d: \n", i); print_direntry(direntries + i)); + if (is_volume_label(direntries + i) || is_dot(direntries + i) || + is_free(direntries + i)) + continue; + + subret = parse_long_name(&lfn, direntries + i); + if (subret < 0) { + fprintf(stderr, "Error in long name\n"); + goto fail; + } + if (subret == 0 || is_free(direntries + i)) + continue; + + if (fat_chksum(direntries+i) != lfn.checksum) { + subret = parse_short_name(s, &lfn, direntries + i); + if (subret < 0) { + fprintf(stderr, "Error in short name (%d)\n", subret); + goto fail; + } + if (subret > 0 || !strcmp((char*)lfn.name, ".") + || !strcmp((char*)lfn.name, "..")) + continue; + } + lfn.checksum = 0x100; /* cannot use long name twice */ + + if (path_len + 1 + lfn.len >= PATH_MAX) { + fprintf(stderr, "Name too long: %s/%s\n", path, lfn.name); + goto fail; + } + strcpy(path2 + path_len + 1, (char*)lfn.name); + + if (is_directory(direntries + i)) { + if (begin_of_direntry(direntries + i) == 0) { + DLOG(fprintf(stderr, "invalid begin for directory: %s\n", path2); print_direntry(direntries + i)); + goto fail; + } + cluster_count = check_directory_consistency(s, + begin_of_direntry(direntries + i), path2); + if (cluster_count == 0) { + DLOG(fprintf(stderr, "problem in directory %s:\n", path2); print_direntry(direntries + i)); + goto fail; + } + } else if (is_file(direntries + i)) { + /* check file size with FAT */ + cluster_count = get_cluster_count_for_direntry(s, direntries + i, path2); + if (cluster_count != + (le32_to_cpu(direntries[i].size) + s->cluster_size + - 1) / s->cluster_size) { + DLOG(fprintf(stderr, "Cluster count mismatch\n")); + goto fail; + } + } else + assert(0); /* cluster_count = 0; */ + + ret += cluster_count; + } + + cluster_num = modified_fat_get(s, cluster_num); + } while(!fat_eof(s, cluster_num)); + + free(cluster); + return ret; +} + +/* returns 1 on success */ +static int is_consistent(BDRVVVFATState* s) +{ + int i, check; + int used_clusters_count = 0; + +DLOG(checkpoint()); + /* + * - get modified FAT + * - compare the two FATs (TODO) + * - get buffer for marking used clusters + * - recurse direntries from root (using bs->bdrv_read to make + * sure to get the new data) + * - check that the FAT agrees with the size + * - count the number of clusters occupied by this directory and + * its files + * - check that the cumulative used cluster count agrees with the + * FAT + * - if all is fine, return number of used clusters + */ + if (s->fat2 == NULL) { + int size = 0x200 * s->sectors_per_fat; + s->fat2 = malloc(size); + memcpy(s->fat2, s->fat.pointer, size); + } + check = vvfat_read(s->bs, + s->first_sectors_number, s->fat2, s->sectors_per_fat); + if (check) { + fprintf(stderr, "Could not copy fat\n"); + return 0; + } + assert (s->used_clusters); + for (i = 0; i < sector2cluster(s, s->sector_count); i++) + s->used_clusters[i] &= ~USED_ANY; + + clear_commits(s); + + /* mark every mapped file/directory as deleted. + * (check_directory_consistency() will unmark those still present). */ + if (s->qcow) + for (i = 0; i < s->mapping.next; i++) { + mapping_t* mapping = array_get(&(s->mapping), i); + if (mapping->first_mapping_index < 0) + mapping->mode |= MODE_DELETED; + } + + used_clusters_count = check_directory_consistency(s, 0, s->path); + if (used_clusters_count <= 0) { + DLOG(fprintf(stderr, "problem in directory\n")); + return 0; + } + + check = s->last_cluster_of_root_directory; + for (i = check; i < sector2cluster(s, s->sector_count); i++) { + if (modified_fat_get(s, i)) { + if(!s->used_clusters[i]) { + DLOG(fprintf(stderr, "FAT was modified (%d), but cluster is not used?\n", i)); + return 0; + } + check++; + } + + if (s->used_clusters[i] == USED_ALLOCATED) { + /* allocated, but not used... */ + DLOG(fprintf(stderr, "unused, modified cluster: %d\n", i)); + return 0; + } + } + + if (check != used_clusters_count) + return 0; + + return used_clusters_count; +} + +static inline void adjust_mapping_indices(BDRVVVFATState* s, + int offset, int adjust) +{ + int i; + + for (i = 0; i < s->mapping.next; i++) { + mapping_t* mapping = array_get(&(s->mapping), i); + +#define ADJUST_MAPPING_INDEX(name) \ + if (mapping->name >= offset) \ + mapping->name += adjust + + ADJUST_MAPPING_INDEX(first_mapping_index); + if (mapping->mode & MODE_DIRECTORY) + ADJUST_MAPPING_INDEX(info.dir.parent_mapping_index); + } +} + +/* insert or update mapping */ +static mapping_t* insert_mapping(BDRVVVFATState* s, + uint32_t begin, uint32_t end) +{ + /* + * - find mapping where mapping->begin >= begin, + * - if mapping->begin > begin: insert + * - adjust all references to mappings! + * - else: adjust + * - replace name + */ + int index = find_mapping_for_cluster_aux(s, begin, 0, s->mapping.next); + mapping_t* mapping = NULL; + mapping_t* first_mapping = array_get(&(s->mapping), 0); + + if (index < s->mapping.next && (mapping = array_get(&(s->mapping), index)) + && mapping->begin < begin) { + mapping->end = begin; + index++; + mapping = array_get(&(s->mapping), index); + } + if (index >= s->mapping.next || mapping->begin > begin) { + mapping = array_insert(&(s->mapping), index, 1); + mapping->path = NULL; + adjust_mapping_indices(s, index, +1); + } + + mapping->begin = begin; + mapping->end = end; + +DLOG(mapping_t* next_mapping; +assert(index + 1 >= s->mapping.next || +((next_mapping = array_get(&(s->mapping), index + 1)) && + next_mapping->begin >= end))); + + if (s->current_mapping && first_mapping != (mapping_t*)s->mapping.pointer) + s->current_mapping = array_get(&(s->mapping), + s->current_mapping - first_mapping); + + return mapping; +} + +static int remove_mapping(BDRVVVFATState* s, int mapping_index) +{ + mapping_t* mapping = array_get(&(s->mapping), mapping_index); + mapping_t* first_mapping = array_get(&(s->mapping), 0); + + /* free mapping */ + if (mapping->first_mapping_index < 0) + free(mapping->path); + + /* remove from s->mapping */ + array_remove(&(s->mapping), mapping_index); + + /* adjust all references to mappings */ + adjust_mapping_indices(s, mapping_index, -1); + + if (s->current_mapping && first_mapping != (mapping_t*)s->mapping.pointer) + s->current_mapping = array_get(&(s->mapping), + s->current_mapping - first_mapping); + + return 0; +} + +static void adjust_dirindices(BDRVVVFATState* s, int offset, int adjust) +{ + int i; + for (i = 0; i < s->mapping.next; i++) { + mapping_t* mapping = array_get(&(s->mapping), i); + if (mapping->dir_index >= offset) + mapping->dir_index += adjust; + if ((mapping->mode & MODE_DIRECTORY) && + mapping->info.dir.first_dir_index >= offset) + mapping->info.dir.first_dir_index += adjust; + } +} + +static direntry_t* insert_direntries(BDRVVVFATState* s, + int dir_index, int count) +{ + /* + * make room in s->directory, + * adjust_dirindices + */ + direntry_t* result = array_insert(&(s->directory), dir_index, count); + if (result == NULL) + return NULL; + adjust_dirindices(s, dir_index, count); + return result; +} + +static int remove_direntries(BDRVVVFATState* s, int dir_index, int count) +{ + int ret = array_remove_slice(&(s->directory), dir_index, count); + if (ret) + return ret; + adjust_dirindices(s, dir_index, -count); + return 0; +} + +/* + * Adapt the mappings of the cluster chain starting at first cluster + * (i.e. if a file starts at first_cluster, the chain is followed according + * to the modified fat, and the corresponding entries in s->mapping are + * adjusted) + */ +static int commit_mappings(BDRVVVFATState* s, + uint32_t first_cluster, int dir_index) +{ + mapping_t* mapping = find_mapping_for_cluster(s, first_cluster); + direntry_t* direntry = array_get(&(s->directory), dir_index); + uint32_t cluster = first_cluster; + + vvfat_close_current_file(s); + + assert(mapping); + assert(mapping->begin == first_cluster); + mapping->first_mapping_index = -1; + mapping->dir_index = dir_index; + mapping->mode = (dir_index <= 0 || is_directory(direntry)) ? + MODE_DIRECTORY : MODE_NORMAL; + + while (!fat_eof(s, cluster)) { + uint32_t c, c1; + + for (c = cluster, c1 = modified_fat_get(s, c); c + 1 == c1; + c = c1, c1 = modified_fat_get(s, c1)); + + c++; + if (c > mapping->end) { + int index = array_index(&(s->mapping), mapping); + int i, max_i = s->mapping.next - index; + for (i = 1; i < max_i && mapping[i].begin < c; i++); + while (--i > 0) + remove_mapping(s, index + 1); + } + assert(mapping == array_get(&(s->mapping), s->mapping.next - 1) + || mapping[1].begin >= c); + mapping->end = c; + + if (!fat_eof(s, c1)) { + int i = find_mapping_for_cluster_aux(s, c1, 0, s->mapping.next); + mapping_t* next_mapping = i >= s->mapping.next ? NULL : + array_get(&(s->mapping), i); + + if (next_mapping == NULL || next_mapping->begin > c1) { + int i1 = array_index(&(s->mapping), mapping); + + next_mapping = insert_mapping(s, c1, c1+1); + + if (c1 < c) + i1++; + mapping = array_get(&(s->mapping), i1); + } + + next_mapping->dir_index = mapping->dir_index; + next_mapping->first_mapping_index = + mapping->first_mapping_index < 0 ? + array_index(&(s->mapping), mapping) : + mapping->first_mapping_index; + next_mapping->path = mapping->path; + next_mapping->mode = mapping->mode; + next_mapping->read_only = mapping->read_only; + if (mapping->mode & MODE_DIRECTORY) { + next_mapping->info.dir.parent_mapping_index = + mapping->info.dir.parent_mapping_index; + next_mapping->info.dir.first_dir_index = + mapping->info.dir.first_dir_index + + 0x10 * s->sectors_per_cluster * + (mapping->end - mapping->begin); + } else + next_mapping->info.file.offset = mapping->info.file.offset + + mapping->end - mapping->begin; + + mapping = next_mapping; + } + + cluster = c1; + } + + return 0; +} + +static int commit_direntries(BDRVVVFATState* s, + int dir_index, int parent_mapping_index) +{ + direntry_t* direntry = array_get(&(s->directory), dir_index); + uint32_t first_cluster = dir_index == 0 ? 0 : begin_of_direntry(direntry); + mapping_t* mapping = find_mapping_for_cluster(s, first_cluster); + + int factor = 0x10 * s->sectors_per_cluster; + int old_cluster_count, new_cluster_count; + int current_dir_index = mapping->info.dir.first_dir_index; + int first_dir_index = current_dir_index; + int ret, i; + uint32_t c; + +DLOG(fprintf(stderr, "commit_direntries for %s, parent_mapping_index %d\n", mapping->path, parent_mapping_index)); + + assert(direntry); + assert(mapping); + assert(mapping->begin == first_cluster); + assert(mapping->info.dir.first_dir_index < s->directory.next); + assert(mapping->mode & MODE_DIRECTORY); + assert(dir_index == 0 || is_directory(direntry)); + + mapping->info.dir.parent_mapping_index = parent_mapping_index; + + if (first_cluster == 0) { + old_cluster_count = new_cluster_count = + s->last_cluster_of_root_directory; + } else { + for (old_cluster_count = 0, c = first_cluster; !fat_eof(s, c); + c = fat_get(s, c)) + old_cluster_count++; + + for (new_cluster_count = 0, c = first_cluster; !fat_eof(s, c); + c = modified_fat_get(s, c)) + new_cluster_count++; + } + + if (new_cluster_count > old_cluster_count) { + if (insert_direntries(s, + current_dir_index + factor * old_cluster_count, + factor * (new_cluster_count - old_cluster_count)) == NULL) + return -1; + } else if (new_cluster_count < old_cluster_count) + remove_direntries(s, + current_dir_index + factor * new_cluster_count, + factor * (old_cluster_count - new_cluster_count)); + + for (c = first_cluster; !fat_eof(s, c); c = modified_fat_get(s, c)) { + void* direntry = array_get(&(s->directory), current_dir_index); + int ret = vvfat_read(s->bs, cluster2sector(s, c), direntry, + s->sectors_per_cluster); + if (ret) + return ret; + assert(!strncmp(s->directory.pointer, "QEMU", 4)); + current_dir_index += factor; + } + + ret = commit_mappings(s, first_cluster, dir_index); + if (ret) + return ret; + + /* recurse */ + for (i = 0; i < factor * new_cluster_count; i++) { + direntry = array_get(&(s->directory), first_dir_index + i); + if (is_directory(direntry) && !is_dot(direntry)) { + mapping = find_mapping_for_cluster(s, first_cluster); + assert(mapping->mode & MODE_DIRECTORY); + ret = commit_direntries(s, first_dir_index + i, + array_index(&(s->mapping), mapping)); + if (ret) + return ret; + } + } + + return 0; +} + +/* commit one file (adjust contents, adjust mapping), + return first_mapping_index */ +static int commit_one_file(BDRVVVFATState* s, + int dir_index, uint32_t offset) +{ + direntry_t* direntry = array_get(&(s->directory), dir_index); + uint32_t c = begin_of_direntry(direntry); + uint32_t first_cluster = c; + mapping_t* mapping = find_mapping_for_cluster(s, c); + uint32_t size = filesize_of_direntry(direntry); + char* cluster = malloc(s->cluster_size); + uint32_t i; + int fd = 0; + + assert(offset < size); + assert((offset % s->cluster_size) == 0); + + for (i = s->cluster_size; i < offset; i += s->cluster_size) + c = modified_fat_get(s, c); + + fd = open(mapping->path, O_RDWR | O_CREAT | O_BINARY, 0666); + if (fd < 0) { + fprintf(stderr, "Could not open %s... (%s, %d)\n", mapping->path, + strerror(errno), errno); + return fd; + } + if (offset > 0) + if (lseek(fd, offset, SEEK_SET) != offset) + return -3; + + while (offset < size) { + uint32_t c1; + int rest_size = (size - offset > s->cluster_size ? + s->cluster_size : size - offset); + int ret; + + c1 = modified_fat_get(s, c); + + assert((size - offset == 0 && fat_eof(s, c)) || + (size > offset && c >=2 && !fat_eof(s, c))); + assert(size >= 0); + + ret = vvfat_read(s->bs, cluster2sector(s, c), + (uint8_t*)cluster, (rest_size + 0x1ff) / 0x200); + + if (ret < 0) + return ret; + + if (write(fd, cluster, rest_size) < 0) + return -2; + + offset += rest_size; + c = c1; + } + + if(ftruncate(fd, size)<0) return -1; + close(fd); + + return commit_mappings(s, first_cluster, dir_index); +} + +#ifdef DEBUG +/* test, if all mappings point to valid direntries */ +static void check1(BDRVVVFATState* s) +{ + int i; + for (i = 0; i < s->mapping.next; i++) { + mapping_t* mapping = array_get(&(s->mapping), i); + if (mapping->mode & MODE_DELETED) { + fprintf(stderr, "deleted\n"); + continue; + } + assert(mapping->dir_index >= 0); + assert(mapping->dir_index < s->directory.next); + direntry_t* direntry = array_get(&(s->directory), mapping->dir_index); + assert(mapping->begin == begin_of_direntry(direntry) || mapping->first_mapping_index >= 0); + if (mapping->mode & MODE_DIRECTORY) { + assert(mapping->info.dir.first_dir_index + 0x10 * s->sectors_per_cluster * (mapping->end - mapping->begin) <= s->directory.next); + assert((mapping->info.dir.first_dir_index % (0x10 * s->sectors_per_cluster)) == 0); + } + } +} + +/* test, if all direntries have mappings */ +static void check2(BDRVVVFATState* s) +{ + int i; + int first_mapping = -1; + + for (i = 0; i < s->directory.next; i++) { + direntry_t* direntry = array_get(&(s->directory), i); + + if (is_short_name(direntry) && begin_of_direntry(direntry)) { + mapping_t* mapping = find_mapping_for_cluster(s, begin_of_direntry(direntry)); + assert(mapping); + assert(mapping->dir_index == i || is_dot(direntry)); + assert(mapping->begin == begin_of_direntry(direntry) || is_dot(direntry)); + } + + if ((i % (0x10 * s->sectors_per_cluster)) == 0) { + /* cluster start */ + int j, count = 0; + + for (j = 0; j < s->mapping.next; j++) { + mapping_t* mapping = array_get(&(s->mapping), j); + if (mapping->mode & MODE_DELETED) + continue; + if (mapping->mode & MODE_DIRECTORY) { + if (mapping->info.dir.first_dir_index <= i && mapping->info.dir.first_dir_index + 0x10 * s->sectors_per_cluster > i) { + assert(++count == 1); + if (mapping->first_mapping_index == -1) + first_mapping = array_index(&(s->mapping), mapping); + else + assert(first_mapping == mapping->first_mapping_index); + if (mapping->info.dir.parent_mapping_index < 0) + assert(j == 0); + else { + mapping_t* parent = array_get(&(s->mapping), mapping->info.dir.parent_mapping_index); + assert(parent->mode & MODE_DIRECTORY); + assert(parent->info.dir.first_dir_index < mapping->info.dir.first_dir_index); + } + } + } + } + if (count == 0) + first_mapping = -1; + } + } +} +#endif + +static int handle_renames_and_mkdirs(BDRVVVFATState* s) +{ + int i; + +#ifdef DEBUG + fprintf(stderr, "handle_renames\n"); + for (i = 0; i < s->commits.next; i++) { + commit_t* commit = array_get(&(s->commits), i); + fprintf(stderr, "%d, %s (%d, %d)\n", i, commit->path ? commit->path : "(null)", commit->param.rename.cluster, commit->action); + } +#endif + + for (i = 0; i < s->commits.next;) { + commit_t* commit = array_get(&(s->commits), i); + if (commit->action == ACTION_RENAME) { + mapping_t* mapping = find_mapping_for_cluster(s, + commit->param.rename.cluster); + char* old_path = mapping->path; + + assert(commit->path); + mapping->path = commit->path; + if (rename(old_path, mapping->path)) + return -2; + + if (mapping->mode & MODE_DIRECTORY) { + int l1 = strlen(mapping->path); + int l2 = strlen(old_path); + int diff = l1 - l2; + direntry_t* direntry = array_get(&(s->directory), + mapping->info.dir.first_dir_index); + uint32_t c = mapping->begin; + int i = 0; + + /* recurse */ + while (!fat_eof(s, c)) { + do { + direntry_t* d = direntry + i; + + if (is_file(d) || (is_directory(d) && !is_dot(d))) { + mapping_t* m = find_mapping_for_cluster(s, + begin_of_direntry(d)); + int l = strlen(m->path); + char* new_path = malloc(l + diff + 1); + + assert(!strncmp(m->path, mapping->path, l2)); + + strcpy(new_path, mapping->path); + strcpy(new_path + l1, m->path + l2); + + schedule_rename(s, m->begin, new_path); + } + i++; + } while((i % (0x10 * s->sectors_per_cluster)) != 0); + c = fat_get(s, c); + } + } + + free(old_path); + array_remove(&(s->commits), i); + continue; + } else if (commit->action == ACTION_MKDIR) { + mapping_t* mapping; + int j, parent_path_len; + +#ifdef __MINGW32__ + if (mkdir(commit->path)) + return -5; +#else + if (mkdir(commit->path, 0755)) + return -5; +#endif + + mapping = insert_mapping(s, commit->param.mkdir.cluster, + commit->param.mkdir.cluster + 1); + if (mapping == NULL) + return -6; + + mapping->mode = MODE_DIRECTORY; + mapping->read_only = 0; + mapping->path = commit->path; + j = s->directory.next; + assert(j); + insert_direntries(s, s->directory.next, + 0x10 * s->sectors_per_cluster); + mapping->info.dir.first_dir_index = j; + + parent_path_len = strlen(commit->path) + - strlen(get_basename(commit->path)) - 1; + for (j = 0; j < s->mapping.next; j++) { + mapping_t* m = array_get(&(s->mapping), j); + if (m->first_mapping_index < 0 && m != mapping && + !strncmp(m->path, mapping->path, parent_path_len) && + strlen(m->path) == parent_path_len) + break; + } + assert(j < s->mapping.next); + mapping->info.dir.parent_mapping_index = j; + + array_remove(&(s->commits), i); + continue; + } + + i++; + } + return 0; +} + +/* + * TODO: make sure that the short name is not matching *another* file + */ +static int handle_commits(BDRVVVFATState* s) +{ + int i, fail = 0; + + vvfat_close_current_file(s); + + for (i = 0; !fail && i < s->commits.next; i++) { + commit_t* commit = array_get(&(s->commits), i); + switch(commit->action) { + case ACTION_RENAME: case ACTION_MKDIR: + assert(0); + fail = -2; + break; + case ACTION_WRITEOUT: { + direntry_t* entry = array_get(&(s->directory), + commit->param.writeout.dir_index); + uint32_t begin = begin_of_direntry(entry); + mapping_t* mapping = find_mapping_for_cluster(s, begin); + + assert(mapping); + assert(mapping->begin == begin); + assert(commit->path == NULL); + + if (commit_one_file(s, commit->param.writeout.dir_index, + commit->param.writeout.modified_offset)) + fail = -3; + + break; + } + case ACTION_NEW_FILE: { + int begin = commit->param.new_file.first_cluster; + mapping_t* mapping = find_mapping_for_cluster(s, begin); + direntry_t* entry; + int i; + + /* find direntry */ + for (i = 0; i < s->directory.next; i++) { + entry = array_get(&(s->directory), i); + if (is_file(entry) && begin_of_direntry(entry) == begin) + break; + } + + if (i >= s->directory.next) { + fail = -6; + continue; + } + + /* make sure there exists an initial mapping */ + if (mapping && mapping->begin != begin) { + mapping->end = begin; + mapping = NULL; + } + if (mapping == NULL) { + mapping = insert_mapping(s, begin, begin+1); + } + /* most members will be fixed in commit_mappings() */ + assert(commit->path); + mapping->path = commit->path; + mapping->read_only = 0; + mapping->mode = MODE_NORMAL; + mapping->info.file.offset = 0; + + if (commit_one_file(s, i, 0)) + fail = -7; + + break; + } + default: + assert(0); + } + } + if (i > 0 && array_remove_slice(&(s->commits), 0, i)) + return -1; + return fail; +} + +static int handle_deletes(BDRVVVFATState* s) +{ + int i, deferred = 1, deleted = 1; + + /* delete files corresponding to mappings marked as deleted */ + /* handle DELETEs and unused mappings (modified_fat_get(s, mapping->begin) == 0) */ + while (deferred && deleted) { + deferred = 0; + deleted = 0; + + for (i = 1; i < s->mapping.next; i++) { + mapping_t* mapping = array_get(&(s->mapping), i); + if (mapping->mode & MODE_DELETED) { + direntry_t* entry = array_get(&(s->directory), + mapping->dir_index); + + if (is_free(entry)) { + /* remove file/directory */ + if (mapping->mode & MODE_DIRECTORY) { + int j, next_dir_index = s->directory.next, + first_dir_index = mapping->info.dir.first_dir_index; + + if (rmdir(mapping->path) < 0) { + if (errno == ENOTEMPTY) { + deferred++; + continue; + } else + return -5; + } + + for (j = 1; j < s->mapping.next; j++) { + mapping_t* m = array_get(&(s->mapping), j); + if (m->mode & MODE_DIRECTORY && + m->info.dir.first_dir_index > + first_dir_index && + m->info.dir.first_dir_index < + next_dir_index) + next_dir_index = + m->info.dir.first_dir_index; + } + remove_direntries(s, first_dir_index, + next_dir_index - first_dir_index); + + deleted++; + } + } else { + if (unlink(mapping->path)) + return -4; + deleted++; + } + DLOG(fprintf(stderr, "DELETE (%d)\n", i); print_mapping(mapping); print_direntry(entry)); + remove_mapping(s, i); + } + } + } + + return 0; +} + +/* + * synchronize mapping with new state: + * + * - copy FAT (with bdrv_read) + * - mark all filenames corresponding to mappings as deleted + * - recurse direntries from root (using bs->bdrv_read) + * - delete files corresponding to mappings marked as deleted + */ +static int do_commit(BDRVVVFATState* s) +{ + int ret = 0; + + /* the real meat are the commits. Nothing to do? Move along! */ + if (s->commits.next == 0) + return 0; + + vvfat_close_current_file(s); + + ret = handle_renames_and_mkdirs(s); + if (ret) { + fprintf(stderr, "Error handling renames (%d)\n", ret); + assert(0); + return ret; + } + + /* copy FAT (with bdrv_read) */ + memcpy(s->fat.pointer, s->fat2, 0x200 * s->sectors_per_fat); + + /* recurse direntries from root (using bs->bdrv_read) */ + ret = commit_direntries(s, 0, -1); + if (ret) { + fprintf(stderr, "Fatal: error while committing (%d)\n", ret); + assert(0); + return ret; + } + + ret = handle_commits(s); + if (ret) { + fprintf(stderr, "Error handling commits (%d)\n", ret); + assert(0); + return ret; + } + + ret = handle_deletes(s); + if (ret) { + fprintf(stderr, "Error deleting\n"); + assert(0); + return ret; + } + + s->qcow->drv->bdrv_make_empty(s->qcow); + + memset(s->used_clusters, 0, sector2cluster(s, s->sector_count)); + +DLOG(checkpoint()); + return 0; +} + +static int try_commit(BDRVVVFATState* s) +{ + vvfat_close_current_file(s); +DLOG(checkpoint()); + if(!is_consistent(s)) + return -1; + return do_commit(s); +} + +static int vvfat_write(BlockDriverState *bs, int64_t sector_num, + const uint8_t *buf, int nb_sectors) +{ + BDRVVVFATState *s = bs->opaque; + int i, ret; + +DLOG(checkpoint()); + + vvfat_close_current_file(s); + + /* + * Some sanity checks: + * - do not allow writing to the boot sector + * - do not allow to write non-ASCII filenames + */ + + if (sector_num < s->first_sectors_number) + return -1; + + for (i = sector2cluster(s, sector_num); + i <= sector2cluster(s, sector_num + nb_sectors - 1);) { + mapping_t* mapping = find_mapping_for_cluster(s, i); + if (mapping) { + if (mapping->read_only) { + fprintf(stderr, "Tried to write to write-protected file %s\n", + mapping->path); + return -1; + } + + if (mapping->mode & MODE_DIRECTORY) { + int begin = cluster2sector(s, i); + int end = begin + s->sectors_per_cluster, k; + int dir_index; + const direntry_t* direntries; + long_file_name lfn; + + lfn_init(&lfn); + + if (begin < sector_num) + begin = sector_num; + if (end > sector_num + nb_sectors) + end = sector_num + nb_sectors; + dir_index = mapping->dir_index + + 0x10 * (begin - mapping->begin * s->sectors_per_cluster); + direntries = (direntry_t*)(buf + 0x200 * (begin - sector_num)); + + for (k = 0; k < (end - begin) * 0x10; k++) { + /* do not allow non-ASCII filenames */ + if (parse_long_name(&lfn, direntries + k) < 0) { + fprintf(stderr, "Warning: non-ASCII filename\n"); + return -1; + } + /* no access to the direntry of a read-only file */ + else if (is_short_name(direntries+k) && + (direntries[k].attributes & 1)) { + if (memcmp(direntries + k, + array_get(&(s->directory), dir_index + k), + sizeof(direntry_t))) { + fprintf(stderr, "Warning: tried to write to write-protected file\n"); + return -1; + } + } + } + } + i = mapping->end; + } else + i++; + } + + /* + * Use qcow backend. Commit later. + */ +DLOG(fprintf(stderr, "Write to qcow backend: %d + %d\n", (int)sector_num, nb_sectors)); + ret = s->qcow->drv->bdrv_write(s->qcow, sector_num, buf, nb_sectors); + if (ret < 0) { + fprintf(stderr, "Error writing to qcow backend\n"); + return ret; + } + + for (i = sector2cluster(s, sector_num); + i <= sector2cluster(s, sector_num + nb_sectors - 1); i++) + if (i >= 0) + s->used_clusters[i] |= USED_ALLOCATED; + +DLOG(checkpoint()); + /* TODO: add timeout */ + try_commit(s); + +DLOG(checkpoint()); + return 0; +} + +static int vvfat_is_allocated(BlockDriverState *bs, + int64_t sector_num, int nb_sectors, int* n) +{ + BDRVVVFATState* s = bs->opaque; + *n = s->sector_count - sector_num; + if (*n > nb_sectors) + *n = nb_sectors; + else if (*n < 0) + return 0; + return 1; +} + +static int write_target_commit(BlockDriverState *bs, int64_t sector_num, + const uint8_t* buffer, int nb_sectors) { + BDRVVVFATState* s = bs->opaque; + return try_commit(s); +} + +static void write_target_close(BlockDriverState *bs) { + BDRVVVFATState* s = bs->opaque; + bdrv_delete(s->qcow); + free(s->qcow_filename); +} + +static BlockDriver vvfat_write_target = { + "vvfat_write_target", 0, NULL, NULL, NULL, + write_target_commit, + write_target_close, + NULL, NULL, NULL +}; + +static int enable_write_target(BDRVVVFATState *s) +{ + int size = sector2cluster(s, s->sector_count); + s->used_clusters = calloc(size, 1); + + array_init(&(s->commits), sizeof(commit_t)); + + s->qcow_filename = malloc(1024); + get_tmp_filename(s->qcow_filename, 1024); + if (bdrv_create(&bdrv_qcow, + s->qcow_filename, s->sector_count, "fat:", 0) < 0) + return -1; + s->qcow = bdrv_new(""); + if (s->qcow == NULL || bdrv_open(s->qcow, s->qcow_filename, 0) < 0) + return -1; + +#ifndef _WIN32 + unlink(s->qcow_filename); +#endif + + s->bs->backing_hd = calloc(sizeof(BlockDriverState), 1); + s->bs->backing_hd->drv = &vvfat_write_target; + s->bs->backing_hd->opaque = s; + + return 0; +} + +static void vvfat_close(BlockDriverState *bs) +{ + BDRVVVFATState *s = bs->opaque; + + vvfat_close_current_file(s); + array_free(&(s->fat)); + array_free(&(s->directory)); + array_free(&(s->mapping)); + if(s->cluster_buffer) + free(s->cluster_buffer); +} + +BlockDriver bdrv_vvfat = { + "vvfat", + sizeof(BDRVVVFATState), + NULL, /* no probe for protocols */ + vvfat_open, + vvfat_read, + vvfat_write, + vvfat_close, + NULL, /* ??? Not sure if we can do any meaningful flushing. */ + NULL, + vvfat_is_allocated, + .protocol_name = "fat", +}; + +#ifdef DEBUG +static void checkpoint() { + assert(((mapping_t*)array_get(&(vvv->mapping), 0))->end == 2); + check1(vvv); + check2(vvv); + assert(!vvv->current_mapping || vvv->current_fd || (vvv->current_mapping->mode & MODE_DIRECTORY)); +#if 0 + if (((direntry_t*)vvv->directory.pointer)[1].attributes != 0xf) + fprintf(stderr, "Nonono!\n"); + mapping_t* mapping; + direntry_t* direntry; + assert(vvv->mapping.size >= vvv->mapping.item_size * vvv->mapping.next); + assert(vvv->directory.size >= vvv->directory.item_size * vvv->directory.next); + if (vvv->mapping.next<47) + return; + assert((mapping = array_get(&(vvv->mapping), 47))); + assert(mapping->dir_index < vvv->directory.next); + direntry = array_get(&(vvv->directory), mapping->dir_index); + assert(!memcmp(direntry->name, "USB H ", 11) || direntry->name[0]==0); +#endif + return; + /* avoid compiler warnings: */ + hexdump(NULL, 100); + remove_mapping(vvv, NULL); + print_mapping(NULL); + print_direntry(NULL); +} +#endif + diff --git a/lib/qemu/block.c b/lib/qemu/block.c new file mode 100644 index 0000000..a20c33d --- /dev/null +++ b/lib/qemu/block.c @@ -0,0 +1,1414 @@ +/* + * QEMU System Emulator block driver + * + * Copyright (c) 2003 Fabrice Bellard + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include "qemu-common.h" +#ifndef QEMU_IMG +#include "console.h" +#endif +#include "block_int.h" + +#ifdef _BSD +#include +#include +#include +#include +#include +#endif + +#define SECTOR_BITS 9 +#define SECTOR_SIZE (1 << SECTOR_BITS) + +typedef struct BlockDriverAIOCBSync { + BlockDriverAIOCB common; + QEMUBH *bh; + int ret; +} BlockDriverAIOCBSync; + +static BlockDriverAIOCB *bdrv_aio_read_em(BlockDriverState *bs, + int64_t sector_num, uint8_t *buf, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque); +static BlockDriverAIOCB *bdrv_aio_write_em(BlockDriverState *bs, + int64_t sector_num, const uint8_t *buf, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque); +static void bdrv_aio_cancel_em(BlockDriverAIOCB *acb); +static int bdrv_read_em(BlockDriverState *bs, int64_t sector_num, + uint8_t *buf, int nb_sectors); +static int bdrv_write_em(BlockDriverState *bs, int64_t sector_num, + const uint8_t *buf, int nb_sectors); + +BlockDriverState *bdrv_first; +static BlockDriver *first_drv; + +int path_is_absolute(const char *path) +{ + const char *p; +#ifdef _WIN32 + /* specific case for names like: "\\.\d:" */ + if (*path == '/' || *path == '\\') + return 1; +#endif + p = strchr(path, ':'); + if (p) + p++; + else + p = path; +#ifdef _WIN32 + return (*p == '/' || *p == '\\'); +#else + return (*p == '/'); +#endif +} + +/* if filename is absolute, just copy it to dest. Otherwise, build a + path to it by considering it is relative to base_path. URL are + supported. */ +void path_combine(char *dest, int dest_size, + const char *base_path, + const char *filename) +{ + const char *p, *p1; + int len; + + if (dest_size <= 0) + return; + if (path_is_absolute(filename)) { + pstrcpy(dest, dest_size, filename); + } else { + p = strchr(base_path, ':'); + if (p) + p++; + else + p = base_path; + p1 = strrchr(base_path, '/'); +#ifdef _WIN32 + { + const char *p2; + p2 = strrchr(base_path, '\\'); + if (!p1 || p2 > p1) + p1 = p2; + } +#endif + if (p1) + p1++; + else + p1 = base_path; + if (p1 > p) + p = p1; + len = p - base_path; + if (len > dest_size - 1) + len = dest_size - 1; + memcpy(dest, base_path, len); + dest[len] = '\0'; + pstrcat(dest, dest_size, filename); + } +} + + +static void bdrv_register(BlockDriver *bdrv) +{ + if (!bdrv->bdrv_aio_read) { + /* add AIO emulation layer */ + bdrv->bdrv_aio_read = bdrv_aio_read_em; + bdrv->bdrv_aio_write = bdrv_aio_write_em; + bdrv->bdrv_aio_cancel = bdrv_aio_cancel_em; + bdrv->aiocb_size = sizeof(BlockDriverAIOCBSync); + } else if (!bdrv->bdrv_read && !bdrv->bdrv_pread) { + /* add synchronous IO emulation layer */ + bdrv->bdrv_read = bdrv_read_em; + bdrv->bdrv_write = bdrv_write_em; + } + bdrv->next = first_drv; + first_drv = bdrv; +} + +/* create a new block device (by default it is empty) */ +BlockDriverState *bdrv_new(const char *device_name) +{ + BlockDriverState **pbs, *bs; + + bs = qemu_mallocz(sizeof(BlockDriverState)); + if(!bs) + return NULL; + pstrcpy(bs->device_name, sizeof(bs->device_name), device_name); + if (device_name[0] != '\0') { + /* insert at the end */ + pbs = &bdrv_first; + while (*pbs != NULL) + pbs = &(*pbs)->next; + *pbs = bs; + } + return bs; +} + +BlockDriver *bdrv_find_format(const char *format_name) +{ + BlockDriver *drv1; + for(drv1 = first_drv; drv1 != NULL; drv1 = drv1->next) { + if (!strcmp(drv1->format_name, format_name)) + return drv1; + } + return NULL; +} + +int bdrv_create(BlockDriver *drv, + const char *filename, int64_t size_in_sectors, + const char *backing_file, int flags) +{ + if (!drv->bdrv_create) + return -ENOTSUP; + return drv->bdrv_create(filename, size_in_sectors, backing_file, flags); +} + +#ifdef _WIN32 +void get_tmp_filename(char *filename, int size) +{ + char temp_dir[MAX_PATH]; + + GetTempPath(MAX_PATH, temp_dir); + GetTempFileName(temp_dir, "qem", 0, filename); +} +#else +void get_tmp_filename(char *filename, int size) +{ + int fd; + /* XXX: race condition possible */ + pstrcpy(filename, size, "/tmp/vl.XXXXXX"); + fd = mkstemp(filename); + close(fd); +} +#endif + +#ifdef _WIN32 +static int is_windows_drive_prefix(const char *filename) +{ + return (((filename[0] >= 'a' && filename[0] <= 'z') || + (filename[0] >= 'A' && filename[0] <= 'Z')) && + filename[1] == ':'); +} + +static int is_windows_drive(const char *filename) +{ + if (is_windows_drive_prefix(filename) && + filename[2] == '\0') + return 1; + if (strstart(filename, "\\\\.\\", NULL) || + strstart(filename, "//./", NULL)) + return 1; + return 0; +} +#endif + +static BlockDriver *find_protocol(const char *filename) +{ + BlockDriver *drv1; + char protocol[128]; + int len; + const char *p; + +#ifdef _WIN32 + if (is_windows_drive(filename) || + is_windows_drive_prefix(filename)) + return &bdrv_raw; +#endif + p = strchr(filename, ':'); + if (!p) + return &bdrv_raw; + len = p - filename; + if (len > sizeof(protocol) - 1) + len = sizeof(protocol) - 1; + memcpy(protocol, filename, len); + protocol[len] = '\0'; + for(drv1 = first_drv; drv1 != NULL; drv1 = drv1->next) { + if (drv1->protocol_name && + !strcmp(drv1->protocol_name, protocol)) + return drv1; + } + return NULL; +} + +/* XXX: force raw format if block or character device ? It would + simplify the BSD case */ +static BlockDriver *find_image_format(const char *filename) +{ + int ret, score, score_max; + BlockDriver *drv1, *drv; + uint8_t buf[2048]; + BlockDriverState *bs; + + /* detect host devices. By convention, /dev/cdrom[N] is always + recognized as a host CDROM */ + if (strstart(filename, "/dev/cdrom", NULL)) + return &bdrv_host_device; +#ifdef _WIN32 + if (is_windows_drive(filename)) + return &bdrv_host_device; +#else + { + struct stat st; + if (stat(filename, &st) >= 0 && + (S_ISCHR(st.st_mode) || S_ISBLK(st.st_mode))) { + return &bdrv_host_device; + } + } +#endif + +#if !defined(AFFLIB) + drv = find_protocol(filename); + /* no need to test disk image formats for vvfat */ + if (drv == &bdrv_vvfat) + return drv; +#endif + + ret = bdrv_file_open(&bs, filename, BDRV_O_RDONLY); + if (ret < 0) + return NULL; + ret = bdrv_pread(bs, 0, buf, sizeof(buf)); + bdrv_delete(bs); + if (ret < 0) { + return NULL; + } + + score_max = 0; + for(drv1 = first_drv; drv1 != NULL; drv1 = drv1->next) { + if (drv1->bdrv_probe) { + score = drv1->bdrv_probe(buf, ret, filename); + if (score > score_max) { + score_max = score; + drv = drv1; + } + } + } + return drv; +} + +int bdrv_file_open(BlockDriverState **pbs, const char *filename, int flags) +{ + BlockDriverState *bs; + int ret; + + bs = bdrv_new(""); + if (!bs) + return -ENOMEM; + ret = bdrv_open2(bs, filename, flags | BDRV_O_FILE, NULL); + if (ret < 0) { + bdrv_delete(bs); + return ret; + } + *pbs = bs; + return 0; +} + +int bdrv_open(BlockDriverState *bs, const char *filename, int flags) +{ + return bdrv_open2(bs, filename, flags, NULL); +} + +int bdrv_open2(BlockDriverState *bs, const char *filename, int flags, + BlockDriver *drv) +{ + int ret, open_flags; + char tmp_filename[PATH_MAX]; + char backing_filename[PATH_MAX]; + + bs->read_only = 0; + bs->is_temporary = 0; + bs->encrypted = 0; + + if (flags & BDRV_O_SNAPSHOT) { + BlockDriverState *bs1; + int64_t total_size; + + /* if snapshot, we create a temporary backing file and open it + instead of opening 'filename' directly */ + + /* if there is a backing file, use it */ + bs1 = bdrv_new(""); + if (!bs1) { + return -ENOMEM; + } + if (bdrv_open(bs1, filename, 0) < 0) { + bdrv_delete(bs1); + return -1; + } + total_size = bdrv_getlength(bs1) >> SECTOR_BITS; + bdrv_delete(bs1); + + get_tmp_filename(tmp_filename, sizeof(tmp_filename)); + if(realpath(filename, backing_filename)==0) return -1; + if (bdrv_create(&bdrv_qcow2, tmp_filename, + total_size, backing_filename, 0) < 0) { + return -1; + } + filename = tmp_filename; + bs->is_temporary = 1; + } + + pstrcpy(bs->filename, sizeof(bs->filename), filename); + if (flags & BDRV_O_FILE) { + drv = find_protocol(filename); + if (!drv) + return -ENOENT; + } else { + if (!drv) { + drv = find_image_format(filename); + if (!drv) + return -1; + } + } + bs->drv = drv; + bs->opaque = qemu_mallocz(drv->instance_size); + if (bs->opaque == NULL && drv->instance_size > 0) + return -1; + /* Note: for compatibility, we open disk image files as RDWR, and + RDONLY as fallback */ + if (!(flags & BDRV_O_FILE)) + open_flags = BDRV_O_RDWR | (flags & BDRV_O_DIRECT); + else + open_flags = flags & ~(BDRV_O_FILE | BDRV_O_SNAPSHOT); + ret = drv->bdrv_open(bs, filename, open_flags); + if (ret == -EACCES && !(flags & BDRV_O_FILE)) { + ret = drv->bdrv_open(bs, filename, BDRV_O_RDONLY); + bs->read_only = 1; + } + if (ret < 0) { + qemu_free(bs->opaque); + bs->opaque = NULL; + bs->drv = NULL; + return ret; + } + if (drv->bdrv_getlength) { + bs->total_sectors = bdrv_getlength(bs) >> SECTOR_BITS; + } +#ifndef _WIN32 + if (bs->is_temporary) { + unlink(filename); + } +#endif + if (bs->backing_file[0] != '\0') { + /* if there is a backing file, use it */ + bs->backing_hd = bdrv_new(""); + if (!bs->backing_hd) { + fail: + bdrv_close(bs); + return -ENOMEM; + } + path_combine(backing_filename, sizeof(backing_filename), + filename, bs->backing_file); + if (bdrv_open(bs->backing_hd, backing_filename, 0) < 0) + goto fail; + } + + /* call the change callback */ + bs->media_changed = 1; + if (bs->change_cb) + bs->change_cb(bs->change_opaque); + + return 0; +} + +void bdrv_close(BlockDriverState *bs) +{ + if (bs->drv) { + if (bs->backing_hd) + bdrv_delete(bs->backing_hd); + bs->drv->bdrv_close(bs); + qemu_free(bs->opaque); +#ifdef _WIN32 + if (bs->is_temporary) { + unlink(bs->filename); + } +#endif + bs->opaque = NULL; + bs->drv = NULL; + + /* call the change callback */ + bs->media_changed = 1; + if (bs->change_cb) + bs->change_cb(bs->change_opaque); + } +} + +void bdrv_delete(BlockDriverState *bs) +{ + /* XXX: remove the driver list */ + bdrv_close(bs); + qemu_free(bs); +} + +/* commit COW file into the raw image */ +int bdrv_commit(BlockDriverState *bs) +{ + BlockDriver *drv = bs->drv; + int64_t i, total_sectors; + int n, j; + unsigned char sector[512]; + + if (!drv) + return -ENOMEDIUM; + + if (bs->read_only) { + return -EACCES; + } + + if (!bs->backing_hd) { + return -ENOTSUP; + } + + total_sectors = bdrv_getlength(bs) >> SECTOR_BITS; + for (i = 0; i < total_sectors;) { + if (drv->bdrv_is_allocated(bs, i, 65536, &n)) { + for(j = 0; j < n; j++) { + if (bdrv_read(bs, i, sector, 1) != 0) { + return -EIO; + } + + if (bdrv_write(bs->backing_hd, i, sector, 1) != 0) { + return -EIO; + } + i++; + } + } else { + i += n; + } + } + + if (drv->bdrv_make_empty) + return drv->bdrv_make_empty(bs); + + return 0; +} + +/* return < 0 if error. See bdrv_write() for the return codes */ +int bdrv_read(BlockDriverState *bs, int64_t sector_num, + uint8_t *buf, int nb_sectors) +{ + BlockDriver *drv = bs->drv; + + if (!drv) + return -ENOMEDIUM; + + if (sector_num == 0 && bs->boot_sector_enabled && nb_sectors > 0) { + memcpy(buf, bs->boot_sector_data, 512); + sector_num++; + nb_sectors--; + buf += 512; + if (nb_sectors == 0) + return 0; + } + if (drv->bdrv_pread) { + int ret, len; + len = nb_sectors * 512; + ret = drv->bdrv_pread(bs, sector_num * 512, buf, len); + if (ret < 0) + return ret; + else if (ret != len) + return -EINVAL; + else { + bs->rd_bytes += (unsigned) len; + bs->rd_ops ++; + return 0; + } + } else { + return drv->bdrv_read(bs, sector_num, buf, nb_sectors); + } +} + +/* Return < 0 if error. Important errors are: + -EIO generic I/O error (may happen for all errors) + -ENOMEDIUM No media inserted. + -EINVAL Invalid sector number or nb_sectors + -EACCES Trying to write a read-only device +*/ +int bdrv_write(BlockDriverState *bs, int64_t sector_num, + const uint8_t *buf, int nb_sectors) +{ + BlockDriver *drv = bs->drv; + if (!bs->drv) + return -ENOMEDIUM; + if (bs->read_only) + return -EACCES; + if (sector_num == 0 && bs->boot_sector_enabled && nb_sectors > 0) { + memcpy(bs->boot_sector_data, buf, 512); + } + if (drv->bdrv_pwrite) { + int ret, len; + len = nb_sectors * 512; + ret = drv->bdrv_pwrite(bs, sector_num * 512, buf, len); + if (ret < 0) + return ret; + else if (ret != len) + return -EIO; + else { + bs->wr_bytes += (unsigned) len; + bs->wr_ops ++; + return 0; + } + } else { + return drv->bdrv_write(bs, sector_num, buf, nb_sectors); + } +} + +static int bdrv_pread_em(BlockDriverState *bs, int64_t offset, + uint8_t *buf, int count1) +{ + uint8_t tmp_buf[SECTOR_SIZE]; + int len, nb_sectors, count; + int64_t sector_num; + + count = count1; + /* first read to align to sector start */ + len = (SECTOR_SIZE - offset) & (SECTOR_SIZE - 1); + if (len > count) + len = count; + sector_num = offset >> SECTOR_BITS; + if (len > 0) { + if (bdrv_read(bs, sector_num, tmp_buf, 1) < 0) + return -EIO; + memcpy(buf, tmp_buf + (offset & (SECTOR_SIZE - 1)), len); + count -= len; + if (count == 0) + return count1; + sector_num++; + buf += len; + } + + /* read the sectors "in place" */ + nb_sectors = count >> SECTOR_BITS; + if (nb_sectors > 0) { + if (bdrv_read(bs, sector_num, buf, nb_sectors) < 0) + return -EIO; + sector_num += nb_sectors; + len = nb_sectors << SECTOR_BITS; + buf += len; + count -= len; + } + + /* add data from the last sector */ + if (count > 0) { + if (bdrv_read(bs, sector_num, tmp_buf, 1) < 0) + return -EIO; + memcpy(buf, tmp_buf, count); + } + return count1; +} + +static int bdrv_pwrite_em(BlockDriverState *bs, int64_t offset, + const uint8_t *buf, int count1) +{ + uint8_t tmp_buf[SECTOR_SIZE]; + int len, nb_sectors, count; + int64_t sector_num; + + count = count1; + /* first write to align to sector start */ + len = (SECTOR_SIZE - offset) & (SECTOR_SIZE - 1); + if (len > count) + len = count; + sector_num = offset >> SECTOR_BITS; + if (len > 0) { + if (bdrv_read(bs, sector_num, tmp_buf, 1) < 0) + return -EIO; + memcpy(tmp_buf + (offset & (SECTOR_SIZE - 1)), buf, len); + if (bdrv_write(bs, sector_num, tmp_buf, 1) < 0) + return -EIO; + count -= len; + if (count == 0) + return count1; + sector_num++; + buf += len; + } + + /* write the sectors "in place" */ + nb_sectors = count >> SECTOR_BITS; + if (nb_sectors > 0) { + if (bdrv_write(bs, sector_num, buf, nb_sectors) < 0) + return -EIO; + sector_num += nb_sectors; + len = nb_sectors << SECTOR_BITS; + buf += len; + count -= len; + } + + /* add data from the last sector */ + if (count > 0) { + if (bdrv_read(bs, sector_num, tmp_buf, 1) < 0) + return -EIO; + memcpy(tmp_buf, buf, count); + if (bdrv_write(bs, sector_num, tmp_buf, 1) < 0) + return -EIO; + } + return count1; +} + +/** + * Read with byte offsets (needed only for file protocols) + */ +int bdrv_pread(BlockDriverState *bs, int64_t offset, + void *buf1, int count1) +{ + BlockDriver *drv = bs->drv; + + if (!drv) + return -ENOMEDIUM; + if (!drv->bdrv_pread) + return bdrv_pread_em(bs, offset, buf1, count1); + return drv->bdrv_pread(bs, offset, buf1, count1); +} + +/** + * Write with byte offsets (needed only for file protocols) + */ +int bdrv_pwrite(BlockDriverState *bs, int64_t offset, + const void *buf1, int count1) +{ + BlockDriver *drv = bs->drv; + + if (!drv) + return -ENOMEDIUM; + if (!drv->bdrv_pwrite) + return bdrv_pwrite_em(bs, offset, buf1, count1); + return drv->bdrv_pwrite(bs, offset, buf1, count1); +} + +/** + * Truncate file to 'offset' bytes (needed only for file protocols) + */ +int bdrv_truncate(BlockDriverState *bs, int64_t offset) +{ + BlockDriver *drv = bs->drv; + if (!drv) + return -ENOMEDIUM; + if (!drv->bdrv_truncate) + return -ENOTSUP; + return drv->bdrv_truncate(bs, offset); +} + +/** + * Length of a file in bytes. Return < 0 if error or unknown. + */ +int64_t bdrv_getlength(BlockDriverState *bs) +{ + BlockDriver *drv = bs->drv; + if (!drv) + return -ENOMEDIUM; + if (!drv->bdrv_getlength) { + /* legacy mode */ + return bs->total_sectors * SECTOR_SIZE; + } + return drv->bdrv_getlength(bs); +} + +/* return 0 as number of sectors if no device present or error */ +void bdrv_get_geometry(BlockDriverState *bs, uint64_t *nb_sectors_ptr) +{ + int64_t length; + length = bdrv_getlength(bs); + if (length < 0) + length = 0; + else + length = length >> SECTOR_BITS; + *nb_sectors_ptr = length; +} + +/* force a given boot sector. */ +void bdrv_set_boot_sector(BlockDriverState *bs, const uint8_t *data, int size) +{ + bs->boot_sector_enabled = 1; + if (size > 512) + size = 512; + memcpy(bs->boot_sector_data, data, size); + memset(bs->boot_sector_data + size, 0, 512 - size); +} + +void bdrv_set_geometry_hint(BlockDriverState *bs, + int cyls, int heads, int secs) +{ + bs->cyls = cyls; + bs->heads = heads; + bs->secs = secs; +} + +void bdrv_set_type_hint(BlockDriverState *bs, int type) +{ + bs->type = type; + bs->removable = ((type == BDRV_TYPE_CDROM || + type == BDRV_TYPE_FLOPPY)); +} + +void bdrv_set_translation_hint(BlockDriverState *bs, int translation) +{ + bs->translation = translation; +} + +void bdrv_get_geometry_hint(BlockDriverState *bs, + int *pcyls, int *pheads, int *psecs) +{ + *pcyls = bs->cyls; + *pheads = bs->heads; + *psecs = bs->secs; +} + +int bdrv_get_type_hint(BlockDriverState *bs) +{ + return bs->type; +} + +int bdrv_get_translation_hint(BlockDriverState *bs) +{ + return bs->translation; +} + +int bdrv_is_removable(BlockDriverState *bs) +{ + return bs->removable; +} + +int bdrv_is_read_only(BlockDriverState *bs) +{ + return bs->read_only; +} + +int bdrv_is_sg(BlockDriverState *bs) +{ + return bs->sg; +} + +/* XXX: no longer used */ +void bdrv_set_change_cb(BlockDriverState *bs, + void (*change_cb)(void *opaque), void *opaque) +{ + bs->change_cb = change_cb; + bs->change_opaque = opaque; +} + +int bdrv_is_encrypted(BlockDriverState *bs) +{ + if (bs->backing_hd && bs->backing_hd->encrypted) + return 1; + return bs->encrypted; +} + +int bdrv_set_key(BlockDriverState *bs, const char *key) +{ + int ret; + if (bs->backing_hd && bs->backing_hd->encrypted) { + ret = bdrv_set_key(bs->backing_hd, key); + if (ret < 0) + return ret; + if (!bs->encrypted) + return 0; + } + if (!bs->encrypted || !bs->drv || !bs->drv->bdrv_set_key) + return -1; + return bs->drv->bdrv_set_key(bs, key); +} + +void bdrv_get_format(BlockDriverState *bs, char *buf, int buf_size) +{ + if (!bs->drv) { + buf[0] = '\0'; + } else { + pstrcpy(buf, buf_size, bs->drv->format_name); + } +} + +void bdrv_iterate_format(void (*it)(void *opaque, const char *name), + void *opaque) +{ + BlockDriver *drv; + + for (drv = first_drv; drv != NULL; drv = drv->next) { + it(opaque, drv->format_name); + } +} + +BlockDriverState *bdrv_find(const char *name) +{ + BlockDriverState *bs; + + for (bs = bdrv_first; bs != NULL; bs = bs->next) { + if (!strcmp(name, bs->device_name)) + return bs; + } + return NULL; +} + +void bdrv_iterate(void (*it)(void *opaque, const char *name), void *opaque) +{ + BlockDriverState *bs; + + for (bs = bdrv_first; bs != NULL; bs = bs->next) { + it(opaque, bs->device_name); + } +} + +const char *bdrv_get_device_name(BlockDriverState *bs) +{ + return bs->device_name; +} + +void bdrv_flush(BlockDriverState *bs) +{ + if (bs->drv->bdrv_flush) + bs->drv->bdrv_flush(bs); + if (bs->backing_hd) + bdrv_flush(bs->backing_hd); +} + +#ifndef QEMU_IMG +void bdrv_info(void) +{ + BlockDriverState *bs; + + for (bs = bdrv_first; bs != NULL; bs = bs->next) { + term_printf("%s:", bs->device_name); + term_printf(" type="); + switch(bs->type) { + case BDRV_TYPE_HD: + term_printf("hd"); + break; + case BDRV_TYPE_CDROM: + term_printf("cdrom"); + break; + case BDRV_TYPE_FLOPPY: + term_printf("floppy"); + break; + } + term_printf(" removable=%d", bs->removable); + if (bs->removable) { + term_printf(" locked=%d", bs->locked); + } + if (bs->drv) { + term_printf(" file="); + term_print_filename(bs->filename); + if (bs->backing_file[0] != '\0') { + term_printf(" backing_file="); + term_print_filename(bs->backing_file); + } + term_printf(" ro=%d", bs->read_only); + term_printf(" drv=%s", bs->drv->format_name); + if (bs->encrypted) + term_printf(" encrypted"); + } else { + term_printf(" [not inserted]"); + } + term_printf("\n"); + } +} + +/* The "info blockstats" command. */ +void bdrv_info_stats (void) +{ + BlockDriverState *bs; + + for (bs = bdrv_first; bs != NULL; bs = bs->next) { + term_printf ("%s:" + " rd_bytes=%" PRIu64 + " wr_bytes=%" PRIu64 + " rd_operations=%" PRIu64 + " wr_operations=%" PRIu64 + "\n", + bs->device_name, + bs->rd_bytes, bs->wr_bytes, + bs->rd_ops, bs->wr_ops); + } +} +#endif + +void bdrv_get_backing_filename(BlockDriverState *bs, + char *filename, int filename_size) +{ + if (!bs->backing_hd) { + pstrcpy(filename, filename_size, ""); + } else { + pstrcpy(filename, filename_size, bs->backing_file); + } +} + +int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num, + const uint8_t *buf, int nb_sectors) +{ + BlockDriver *drv = bs->drv; + if (!drv) + return -ENOMEDIUM; + if (!drv->bdrv_write_compressed) + return -ENOTSUP; + return drv->bdrv_write_compressed(bs, sector_num, buf, nb_sectors); +} + +int bdrv_get_info(BlockDriverState *bs, BlockDriverInfo *bdi) +{ + BlockDriver *drv = bs->drv; + if (!drv) + return -ENOMEDIUM; + if (!drv->bdrv_get_info) + return -ENOTSUP; + memset(bdi, 0, sizeof(*bdi)); + return drv->bdrv_get_info(bs, bdi); +} + +/**************************************************************/ +/* handling of snapshots */ + +int bdrv_snapshot_create(BlockDriverState *bs, + QEMUSnapshotInfo *sn_info) +{ + BlockDriver *drv = bs->drv; + if (!drv) + return -ENOMEDIUM; + if (!drv->bdrv_snapshot_create) + return -ENOTSUP; + return drv->bdrv_snapshot_create(bs, sn_info); +} + +int bdrv_snapshot_goto(BlockDriverState *bs, + const char *snapshot_id) +{ + BlockDriver *drv = bs->drv; + if (!drv) + return -ENOMEDIUM; + if (!drv->bdrv_snapshot_goto) + return -ENOTSUP; + return drv->bdrv_snapshot_goto(bs, snapshot_id); +} + +int bdrv_snapshot_delete(BlockDriverState *bs, const char *snapshot_id) +{ + BlockDriver *drv = bs->drv; + if (!drv) + return -ENOMEDIUM; + if (!drv->bdrv_snapshot_delete) + return -ENOTSUP; + return drv->bdrv_snapshot_delete(bs, snapshot_id); +} + +int bdrv_snapshot_list(BlockDriverState *bs, + QEMUSnapshotInfo **psn_info) +{ + BlockDriver *drv = bs->drv; + if (!drv) + return -ENOMEDIUM; + if (!drv->bdrv_snapshot_list) + return -ENOTSUP; + return drv->bdrv_snapshot_list(bs, psn_info); +} + +#define NB_SUFFIXES 4 + +char *get_human_readable_size(char *buf, int buf_size, int64_t size) +{ + static const char suffixes[NB_SUFFIXES] = "KMGT"; + int64_t base; + int i; + + if (size <= 999) { + snprintf(buf, buf_size, "%" PRId64, size); + } else { + base = 1024; + for(i = 0; i < NB_SUFFIXES; i++) { + if (size < (10 * base)) { + snprintf(buf, buf_size, "%0.1f%c", + (double)size / base, + suffixes[i]); + break; + } else if (size < (1000 * base) || i == (NB_SUFFIXES - 1)) { + snprintf(buf, buf_size, "%" PRId64 "%c", + ((size + (base >> 1)) / base), + suffixes[i]); + break; + } + base = base * 1024; + } + } + return buf; +} + +char *bdrv_snapshot_dump(char *buf, int buf_size, QEMUSnapshotInfo *sn) +{ + char buf1[128], date_buf[128], clock_buf[128]; +#ifdef _WIN32 + struct tm *ptm; +#else + struct tm tm; +#endif + time_t ti; + int64_t secs; + + if (!sn) { + snprintf(buf, buf_size, + "%-10s%-20s%7s%20s%15s", + "ID", "TAG", "VM SIZE", "DATE", "VM CLOCK"); + } else { + ti = sn->date_sec; +#ifdef _WIN32 + ptm = localtime(&ti); + strftime(date_buf, sizeof(date_buf), + "%Y-%m-%d %H:%M:%S", ptm); +#else + localtime_r(&ti, &tm); + strftime(date_buf, sizeof(date_buf), + "%Y-%m-%d %H:%M:%S", &tm); +#endif + secs = sn->vm_clock_nsec / 1000000000; + snprintf(clock_buf, sizeof(clock_buf), + "%02d:%02d:%02d.%03d", + (int)(secs / 3600), + (int)((secs / 60) % 60), + (int)(secs % 60), + (int)((sn->vm_clock_nsec / 1000000) % 1000)); + snprintf(buf, buf_size, + "%-10s%-20s%7s%20s%15s", + sn->id_str, sn->name, + get_human_readable_size(buf1, sizeof(buf1), sn->vm_state_size), + date_buf, + clock_buf); + } + return buf; +} + + +/**************************************************************/ +/* async I/Os */ + +BlockDriverAIOCB *bdrv_aio_read(BlockDriverState *bs, int64_t sector_num, + uint8_t *buf, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque) +{ + BlockDriver *drv = bs->drv; + BlockDriverAIOCB *ret; + + if (!drv) + return NULL; + + /* XXX: we assume that nb_sectors == 0 is suppored by the async read */ + if (sector_num == 0 && bs->boot_sector_enabled && nb_sectors > 0) { + memcpy(buf, bs->boot_sector_data, 512); + sector_num++; + nb_sectors--; + buf += 512; + } + + ret = drv->bdrv_aio_read(bs, sector_num, buf, nb_sectors, cb, opaque); + + if (ret) { + /* Update stats even though technically transfer has not happened. */ + bs->rd_bytes += (unsigned) nb_sectors * SECTOR_SIZE; + bs->rd_ops ++; + } + + return ret; +} + +BlockDriverAIOCB *bdrv_aio_write(BlockDriverState *bs, int64_t sector_num, + const uint8_t *buf, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque) +{ + BlockDriver *drv = bs->drv; + BlockDriverAIOCB *ret; + + if (!drv) + return NULL; + if (bs->read_only) + return NULL; + if (sector_num == 0 && bs->boot_sector_enabled && nb_sectors > 0) { + memcpy(bs->boot_sector_data, buf, 512); + } + + ret = drv->bdrv_aio_write(bs, sector_num, buf, nb_sectors, cb, opaque); + + if (ret) { + /* Update stats even though technically transfer has not happened. */ + bs->wr_bytes += (unsigned) nb_sectors * SECTOR_SIZE; + bs->wr_ops ++; + } + + return ret; +} + +void bdrv_aio_cancel(BlockDriverAIOCB *acb) +{ + BlockDriver *drv = acb->bs->drv; + + drv->bdrv_aio_cancel(acb); +} + + +/**************************************************************/ +/* async block device emulation */ + +#ifdef QEMU_IMG +static BlockDriverAIOCB *bdrv_aio_read_em(BlockDriverState *bs, + int64_t sector_num, uint8_t *buf, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque) +{ + int ret; + ret = bdrv_read(bs, sector_num, buf, nb_sectors); + cb(opaque, ret); + return NULL; +} + +static BlockDriverAIOCB *bdrv_aio_write_em(BlockDriverState *bs, + int64_t sector_num, const uint8_t *buf, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque) +{ + int ret; + ret = bdrv_write(bs, sector_num, buf, nb_sectors); + cb(opaque, ret); + return NULL; +} + +static void bdrv_aio_cancel_em(BlockDriverAIOCB *acb) +{ +} +#else +static void bdrv_aio_bh_cb(void *opaque) +{ + BlockDriverAIOCBSync *acb = opaque; + acb->common.cb(acb->common.opaque, acb->ret); + qemu_aio_release(acb); +} + +static BlockDriverAIOCB *bdrv_aio_read_em(BlockDriverState *bs, + int64_t sector_num, uint8_t *buf, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque) +{ + BlockDriverAIOCBSync *acb; + int ret; + + acb = qemu_aio_get(bs, cb, opaque); + if (!acb->bh) + acb->bh = qemu_bh_new(bdrv_aio_bh_cb, acb); + ret = bdrv_read(bs, sector_num, buf, nb_sectors); + acb->ret = ret; + qemu_bh_schedule(acb->bh); + return &acb->common; +} + +static BlockDriverAIOCB *bdrv_aio_write_em(BlockDriverState *bs, + int64_t sector_num, const uint8_t *buf, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque) +{ + BlockDriverAIOCBSync *acb; + int ret; + + acb = qemu_aio_get(bs, cb, opaque); + if (!acb->bh) + acb->bh = qemu_bh_new(bdrv_aio_bh_cb, acb); + ret = bdrv_write(bs, sector_num, buf, nb_sectors); + acb->ret = ret; + qemu_bh_schedule(acb->bh); + return &acb->common; +} + +static void bdrv_aio_cancel_em(BlockDriverAIOCB *blockacb) +{ + BlockDriverAIOCBSync *acb = (BlockDriverAIOCBSync *)blockacb; + qemu_bh_cancel(acb->bh); + qemu_aio_release(acb); +} +#endif /* !QEMU_IMG */ + +/**************************************************************/ +/* sync block device emulation */ + +static void bdrv_rw_em_cb(void *opaque, int ret) +{ + *(int *)opaque = ret; +} + +#define NOT_DONE 0x7fffffff + +static int bdrv_read_em(BlockDriverState *bs, int64_t sector_num, + uint8_t *buf, int nb_sectors) +{ + int async_ret; + BlockDriverAIOCB *acb; + + async_ret = NOT_DONE; + qemu_aio_wait_start(); + acb = bdrv_aio_read(bs, sector_num, buf, nb_sectors, + bdrv_rw_em_cb, &async_ret); + if (acb == NULL) { + qemu_aio_wait_end(); + return -1; + } + while (async_ret == NOT_DONE) { + qemu_aio_wait(); + } + qemu_aio_wait_end(); + return async_ret; +} + +static int bdrv_write_em(BlockDriverState *bs, int64_t sector_num, + const uint8_t *buf, int nb_sectors) +{ + int async_ret; + BlockDriverAIOCB *acb; + + async_ret = NOT_DONE; + qemu_aio_wait_start(); + acb = bdrv_aio_write(bs, sector_num, buf, nb_sectors, + bdrv_rw_em_cb, &async_ret); + if (acb == NULL) { + qemu_aio_wait_end(); + return -1; + } + while (async_ret == NOT_DONE) { + qemu_aio_wait(); + } + qemu_aio_wait_end(); + return async_ret; +} + +void bdrv_init(void) +{ + // bdrv_register(&bdrv_raw); + // bdrv_register(&bdrv_host_device); +#ifndef _WIN32 + // bdrv_register(&bdrv_cow); +#endif + // bdrv_register(&bdrv_qcow); + bdrv_register(&bdrv_vmdk); + //bdrv_register(&bdrv_cloop); + bdrv_register(&bdrv_dmg); + //bdrv_register(&bdrv_bochs); + //bdrv_register(&bdrv_vpc); + //bdrv_register(&bdrv_vvfat); + //bdrv_register(&bdrv_qcow2); + bdrv_register(&bdrv_parallels); +} + +void *qemu_aio_get(BlockDriverState *bs, BlockDriverCompletionFunc *cb, + void *opaque) +{ + BlockDriver *drv; + BlockDriverAIOCB *acb; + + drv = bs->drv; + if (drv->free_aiocb) { + acb = drv->free_aiocb; + drv->free_aiocb = acb->next; + } else { + acb = qemu_mallocz(drv->aiocb_size); + if (!acb) + return NULL; + } + acb->bs = bs; + acb->cb = cb; + acb->opaque = opaque; + return acb; +} + +void qemu_aio_release(void *p) +{ + BlockDriverAIOCB *acb = p; + BlockDriver *drv = acb->bs->drv; + acb->next = drv->free_aiocb; + drv->free_aiocb = acb; +} + +/**************************************************************/ +/* removable device support */ + +/** + * Return TRUE if the media is present + */ +int bdrv_is_inserted(BlockDriverState *bs) +{ + BlockDriver *drv = bs->drv; + int ret; + if (!drv) + return 0; + if (!drv->bdrv_is_inserted) + return 1; + ret = drv->bdrv_is_inserted(bs); + return ret; +} + +/** + * Return TRUE if the media changed since the last call to this + * function. It is currently only used for floppy disks + */ +int bdrv_media_changed(BlockDriverState *bs) +{ + BlockDriver *drv = bs->drv; + int ret; + + if (!drv || !drv->bdrv_media_changed) + ret = -ENOTSUP; + else + ret = drv->bdrv_media_changed(bs); + if (ret == -ENOTSUP) + ret = bs->media_changed; + bs->media_changed = 0; + return ret; +} + +/** + * If eject_flag is TRUE, eject the media. Otherwise, close the tray + */ +void bdrv_eject(BlockDriverState *bs, int eject_flag) +{ + BlockDriver *drv = bs->drv; + int ret; + + if (!drv || !drv->bdrv_eject) { + ret = -ENOTSUP; + } else { + ret = drv->bdrv_eject(bs, eject_flag); + } + if (ret == -ENOTSUP) { + if (eject_flag) + bdrv_close(bs); + } +} + +int bdrv_is_locked(BlockDriverState *bs) +{ + return bs->locked; +} + +/** + * Lock or unlock the media (if it is locked, the user won't be able + * to eject it manually). + */ +void bdrv_set_locked(BlockDriverState *bs, int locked) +{ + BlockDriver *drv = bs->drv; + + bs->locked = locked; + if (drv && drv->bdrv_set_locked) { + drv->bdrv_set_locked(bs, locked); + } +} + +/* needed for generic scsi interface */ + +int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf) +{ + BlockDriver *drv = bs->drv; + + if (drv && drv->bdrv_ioctl) + return drv->bdrv_ioctl(bs, req, buf); + return -ENOTSUP; +} diff --git a/lib/qemu/block.h b/lib/qemu/block.h new file mode 100644 index 0000000..b730505 --- /dev/null +++ b/lib/qemu/block.h @@ -0,0 +1,160 @@ +#ifndef BLOCK_H +#define BLOCK_H + +/* block.c */ +typedef struct BlockDriver BlockDriver; + +extern BlockDriver bdrv_raw; +extern BlockDriver bdrv_host_device; +extern BlockDriver bdrv_cow; +extern BlockDriver bdrv_qcow; +extern BlockDriver bdrv_vmdk; +extern BlockDriver bdrv_cloop; +extern BlockDriver bdrv_dmg; +extern BlockDriver bdrv_bochs; +extern BlockDriver bdrv_vpc; +extern BlockDriver bdrv_vvfat; +extern BlockDriver bdrv_qcow2; +extern BlockDriver bdrv_parallels; + +typedef struct BlockDriverInfo { + /* in bytes, 0 if irrelevant */ + int cluster_size; + /* offset at which the VM state can be saved (0 if not possible) */ + int64_t vm_state_offset; +} BlockDriverInfo; + +typedef struct QEMUSnapshotInfo { + char id_str[128]; /* unique snapshot id */ + /* the following fields are informative. They are not needed for + the consistency of the snapshot */ + char name[256]; /* user choosen name */ + uint32_t vm_state_size; /* VM state info size */ + uint32_t date_sec; /* UTC date of the snapshot */ + uint32_t date_nsec; + uint64_t vm_clock_nsec; /* VM clock relative to boot */ +} QEMUSnapshotInfo; + +#define BDRV_O_RDONLY 0x0000 +#define BDRV_O_RDWR 0x0002 +#define BDRV_O_ACCESS 0x0003 +#define BDRV_O_CREAT 0x0004 /* create an empty file */ +#define BDRV_O_SNAPSHOT 0x0008 /* open the file read only and save writes in a snapshot */ +#define BDRV_O_FILE 0x0010 /* open as a raw file (do not try to + use a disk image format on top of + it (default for + bdrv_file_open()) */ +#define BDRV_O_DIRECT 0x0020 + +#ifndef QEMU_IMG +void bdrv_info(void); +void bdrv_info_stats(void); +#endif + +void bdrv_init(void); +BlockDriver *bdrv_find_format(const char *format_name); +int bdrv_create(BlockDriver *drv, + const char *filename, int64_t size_in_sectors, + const char *backing_file, int flags); +BlockDriverState *bdrv_new(const char *device_name); +void bdrv_delete(BlockDriverState *bs); +int bdrv_file_open(BlockDriverState **pbs, const char *filename, int flags); +int bdrv_open(BlockDriverState *bs, const char *filename, int flags); +int bdrv_open2(BlockDriverState *bs, const char *filename, int flags, + BlockDriver *drv); +void bdrv_close(BlockDriverState *bs); +int bdrv_read(BlockDriverState *bs, int64_t sector_num, + uint8_t *buf, int nb_sectors); +int bdrv_write(BlockDriverState *bs, int64_t sector_num, + const uint8_t *buf, int nb_sectors); +int bdrv_pread(BlockDriverState *bs, int64_t offset, + void *buf, int count); +int bdrv_pwrite(BlockDriverState *bs, int64_t offset, + const void *buf, int count); +int bdrv_truncate(BlockDriverState *bs, int64_t offset); +int64_t bdrv_getlength(BlockDriverState *bs); +void bdrv_get_geometry(BlockDriverState *bs, uint64_t *nb_sectors_ptr); +int bdrv_commit(BlockDriverState *bs); +void bdrv_set_boot_sector(BlockDriverState *bs, const uint8_t *data, int size); +/* async block I/O */ +typedef struct BlockDriverAIOCB BlockDriverAIOCB; +typedef void BlockDriverCompletionFunc(void *opaque, int ret); + +BlockDriverAIOCB *bdrv_aio_read(BlockDriverState *bs, int64_t sector_num, + uint8_t *buf, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque); +BlockDriverAIOCB *bdrv_aio_write(BlockDriverState *bs, int64_t sector_num, + const uint8_t *buf, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque); +void bdrv_aio_cancel(BlockDriverAIOCB *acb); + +void qemu_aio_init(void); +void qemu_aio_poll(void); +void qemu_aio_flush(void); +void qemu_aio_wait_start(void); +void qemu_aio_wait(void); +void qemu_aio_wait_end(void); + +int qemu_key_check(BlockDriverState *bs, const char *name); + +/* Ensure contents are flushed to disk. */ +void bdrv_flush(BlockDriverState *bs); + +#define BDRV_TYPE_HD 0 +#define BDRV_TYPE_CDROM 1 +#define BDRV_TYPE_FLOPPY 2 +#define BIOS_ATA_TRANSLATION_AUTO 0 +#define BIOS_ATA_TRANSLATION_NONE 1 +#define BIOS_ATA_TRANSLATION_LBA 2 +#define BIOS_ATA_TRANSLATION_LARGE 3 +#define BIOS_ATA_TRANSLATION_RECHS 4 + +void bdrv_set_geometry_hint(BlockDriverState *bs, + int cyls, int heads, int secs); +void bdrv_set_type_hint(BlockDriverState *bs, int type); +void bdrv_set_translation_hint(BlockDriverState *bs, int translation); +void bdrv_get_geometry_hint(BlockDriverState *bs, + int *pcyls, int *pheads, int *psecs); +int bdrv_get_type_hint(BlockDriverState *bs); +int bdrv_get_translation_hint(BlockDriverState *bs); +int bdrv_is_removable(BlockDriverState *bs); +int bdrv_is_read_only(BlockDriverState *bs); +int bdrv_is_sg(BlockDriverState *bs); +int bdrv_is_inserted(BlockDriverState *bs); +int bdrv_media_changed(BlockDriverState *bs); +int bdrv_is_locked(BlockDriverState *bs); +void bdrv_set_locked(BlockDriverState *bs, int locked); +void bdrv_eject(BlockDriverState *bs, int eject_flag); +void bdrv_set_change_cb(BlockDriverState *bs, + void (*change_cb)(void *opaque), void *opaque); +void bdrv_get_format(BlockDriverState *bs, char *buf, int buf_size); +BlockDriverState *bdrv_find(const char *name); +void bdrv_iterate(void (*it)(void *opaque, const char *name), void *opaque); +int bdrv_is_encrypted(BlockDriverState *bs); +int bdrv_set_key(BlockDriverState *bs, const char *key); +void bdrv_iterate_format(void (*it)(void *opaque, const char *name), + void *opaque); +const char *bdrv_get_device_name(BlockDriverState *bs); +int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num, + const uint8_t *buf, int nb_sectors); +int bdrv_get_info(BlockDriverState *bs, BlockDriverInfo *bdi); + +void bdrv_get_backing_filename(BlockDriverState *bs, + char *filename, int filename_size); +int bdrv_snapshot_create(BlockDriverState *bs, + QEMUSnapshotInfo *sn_info); +int bdrv_snapshot_goto(BlockDriverState *bs, + const char *snapshot_id); +int bdrv_snapshot_delete(BlockDriverState *bs, const char *snapshot_id); +int bdrv_snapshot_list(BlockDriverState *bs, + QEMUSnapshotInfo **psn_info); +char *bdrv_snapshot_dump(char *buf, int buf_size, QEMUSnapshotInfo *sn); +int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf); + +char *get_human_readable_size(char *buf, int buf_size, int64_t size); +int path_is_absolute(const char *path); +void path_combine(char *dest, int dest_size, + const char *base_path, + const char *filename); + +#endif diff --git a/lib/qemu/block_int.h b/lib/qemu/block_int.h new file mode 100644 index 0000000..564c8db --- /dev/null +++ b/lib/qemu/block_int.h @@ -0,0 +1,150 @@ +/* + * QEMU System Emulator block driver + * + * Copyright (c) 2003 Fabrice Bellard + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef BLOCK_INT_H +#define BLOCK_INT_H + +#include "block.h" + +#define BLOCK_FLAG_ENCRYPT 1 +#define BLOCK_FLAG_COMPRESS 2 +#define BLOCK_FLAG_COMPAT6 4 + +struct BlockDriver { + const char *format_name; + int instance_size; + int (*bdrv_probe)(const uint8_t *buf, int buf_size, const char *filename); + int (*bdrv_open)(BlockDriverState *bs, const char *filename, int flags); + int (*bdrv_read)(BlockDriverState *bs, int64_t sector_num, + uint8_t *buf, int nb_sectors); + int (*bdrv_write)(BlockDriverState *bs, int64_t sector_num, + const uint8_t *buf, int nb_sectors); + void (*bdrv_close)(BlockDriverState *bs); + int (*bdrv_create)(const char *filename, int64_t total_sectors, + const char *backing_file, int flags); + void (*bdrv_flush)(BlockDriverState *bs); + int (*bdrv_is_allocated)(BlockDriverState *bs, int64_t sector_num, + int nb_sectors, int *pnum); + int (*bdrv_set_key)(BlockDriverState *bs, const char *key); + int (*bdrv_make_empty)(BlockDriverState *bs); + /* aio */ + BlockDriverAIOCB *(*bdrv_aio_read)(BlockDriverState *bs, + int64_t sector_num, uint8_t *buf, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque); + BlockDriverAIOCB *(*bdrv_aio_write)(BlockDriverState *bs, + int64_t sector_num, const uint8_t *buf, int nb_sectors, + BlockDriverCompletionFunc *cb, void *opaque); + void (*bdrv_aio_cancel)(BlockDriverAIOCB *acb); + int aiocb_size; + + const char *protocol_name; + int (*bdrv_pread)(BlockDriverState *bs, int64_t offset, + uint8_t *buf, int count); + int (*bdrv_pwrite)(BlockDriverState *bs, int64_t offset, + const uint8_t *buf, int count); + int (*bdrv_truncate)(BlockDriverState *bs, int64_t offset); + int64_t (*bdrv_getlength)(BlockDriverState *bs); + int (*bdrv_write_compressed)(BlockDriverState *bs, int64_t sector_num, + const uint8_t *buf, int nb_sectors); + + int (*bdrv_snapshot_create)(BlockDriverState *bs, + QEMUSnapshotInfo *sn_info); + int (*bdrv_snapshot_goto)(BlockDriverState *bs, + const char *snapshot_id); + int (*bdrv_snapshot_delete)(BlockDriverState *bs, const char *snapshot_id); + int (*bdrv_snapshot_list)(BlockDriverState *bs, + QEMUSnapshotInfo **psn_info); + int (*bdrv_get_info)(BlockDriverState *bs, BlockDriverInfo *bdi); + + /* removable device specific */ + int (*bdrv_is_inserted)(BlockDriverState *bs); + int (*bdrv_media_changed)(BlockDriverState *bs); + int (*bdrv_eject)(BlockDriverState *bs, int eject_flag); + int (*bdrv_set_locked)(BlockDriverState *bs, int locked); + + /* to control generic scsi devices */ + int (*bdrv_ioctl)(BlockDriverState *bs, unsigned long int req, void *buf); + + BlockDriverAIOCB *free_aiocb; + struct BlockDriver *next; +}; + +struct BlockDriverState { + int64_t total_sectors; /* if we are reading a disk image, give its + size in sectors */ + int read_only; /* if true, the media is read only */ + int removable; /* if true, the media can be removed */ + int locked; /* if true, the media cannot temporarily be ejected */ + int encrypted; /* if true, the media is encrypted */ + int sg; /* if true, the device is a /dev/sg* */ + /* event callback when inserting/removing */ + void (*change_cb)(void *opaque); + void *change_opaque; + + BlockDriver *drv; /* NULL means no media */ + void *opaque; + + int boot_sector_enabled; + uint8_t boot_sector_data[512]; + + char filename[1024]; + char backing_file[1024]; /* if non zero, the image is a diff of + this file image */ + int is_temporary; + int media_changed; + + BlockDriverState *backing_hd; + /* async read/write emulation */ + + void *sync_aiocb; + + /* I/O stats (display with "info blockstats"). */ + uint64_t rd_bytes; + uint64_t wr_bytes; + uint64_t rd_ops; + uint64_t wr_ops; + + /* NOTE: the following infos are only hints for real hardware + drivers. They are not used by the block driver */ + int cyls, heads, secs, translation; + int type; + char device_name[32]; + BlockDriverState *next; +}; + +struct BlockDriverAIOCB { + BlockDriverState *bs; + BlockDriverCompletionFunc *cb; + void *opaque; + BlockDriverAIOCB *next; +}; + +void get_tmp_filename(char *filename, int size); + +void *qemu_aio_get(BlockDriverState *bs, BlockDriverCompletionFunc *cb, + void *opaque); +void qemu_aio_release(void *p); + +extern BlockDriverState *bdrv_first; // slg change + +#endif /* BLOCK_INT_H */ diff --git a/lib/qemu/bswap.h b/lib/qemu/bswap.h new file mode 100644 index 0000000..970c1bb --- /dev/null +++ b/lib/qemu/bswap.h @@ -0,0 +1,202 @@ +#ifndef BSWAP_H +#define BSWAP_H + +#include "config-host.h" + +#include + +#ifdef HAVE_BYTESWAP_H +#include +#else + +#define bswap_16(x) \ +({ \ + uint16_t __x = (x); \ + ((uint16_t)( \ + (((uint16_t)(__x) & (uint16_t)0x00ffU) << 8) | \ + (((uint16_t)(__x) & (uint16_t)0xff00U) >> 8) )); \ +}) + +#define bswap_32(x) \ +({ \ + uint32_t __x = (x); \ + ((uint32_t)( \ + (((uint32_t)(__x) & (uint32_t)0x000000ffUL) << 24) | \ + (((uint32_t)(__x) & (uint32_t)0x0000ff00UL) << 8) | \ + (((uint32_t)(__x) & (uint32_t)0x00ff0000UL) >> 8) | \ + (((uint32_t)(__x) & (uint32_t)0xff000000UL) >> 24) )); \ +}) + +#define bswap_64(x) \ +({ \ + uint64_t __x = (x); \ + ((uint64_t)( \ + (uint64_t)(((uint64_t)(__x) & (uint64_t)0x00000000000000ffULL) << 56) | \ + (uint64_t)(((uint64_t)(__x) & (uint64_t)0x000000000000ff00ULL) << 40) | \ + (uint64_t)(((uint64_t)(__x) & (uint64_t)0x0000000000ff0000ULL) << 24) | \ + (uint64_t)(((uint64_t)(__x) & (uint64_t)0x00000000ff000000ULL) << 8) | \ + (uint64_t)(((uint64_t)(__x) & (uint64_t)0x000000ff00000000ULL) >> 8) | \ + (uint64_t)(((uint64_t)(__x) & (uint64_t)0x0000ff0000000000ULL) >> 24) | \ + (uint64_t)(((uint64_t)(__x) & (uint64_t)0x00ff000000000000ULL) >> 40) | \ + (uint64_t)(((uint64_t)(__x) & (uint64_t)0xff00000000000000ULL) >> 56) )); \ +}) + +#endif /* !HAVE_BYTESWAP_H */ + +static inline uint16_t bswap16(uint16_t x) +{ + return bswap_16(x); +} + +static inline uint32_t bswap32(uint32_t x) +{ + return bswap_32(x); +} + +static inline uint64_t bswap64(uint64_t x) +{ + return bswap_64(x); +} + +static inline void bswap16s(uint16_t *s) +{ + *s = bswap16(*s); +} + +static inline void bswap32s(uint32_t *s) +{ + *s = bswap32(*s); +} + +static inline void bswap64s(uint64_t *s) +{ + *s = bswap64(*s); +} + +#if defined(WORDS_BIGENDIAN) +#define be_bswap(v, size) (v) +#define le_bswap(v, size) bswap ## size(v) +#define be_bswaps(v, size) +#define le_bswaps(p, size) *p = bswap ## size(*p); +#else +#define le_bswap(v, size) (v) +#define be_bswap(v, size) bswap ## size(v) +#define le_bswaps(v, size) +#define be_bswaps(p, size) *p = bswap ## size(*p); +#endif + +#define CPU_CONVERT(endian, size, type)\ +static inline type endian ## size ## _to_cpu(type v)\ +{\ + return endian ## _bswap(v, size);\ +}\ +\ +static inline type cpu_to_ ## endian ## size(type v)\ +{\ + return endian ## _bswap(v, size);\ +}\ +\ +static inline void endian ## size ## _to_cpus(type *p)\ +{\ + endian ## _bswaps(p, size)\ +}\ +\ +static inline void cpu_to_ ## endian ## size ## s(type *p)\ +{\ + endian ## _bswaps(p, size)\ +}\ +\ +static inline type endian ## size ## _to_cpup(const type *p)\ +{\ + return endian ## size ## _to_cpu(*p);\ +}\ +\ +static inline void cpu_to_ ## endian ## size ## w(type *p, type v)\ +{\ + *p = cpu_to_ ## endian ## size(v);\ +} + +CPU_CONVERT(be, 16, uint16_t) +CPU_CONVERT(be, 32, uint32_t) +CPU_CONVERT(be, 64, uint64_t) + +CPU_CONVERT(le, 16, uint16_t) +CPU_CONVERT(le, 32, uint32_t) +CPU_CONVERT(le, 64, uint64_t) + +/* unaligned versions (optimized for frequent unaligned accesses)*/ + +#if defined(__i386__) || defined(__powerpc__) + +#define cpu_to_le16wu(p, v) cpu_to_le16w(p, v) +#define cpu_to_le32wu(p, v) cpu_to_le32w(p, v) +#define le16_to_cpupu(p) le16_to_cpup(p) +#define le32_to_cpupu(p) le32_to_cpup(p) + +#define cpu_to_be16wu(p, v) cpu_to_be16w(p, v) +#define cpu_to_be32wu(p, v) cpu_to_be32w(p, v) + +#else + +static inline void cpu_to_le16wu(uint16_t *p, uint16_t v) +{ + uint8_t *p1 = (uint8_t *)p; + + p1[0] = v; + p1[1] = v >> 8; +} + +static inline void cpu_to_le32wu(uint32_t *p, uint32_t v) +{ + uint8_t *p1 = (uint8_t *)p; + + p1[0] = v; + p1[1] = v >> 8; + p1[2] = v >> 16; + p1[3] = v >> 24; +} + +static inline uint16_t le16_to_cpupu(const uint16_t *p) +{ + const uint8_t *p1 = (const uint8_t *)p; + return p1[0] | (p1[1] << 8); +} + +static inline uint32_t le32_to_cpupu(const uint32_t *p) +{ + const uint8_t *p1 = (const uint8_t *)p; + return p1[0] | (p1[1] << 8) | (p1[2] << 16) | (p1[3] << 24); +} + +static inline void cpu_to_be16wu(uint16_t *p, uint16_t v) +{ + uint8_t *p1 = (uint8_t *)p; + + p1[0] = v >> 8; + p1[1] = v; +} + +static inline void cpu_to_be32wu(uint32_t *p, uint32_t v) +{ + uint8_t *p1 = (uint8_t *)p; + + p1[0] = v >> 24; + p1[1] = v >> 16; + p1[2] = v >> 8; + p1[3] = v; +} + +#endif + +#ifdef WORDS_BIGENDIAN +#define cpu_to_32wu cpu_to_be32wu +#else +#define cpu_to_32wu cpu_to_le32wu +#endif + +#undef le_bswap +#undef be_bswap +#undef le_bswaps +#undef be_bswaps + +#endif /* BSWAP_H */ diff --git a/lib/qemu/config-host.h b/lib/qemu/config-host.h new file mode 100644 index 0000000..e69de29 diff --git a/lib/qemu/console.h b/lib/qemu/console.h new file mode 100644 index 0000000..e69de29 diff --git a/lib/qemu/exec-all.h b/lib/qemu/exec-all.h new file mode 100644 index 0000000..e69de29 diff --git a/lib/qemu/osdep.h b/lib/qemu/osdep.h new file mode 100644 index 0000000..bc513ad --- /dev/null +++ b/lib/qemu/osdep.h @@ -0,0 +1,72 @@ +#ifndef QEMU_OSDEP_H +#define QEMU_OSDEP_H + +#include + +#ifndef glue +#define xglue(x, y) x ## y +#define glue(x, y) xglue(x, y) +#define stringify(s) tostring(s) +#define tostring(s) #s +#endif + +#ifndef likely +#if __GNUC__ < 3 +#define __builtin_expect(x, n) (x) +#endif + +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) +#endif + +#ifndef MIN +#define MIN(a, b) (((a) < (b)) ? (a) : (b)) +#endif +#ifndef MAX +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) +#endif + +#ifndef always_inline +#if (__GNUC__ < 3) || defined(__APPLE__) +#define always_inline inline +#else +#define always_inline __attribute__ (( always_inline )) __inline__ +#endif +#endif +#define inline always_inline + +#ifdef __i386__ +#define REGPARM(n) __attribute((regparm(n))) +#else +#define REGPARM(n) +#endif + +#define qemu_printf printf + +void *qemu_malloc(size_t size); +void *qemu_mallocz(size_t size); +void qemu_free(void *ptr); +char *qemu_strdup(const char *str); + +void *qemu_memalign(size_t alignment, size_t size); +void *qemu_vmalloc(size_t size); +void qemu_vfree(void *ptr); + +void *get_mmap_addr(unsigned long size); + +int qemu_create_pidfile(const char *filename); + +#ifdef _WIN32 +int ffs(int i); + +typedef struct { + long tv_sec; + long tv_usec; +} qemu_timeval; +int qemu_gettimeofday(qemu_timeval *tp); +#else +typedef struct timeval qemu_timeval; +#define qemu_gettimeofday(tp) gettimeofday(tp, NULL); +#endif /* !_WIN32 */ + +#endif diff --git a/lib/qemu/qemu-common.h b/lib/qemu/qemu-common.h new file mode 100644 index 0000000..e8ea687 --- /dev/null +++ b/lib/qemu/qemu-common.h @@ -0,0 +1,124 @@ +/* Common header file that is included by all of qemu. */ +#ifndef QEMU_COMMON_H +#define QEMU_COMMON_H + +/* we put basic includes here to avoid repeating them in device drivers */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef O_LARGEFILE +#define O_LARGEFILE 0 +#endif +#ifndef O_BINARY +#define O_BINARY 0 +#endif + +#ifndef ENOMEDIUM +#define ENOMEDIUM ENODEV +#endif + +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#include +#define fsync _commit +#define lseek _lseeki64 +#define ENOTSUP 4096 +extern int qemu_ftruncate64(int, int64_t); +#define ftruncate qemu_ftruncate64 + + +static inline char *realpath(const char *path, char *resolved_path) +{ + _fullpath(resolved_path, path, _MAX_PATH); + return resolved_path; +} + +#define PRId64 "I64d" +#define PRIx64 "I64x" +#define PRIu64 "I64u" +#define PRIo64 "I64o" +#endif + +/* FIXME: Remove NEED_CPU_H. */ +#ifndef NEED_CPU_H + +#include "config-host.h" +#include +#include "osdep.h" +#include "bswap.h" + +#else + +#include "cpu.h" + +#endif /* !defined(NEED_CPU_H) */ + +/* bottom halves */ +typedef struct QEMUBH QEMUBH; + +typedef void QEMUBHFunc(void *opaque); + +QEMUBH *qemu_bh_new(QEMUBHFunc *cb, void *opaque); +void qemu_bh_schedule(QEMUBH *bh); +void qemu_bh_cancel(QEMUBH *bh); +void qemu_bh_delete(QEMUBH *bh); +int qemu_bh_poll(void); + +uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c); + +/* cutils.c */ +void pstrcpy(char *buf, int buf_size, const char *str); +char *pstrcat(char *buf, int buf_size, const char *s); +int strstart(const char *str, const char *val, const char **ptr); +int stristart(const char *str, const char *val, const char **ptr); +time_t mktimegm(struct tm *tm); + +/* Error handling. */ + +void hw_error(const char *fmt, ...) + __attribute__ ((__format__ (__printf__, 1, 2))) + __attribute__ ((__noreturn__)); + +/* IO callbacks. */ +typedef void IOReadHandler(void *opaque, const uint8_t *buf, int size); +typedef int IOCanRWHandler(void *opaque); +typedef void IOHandler(void *opaque); + +struct ParallelIOArg { + void *buffer; + int count; +}; + +typedef int (*DMA_transfer_handler) (void *opaque, int nchan, int pos, int size); + +/* A load of opaque types so that device init declarations don't have to + pull in all the real definitions. */ +typedef struct NICInfo NICInfo; +typedef struct AudioState AudioState; +typedef struct BlockDriverState BlockDriverState; +typedef struct DisplayState DisplayState; +typedef struct TextConsole TextConsole; +typedef struct CharDriverState CharDriverState; +typedef struct VLANState VLANState; +typedef struct QEMUFile QEMUFile; +typedef struct i2c_bus i2c_bus; +typedef struct i2c_slave i2c_slave; +typedef struct SMBusDevice SMBusDevice; +typedef struct QEMUTimer QEMUTimer; +typedef struct PCIBus PCIBus; +typedef struct PCIDevice PCIDevice; +typedef struct SerialState SerialState; +typedef struct IRQState *qemu_irq; +struct pcmcia_card_s; + +#endif diff --git a/lib/qemu/qemu-img.c b/lib/qemu/qemu-img.c new file mode 100644 index 0000000..ac1d331 --- /dev/null +++ b/lib/qemu/qemu-img.c @@ -0,0 +1,753 @@ +/* + * QEMU disk image utility + * + * Copyright (c) 2003-2008 Fabrice Bellard + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include "qemu-common.h" +#include "block_int.h" +#include + +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#include +#endif + +void *get_mmap_addr(unsigned long size) +{ + return NULL; +} + +void qemu_free(void *ptr) +{ + free(ptr); +} + +void *qemu_malloc(size_t size) +{ + return malloc(size); +} + +void *qemu_mallocz(size_t size) +{ + void *ptr; + ptr = qemu_malloc(size); + if (!ptr) + return NULL; + memset(ptr, 0, size); + return ptr; +} + +char *qemu_strdup(const char *str) +{ + char *ptr; + ptr = qemu_malloc(strlen(str) + 1); + if (!ptr) + return NULL; + strcpy(ptr, str); + return ptr; +} + +static void __attribute__((noreturn)) error(const char *fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + fprintf(stderr, "qemu-img: "); + vfprintf(stderr, fmt, ap); + fprintf(stderr, "\n"); + exit(1); + va_end(ap); +} + +static void format_print(void *opaque, const char *name) +{ + printf(" %s", name); +} + +static void help(void) +{ + printf("qemu-img version " QEMU_VERSION ", Copyright (c) 2004-2008 Fabrice Bellard\n" + "usage: qemu-img command [command options]\n" + "QEMU disk image utility\n" + "\n" + "Command syntax:\n" + " create [-e] [-6] [-b base_image] [-f fmt] filename [size]\n" + " commit [-f fmt] filename\n" + " convert [-c] [-e] [-6] [-f fmt] filename [filename2 [...]] [-O output_fmt] output_filename\n" + " info [-f fmt] filename\n" + "\n" + "Command parameters:\n" + " 'filename' is a disk image filename\n" + " 'base_image' is the read-only disk image which is used as base for a copy on\n" + " write image; the copy on write image only stores the modified data\n" + " 'fmt' is the disk image format. It is guessed automatically in most cases\n" + " 'size' is the disk image size in kilobytes. Optional suffixes 'M' (megabyte)\n" + " and 'G' (gigabyte) are supported\n" + " 'output_filename' is the destination disk image filename\n" + " 'output_fmt' is the destination format\n" + " '-c' indicates that target image must be compressed (qcow format only)\n" + " '-e' indicates that the target image must be encrypted (qcow format only)\n" + " '-6' indicates that the target image must use compatibility level 6 (vmdk format only)\n" + ); + printf("\nSupported format:"); + bdrv_iterate_format(format_print, NULL); + printf("\n"); + exit(1); +} + +#if defined(WIN32) +/* XXX: put correct support for win32 */ +static int read_password(char *buf, int buf_size) +{ + int c, i; + printf("Password: "); + fflush(stdout); + i = 0; + for(;;) { + c = getchar(); + if (c == '\n') + break; + if (i < (buf_size - 1)) + buf[i++] = c; + } + buf[i] = '\0'; + return 0; +} + +#else + +#include + +static struct termios oldtty; + +static void term_exit(void) +{ + tcsetattr (0, TCSANOW, &oldtty); +} + +static void term_init(void) +{ + struct termios tty; + + tcgetattr (0, &tty); + oldtty = tty; + + tty.c_iflag &= ~(IGNBRK|BRKINT|PARMRK|ISTRIP + |INLCR|IGNCR|ICRNL|IXON); + tty.c_oflag |= OPOST; + tty.c_lflag &= ~(ECHO|ECHONL|ICANON|IEXTEN); + tty.c_cflag &= ~(CSIZE|PARENB); + tty.c_cflag |= CS8; + tty.c_cc[VMIN] = 1; + tty.c_cc[VTIME] = 0; + + tcsetattr (0, TCSANOW, &tty); + + atexit(term_exit); +} + +static int read_password(char *buf, int buf_size) +{ + uint8_t ch; + int i, ret; + + printf("password: "); + fflush(stdout); + term_init(); + i = 0; + for(;;) { + ret = read(0, &ch, 1); + if (ret == -1) { + if (errno == EAGAIN || errno == EINTR) { + continue; + } else { + ret = -1; + break; + } + } else if (ret == 0) { + ret = -1; + break; + } else { + if (ch == '\r') { + ret = 0; + break; + } + if (i < (buf_size - 1)) + buf[i++] = ch; + } + } + term_exit(); + buf[i] = '\0'; + printf("\n"); + return ret; +} +#endif + +static BlockDriverState *bdrv_new_open(const char *filename, + const char *fmt) +{ + BlockDriverState *bs; + BlockDriver *drv; + char password[256]; + + bs = bdrv_new(""); + if (!bs) + error("Not enough memory"); + if (fmt) { + drv = bdrv_find_format(fmt); + if (!drv) + error("Unknown file format '%s'", fmt); + } else { + drv = NULL; + } + if (bdrv_open2(bs, filename, 0, drv) < 0) { + error("Could not open '%s'", filename); + } + if (bdrv_is_encrypted(bs)) { + printf("Disk image '%s' is encrypted.\n", filename); + if (read_password(password, sizeof(password)) < 0) + error("No password given"); + if (bdrv_set_key(bs, password) < 0) + error("invalid password"); + } + return bs; +} + +static int img_create(int argc, char **argv) +{ + int c, ret, flags; + const char *fmt = "raw"; + const char *filename; + const char *base_filename = NULL; + uint64_t size; + const char *p; + BlockDriver *drv; + + flags = 0; + for(;;) { + c = getopt(argc, argv, "b:f:he6"); + if (c == -1) + break; + switch(c) { + case 'h': + help(); + break; + case 'b': + base_filename = optarg; + break; + case 'f': + fmt = optarg; + break; + case 'e': + flags |= BLOCK_FLAG_ENCRYPT; + break; + case '6': + flags |= BLOCK_FLAG_COMPAT6; + break; + } + } + if (optind >= argc) + help(); + filename = argv[optind++]; + size = 0; + if (base_filename) { + BlockDriverState *bs; + bs = bdrv_new_open(base_filename, NULL); + bdrv_get_geometry(bs, &size); + size *= 512; + bdrv_delete(bs); + } else { + if (optind >= argc) + help(); + p = argv[optind]; + size = strtoul(p, (char **)&p, 0); + if (*p == 'M') { + size *= 1024 * 1024; + } else if (*p == 'G') { + size *= 1024 * 1024 * 1024; + } else if (*p == 'k' || *p == 'K' || *p == '\0') { + size *= 1024; + } else { + help(); + } + } + drv = bdrv_find_format(fmt); + if (!drv) + error("Unknown file format '%s'", fmt); + printf("Formatting '%s', fmt=%s", + filename, fmt); + if (flags & BLOCK_FLAG_ENCRYPT) + printf(", encrypted"); + if (flags & BLOCK_FLAG_COMPAT6) + printf(", compatibility level=6"); + if (base_filename) { + printf(", backing_file=%s", + base_filename); + } + printf(", size=%" PRIu64 " kB\n", size / 1024); + ret = bdrv_create(drv, filename, size / 512, base_filename, flags); + if (ret < 0) { + if (ret == -ENOTSUP) { + error("Formatting or formatting option not supported for file format '%s'", fmt); + } else { + error("Error while formatting"); + } + } + return 0; +} + +static int img_commit(int argc, char **argv) +{ + int c, ret; + const char *filename, *fmt; + BlockDriver *drv; + BlockDriverState *bs; + + fmt = NULL; + for(;;) { + c = getopt(argc, argv, "f:h"); + if (c == -1) + break; + switch(c) { + case 'h': + help(); + break; + case 'f': + fmt = optarg; + break; + } + } + if (optind >= argc) + help(); + filename = argv[optind++]; + + bs = bdrv_new(""); + if (!bs) + error("Not enough memory"); + if (fmt) { + drv = bdrv_find_format(fmt); + if (!drv) + error("Unknown file format '%s'", fmt); + } else { + drv = NULL; + } + if (bdrv_open2(bs, filename, 0, drv) < 0) { + error("Could not open '%s'", filename); + } + ret = bdrv_commit(bs); + switch(ret) { + case 0: + printf("Image committed.\n"); + break; + case -ENOENT: + error("No disk inserted"); + break; + case -EACCES: + error("Image is read-only"); + break; + case -ENOTSUP: + error("Image is already committed"); + break; + default: + error("Error while committing image"); + break; + } + + bdrv_delete(bs); + return 0; +} + +static int is_not_zero(const uint8_t *sector, int len) +{ + int i; + len >>= 2; + for(i = 0;i < len; i++) { + if (((uint32_t *)sector)[i] != 0) + return 1; + } + return 0; +} + +static int is_allocated_sectors(const uint8_t *buf, int n, int *pnum) +{ + int v, i; + + if (n <= 0) { + *pnum = 0; + return 0; + } + v = is_not_zero(buf, 512); + for(i = 1; i < n; i++) { + buf += 512; + if (v != is_not_zero(buf, 512)) + break; + } + *pnum = i; + return v; +} + +#define IO_BUF_SIZE 65536 + +static int img_convert(int argc, char **argv) +{ + int c, ret, n, n1, bs_n, bs_i, flags, cluster_size, cluster_sectors; + const char *fmt, *out_fmt, *out_filename; + BlockDriver *drv; + BlockDriverState **bs, *out_bs; + int64_t total_sectors, nb_sectors, sector_num, bs_offset; + uint64_t bs_sectors; + uint8_t buf[IO_BUF_SIZE]; + const uint8_t *buf1; + BlockDriverInfo bdi; + + fmt = NULL; + out_fmt = "raw"; + flags = 0; + for(;;) { + c = getopt(argc, argv, "f:O:hce6"); + if (c == -1) + break; + switch(c) { + case 'h': + help(); + break; + case 'f': + fmt = optarg; + break; + case 'O': + out_fmt = optarg; + break; + case 'c': + flags |= BLOCK_FLAG_COMPRESS; + break; + case 'e': + flags |= BLOCK_FLAG_ENCRYPT; + break; + case '6': + flags |= BLOCK_FLAG_COMPAT6; + break; + } + } + + bs_n = argc - optind - 1; + if (bs_n < 1) help(); + + out_filename = argv[argc - 1]; + + bs = calloc(bs_n, sizeof(BlockDriverState *)); + if (!bs) + error("Out of memory"); + + total_sectors = 0; + for (bs_i = 0; bs_i < bs_n; bs_i++) { + bs[bs_i] = bdrv_new_open(argv[optind + bs_i], fmt); + if (!bs[bs_i]) + error("Could not open '%s'", argv[optind + bs_i]); + bdrv_get_geometry(bs[bs_i], &bs_sectors); + total_sectors += bs_sectors; + } + + drv = bdrv_find_format(out_fmt); + if (!drv) + error("Unknown file format '%s'", out_fmt); + if (flags & BLOCK_FLAG_COMPRESS && drv != &bdrv_qcow && drv != &bdrv_qcow2) + error("Compression not supported for this file format"); + if (flags & BLOCK_FLAG_ENCRYPT && drv != &bdrv_qcow && drv != &bdrv_qcow2) + error("Encryption not supported for this file format"); + if (flags & BLOCK_FLAG_COMPAT6 && drv != &bdrv_vmdk) + error("Alternative compatibility level not supported for this file format"); + if (flags & BLOCK_FLAG_ENCRYPT && flags & BLOCK_FLAG_COMPRESS) + error("Compression and encryption not supported at the same time"); + + ret = bdrv_create(drv, out_filename, total_sectors, NULL, flags); + if (ret < 0) { + if (ret == -ENOTSUP) { + error("Formatting not supported for file format '%s'", fmt); + } else { + error("Error while formatting '%s'", out_filename); + } + } + + out_bs = bdrv_new_open(out_filename, out_fmt); + + bs_i = 0; + bs_offset = 0; + bdrv_get_geometry(bs[0], &bs_sectors); + + if (flags & BLOCK_FLAG_COMPRESS) { + if (bdrv_get_info(out_bs, &bdi) < 0) + error("could not get block driver info"); + cluster_size = bdi.cluster_size; + if (cluster_size <= 0 || cluster_size > IO_BUF_SIZE) + error("invalid cluster size"); + cluster_sectors = cluster_size >> 9; + sector_num = 0; + for(;;) { + int64_t bs_num; + int remainder; + uint8_t *buf2; + + nb_sectors = total_sectors - sector_num; + if (nb_sectors <= 0) + break; + if (nb_sectors >= cluster_sectors) + n = cluster_sectors; + else + n = nb_sectors; + + bs_num = sector_num - bs_offset; + assert (bs_num >= 0); + remainder = n; + buf2 = buf; + while (remainder > 0) { + int nlow; + while (bs_num == bs_sectors) { + bs_i++; + assert (bs_i < bs_n); + bs_offset += bs_sectors; + bdrv_get_geometry(bs[bs_i], &bs_sectors); + bs_num = 0; + /* printf("changing part: sector_num=%lld, " + "bs_i=%d, bs_offset=%lld, bs_sectors=%lld\n", + sector_num, bs_i, bs_offset, bs_sectors); */ + } + assert (bs_num < bs_sectors); + + nlow = (remainder > bs_sectors - bs_num) ? bs_sectors - bs_num : remainder; + + if (bdrv_read(bs[bs_i], bs_num, buf2, nlow) < 0) + error("error while reading"); + + buf2 += nlow * 512; + bs_num += nlow; + + remainder -= nlow; + } + assert (remainder == 0); + + if (n < cluster_sectors) + memset(buf + n * 512, 0, cluster_size - n * 512); + if (is_not_zero(buf, cluster_size)) { + if (bdrv_write_compressed(out_bs, sector_num, buf, + cluster_sectors) != 0) + error("error while compressing sector %" PRId64, + sector_num); + } + sector_num += n; + } + /* signal EOF to align */ + bdrv_write_compressed(out_bs, 0, NULL, 0); + } else { + sector_num = 0; + for(;;) { + nb_sectors = total_sectors - sector_num; + if (nb_sectors <= 0) + break; + if (nb_sectors >= (IO_BUF_SIZE / 512)) + n = (IO_BUF_SIZE / 512); + else + n = nb_sectors; + + while (sector_num - bs_offset >= bs_sectors) { + bs_i ++; + assert (bs_i < bs_n); + bs_offset += bs_sectors; + bdrv_get_geometry(bs[bs_i], &bs_sectors); + /* printf("changing part: sector_num=%lld, bs_i=%d, " + "bs_offset=%lld, bs_sectors=%lld\n", + sector_num, bs_i, bs_offset, bs_sectors); */ + } + + if (n > bs_offset + bs_sectors - sector_num) + n = bs_offset + bs_sectors - sector_num; + + if (bdrv_read(bs[bs_i], sector_num - bs_offset, buf, n) < 0) + error("error while reading"); + /* NOTE: at the same time we convert, we do not write zero + sectors to have a chance to compress the image. Ideally, we + should add a specific call to have the info to go faster */ + buf1 = buf; + while (n > 0) { + if (is_allocated_sectors(buf1, n, &n1)) { + if (bdrv_write(out_bs, sector_num, buf1, n1) < 0) + error("error while writing"); + } + sector_num += n1; + n -= n1; + buf1 += n1 * 512; + } + } + } + bdrv_delete(out_bs); + for (bs_i = 0; bs_i < bs_n; bs_i++) + bdrv_delete(bs[bs_i]); + free(bs); + return 0; +} + +#ifdef _WIN32 +static int64_t get_allocated_file_size(const char *filename) +{ + typedef DWORD (WINAPI * get_compressed_t)(const char *filename, DWORD *high); + get_compressed_t get_compressed; + struct _stati64 st; + + /* WinNT support GetCompressedFileSize to determine allocate size */ + get_compressed = (get_compressed_t) GetProcAddress(GetModuleHandle("kernel32"), "GetCompressedFileSizeA"); + if (get_compressed) { + DWORD high, low; + low = get_compressed(filename, &high); + if (low != 0xFFFFFFFFlu || GetLastError() == NO_ERROR) + return (((int64_t) high) << 32) + low; + } + + if (_stati64(filename, &st) < 0) + return -1; + return st.st_size; +} +#else +static int64_t get_allocated_file_size(const char *filename) +{ + struct stat st; + if (stat(filename, &st) < 0) + return -1; + return (int64_t)st.st_blocks * 512; +} +#endif + +static void dump_snapshots(BlockDriverState *bs) +{ + QEMUSnapshotInfo *sn_tab, *sn; + int nb_sns, i; + char buf[256]; + + nb_sns = bdrv_snapshot_list(bs, &sn_tab); + if (nb_sns <= 0) + return; + printf("Snapshot list:\n"); + printf("%s\n", bdrv_snapshot_dump(buf, sizeof(buf), NULL)); + for(i = 0; i < nb_sns; i++) { + sn = &sn_tab[i]; + printf("%s\n", bdrv_snapshot_dump(buf, sizeof(buf), sn)); + } + qemu_free(sn_tab); +} + +static int img_info(int argc, char **argv) +{ + int c; + const char *filename, *fmt; + BlockDriver *drv; + BlockDriverState *bs; + char fmt_name[128], size_buf[128], dsize_buf[128]; + uint64_t total_sectors; + int64_t allocated_size; + char backing_filename[1024]; + char backing_filename2[1024]; + BlockDriverInfo bdi; + + fmt = NULL; + for(;;) { + c = getopt(argc, argv, "f:h"); + if (c == -1) + break; + switch(c) { + case 'h': + help(); + break; + case 'f': + fmt = optarg; + break; + } + } + if (optind >= argc) + help(); + filename = argv[optind++]; + + bs = bdrv_new(""); + if (!bs) + error("Not enough memory"); + if (fmt) { + drv = bdrv_find_format(fmt); + if (!drv) + error("Unknown file format '%s'", fmt); + } else { + drv = NULL; + } + if (bdrv_open2(bs, filename, 0, drv) < 0) { + error("Could not open '%s'", filename); + } + bdrv_get_format(bs, fmt_name, sizeof(fmt_name)); + bdrv_get_geometry(bs, &total_sectors); + get_human_readable_size(size_buf, sizeof(size_buf), total_sectors * 512); + allocated_size = get_allocated_file_size(filename); + if (allocated_size < 0) + sprintf(dsize_buf, "unavailable"); + else + get_human_readable_size(dsize_buf, sizeof(dsize_buf), + allocated_size); + printf("image: %s\n" + "file format: %s\n" + "virtual size: %s (%" PRId64 " bytes)\n" + "disk size: %s\n", + filename, fmt_name, size_buf, + (total_sectors * 512), + dsize_buf); + if (bdrv_is_encrypted(bs)) + printf("encrypted: yes\n"); + if (bdrv_get_info(bs, &bdi) >= 0) { + if (bdi.cluster_size != 0) + printf("cluster_size: %d\n", bdi.cluster_size); + } + bdrv_get_backing_filename(bs, backing_filename, sizeof(backing_filename)); + if (backing_filename[0] != '\0') { + path_combine(backing_filename2, sizeof(backing_filename2), + filename, backing_filename); + printf("backing file: %s (actual path: %s)\n", + backing_filename, + backing_filename2); + } + dump_snapshots(bs); + bdrv_delete(bs); + return 0; +} + +int main(int argc, char **argv) +{ + const char *cmd; + + bdrv_init(); + if (argc < 2) + help(); + cmd = argv[1]; + optind++; + if (!strcmp(cmd, "create")) { + img_create(argc, argv); + } else if (!strcmp(cmd, "commit")) { + img_commit(argc, argv); + } else if (!strcmp(cmd, "convert")) { + img_convert(argc, argv); + } else if (!strcmp(cmd, "info")) { + img_info(argc, argv); + } else { + help(); + } + return 0; +} diff --git a/lib/qemu/qemu-timer.h b/lib/qemu/qemu-timer.h new file mode 100644 index 0000000..e69de29 diff --git a/lib/qemu/qemu_glue.c b/lib/qemu/qemu_glue.c new file mode 100644 index 0000000..a16e207 --- /dev/null +++ b/lib/qemu/qemu_glue.c @@ -0,0 +1,100 @@ +#include "qemu-common.h" +#include "block_int.h" + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" + + +void *get_mmap_addr(unsigned long size) +{ + return NULL; +} + +void qemu_free(void *ptr) +{ + free(ptr); +} + +void *qemu_malloc(size_t size) +{ + return malloc(size); +} + +void *qemu_mallocz(size_t size) +{ + void *ptr; + ptr = qemu_malloc(size); + if (!ptr) + return NULL; + memset(ptr, 0, size); + return ptr; +} + +char *qemu_strdup(const char *str) +{ + char *ptr; + ptr = qemu_malloc(strlen(str) + 1); + if (!ptr) + return NULL; + strcpy(ptr, str); + return ptr; +} + +void term_printf(const char *str) +{ + puts(str); +} + +void term_print_filename(const char *filename) +{ + puts(filename); +} + +void pstrcpy(char *buf, int buf_size, const char *str) +{ + int c; + char *q = buf; + + if (buf_size <= 0) + return; + + for(;;) { + c = *str++; + if (c == 0 || q >= buf + buf_size - 1) + break; + *q++ = c; + } + *q = '\0'; +} + +/* strcat and truncate. */ +char *pstrcat(char *buf, int buf_size, const char *s) +{ + int len; + len = strlen(buf); + if (len < buf_size) + pstrcpy(buf + len, buf_size - len, s); + return buf; +} + + + +int strstart(const char *str, const char *val, const char **ptr) +{ + const char *p, *q; + p = str; + q = val; + while (*q != '\0') { + if (*p != *q) + return 0; + p++; + q++; + } + if (ptr) + *ptr = p; + return 1; +} + + + diff --git a/lib/qemu/qtest.c b/lib/qemu/qtest.c new file mode 100644 index 0000000..56406f0 --- /dev/null +++ b/lib/qemu/qtest.c @@ -0,0 +1,43 @@ +#include "qemu-common.h" +#include "block_int.h" + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" + + +int main(int argc,char **argv) +{ + BlockDriverState *bs; + BlockDriver *drv=NULL; + char fmt_name[128], size_buf[128], dsize_buf[128]; + uint64_t total_sectors=0; + //BlockDriverInfo bdi; + int sector_start = atoi(argv[2]); + int sector_count = atoi(argv[3]); + + unsigned char *buf = malloc(sector_count*512); + printf("start: %d count: %d\n",sector_start,sector_count); + + bdrv_init(); + + bs = bdrv_new(""); + if(!bs) errx(1,"not enough memory for qemu"); + + if(bdrv_open2(bs,argv[1],0,drv) < 0){ + errx(1,"Could not open %s\n",argv[1]); + } + bdrv_get_format(bs, fmt_name, sizeof(fmt_name)); + bdrv_get_geometry(bs, &total_sectors); + printf("image: %s format: %s size: %"PRId64" bytes\n",argv[1],fmt_name,total_sectors); + + if(bdrv_read(bs,sector_start,buf,sector_count)<0){ + errx(1,"Can't read"); + } + write(fileno(stdout),buf,sector_count*512); + + return(0); + + +} + diff --git a/lib/s3 b/lib/s3 new file mode 100755 index 0000000..0dde9aa --- /dev/null +++ b/lib/s3 @@ -0,0 +1,99 @@ +#! /bin/sh + +# s3 - temporary wrapper script for .libs/s3 +# Generated by ltmain.sh - GNU libtool 1.5.22 (1.1220.2.365 2005/12/18 22:14:06) +# +# The s3 program cannot be directly executed until all the libtool +# libraries that it depends on are installed. +# +# This wrapper script should never be moved out of the build directory. +# If it is, it will not operate correctly. + +# Sed substitution that helps us do robust quoting. It backslashifies +# metacharacters that are still active within double-quoted strings. +Xsed='/opt/local/bin/gsed -e 1s/^X//' +sed_quote_subst='s/\([\\`\\"$\\\\]\)/\\\1/g' + +# The HP-UX ksh and POSIX shell print the target directory to stdout +# if CDPATH is set. +(unset CDPATH) >/dev/null 2>&1 && unset CDPATH + +relink_command="" + +# This environment variable determines our operation mode. +if test "$libtool_install_magic" = "%%%MAGIC variable%%%"; then + # install mode needs the following variable: + notinst_deplibs=' libafflib.la' +else + # When we are sourced in execute mode, $file and $echo are already set. + if test "$libtool_execute_magic" != "%%%MAGIC variable%%%"; then + echo="/bin/echo" + file="$0" + # Make sure echo works. + if test "X$1" = X--no-reexec; then + # Discard the --no-reexec flag, and continue. + shift + elif test "X`($echo '\t') 2>/dev/null`" = 'X\t'; then + # Yippee, $echo works! + : + else + # Restart under the correct shell, and then maybe $echo will work. + exec /bin/sh "$0" --no-reexec ${1+"$@"} + fi + fi + + # Find the directory that this script lives in. + thisdir=`$echo "X$file" | $Xsed -e 's%/[^/]*$%%'` + test "x$thisdir" = "x$file" && thisdir=. + + # Follow symbolic links until we get to the real thisdir. + file=`ls -ld "$file" | /opt/local/bin/gsed -n 's/.*-> //p'` + while test -n "$file"; do + destdir=`$echo "X$file" | $Xsed -e 's%/[^/]*$%%'` + + # If there was a directory component, then change thisdir. + if test "x$destdir" != "x$file"; then + case "$destdir" in + [\\/]* | [A-Za-z]:[\\/]*) thisdir="$destdir" ;; + *) thisdir="$thisdir/$destdir" ;; + esac + fi + + file=`$echo "X$file" | $Xsed -e 's%^.*/%%'` + file=`ls -ld "$thisdir/$file" | /opt/local/bin/gsed -n 's/.*-> //p'` + done + + # Try to get the absolute directory name. + absdir=`cd "$thisdir" && pwd` + test -n "$absdir" && thisdir="$absdir" + + program='s3' + progdir="$thisdir/.libs" + + + if test -f "$progdir/$program"; then + # Add our own library path to DYLD_LIBRARY_PATH + DYLD_LIBRARY_PATH="/Users/simsong/domex/src/afflib/lib/.libs:$DYLD_LIBRARY_PATH" + + # Some systems cannot cope with colon-terminated DYLD_LIBRARY_PATH + # The second colon is a workaround for a bug in BeOS R4 sed + DYLD_LIBRARY_PATH=`$echo "X$DYLD_LIBRARY_PATH" | $Xsed -e 's/::*$//'` + + export DYLD_LIBRARY_PATH + + if test "$libtool_execute_magic" != "%%%MAGIC variable%%%"; then + # Run the actual program with our arguments. + + exec "$progdir/$program" ${1+"$@"} + + $echo "$0: cannot exec $program ${1+"$@"}" + exit 1 + fi + else + # The program doesn't exist. + $echo "$0: error: \`$progdir/$program' does not exist" 1>&2 + $echo "This script is just a wrapper for $program." 1>&2 + /bin/echo "See the libtool documentation for more information." 1>&2 + exit 1 + fi +fi diff --git a/lib/s3.cpp b/lib/s3.cpp new file mode 100644 index 0000000..0e42834 --- /dev/null +++ b/lib/s3.cpp @@ -0,0 +1,552 @@ +/* + * s3.cpp: + * The stand-alone S3 program. + * + * These features would be nice: + * have "ls" use Delimiter option to just list the drives + * Give "ls" an option to list just the AFF files. + * Have implementation read a list of all the segments on open, and cache this. + * + * Distributed under the Berkeley 4-part license. + * Simson L. Garfinkel, March 12, 2012 + */ + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" +#include "aftimer.h" + +#include +#include +#include + +#ifdef USE_S3 +#include "s3_glue.h" + +#include + +#ifdef HAVE_OPENSSL_MD5_H +#include +#endif + + + +char *outfile = 0; +char *opt_bucket = 0; +int opt_flag = 0; +int opt_meta = 0; +int verbose = 0; +int tag = 0; + +using namespace s3; + +#define BANDWIDTH_PREFIX ".bandwidth_test" +#define BANDWIDTH_DEFAULT_SIZE 1000000 +int bandwidth_offset = 0; + +#ifndef HAVE_STRLCPY +size_t strlcpy(char *dest,const char *src,size_t dest_size) +{ + strncpy(dest,src,dest_size); + dest[dest_size-1] = '\000'; + return strlen(dest); +} +#endif + +#ifndef HAVE_STRLCAT +size_t strlcat(char *dest,const char *src,size_t dest_size) +{ + int dest_len = strlen(dest); + int src_len = strlen(src); + int room = dest_size - (dest_len +src_len+1); + if(room>0){ + /* There is room; just copy over what we have and return */ + strcat(dest,src); + return strlen(dest); + } + /* Not room; figure out how many bytes we can copy... */ + int left = dest_size - (dest_len+1); + strncpy(dest+dest_len,src,left); + dest[dest_len-1] = '\000'; + return strlen(dest); +} +#endif + + +void s3_df() +{ + class s3_result *e = list_buckets(); + if(!e->lambr) errx(1,"S3 did not return ListAllMyBucketsResult."); + printf("Owner ID: %s\n",e->lambr->OwnerID.c_str()); + printf("Owner Display Name: %s\n",e->lambr->OwnerDisplayName.c_str()); + printf("\n"); + for(vector::const_iterator i = e->lambr->Buckets.begin(); + i != e->lambr->Buckets.end(); + i++){ + printf("%s %s\n",(*i)->CreationDate.c_str(),(*i)->Name.c_str()); + } + exit(0); +} + +typedef vector cvector; +void s3_ls(FILE *out,const char *prefix,cvector *cv) +{ + uint64_t total=0; + string bucket = opt_bucket; + if(out) fprintf(out,"S3 BUCKET %s:",bucket.c_str()); + if(strlen(prefix)>0){ + if(out) fprintf(out,"PREFIX %s",prefix); + } + if(out) fprintf(out,"\n\n"); + string marker; + class s3_result *e; + bool isTruncated = false; + do { + e = list_bucket(bucket,prefix,marker,00); + if(e==0) err(1,"Error loading bucket."); + if(e->lbr==0) err(1,"Error: no LBR"); + if(e->lbr->contents.size()==0){ + delete e; + break; // + } + for(vector::const_iterator i = e->lbr->contents.begin(); + i != e->lbr->contents.end(); + i++){ + + if(cv) cv->push_back(**i); + /* Make date nice */ + char tstamp[64]; + strlcpy(tstamp,(*i)->LastModified.c_str(),sizeof(tstamp)); + tstamp[10] = ' '; + tstamp[19] = '\000'; + + if(out){ + fprintf(out,"%s ",(*i)->OwnerDisplayName.c_str()); + fprintf(out,"%8d ",(int)(*i)->Size); + fprintf(out,"%s ",tstamp); + fprintf(out,"%s ",(*i)->Key.c_str()); + fprintf(out,"\n"); + } + total += (*i)->Size; + } + + /* "To get the next page of results use the last key of the + * current page as the marker." + */ + + marker = e->lbr->contents.back()->Key; + isTruncated = e->lbr->IsTruncated; + delete e; + } while(isTruncated); + + char buf[64]; + if(out) fprintf(out,"Total: %"PRId64"\n",total); +} + +void s3_cat(int argc, char **argv) +{ + argc--;argv++; + while(*argv){ + string key(*argv); + class response_buffer *b = 0; + if(opt_meta==0){ + b = object_get(opt_bucket,key,0); + } + else { + b = object_head(opt_bucket,key,0); + } + if(!b) errx(1,"HTTP transport error"); + if(b->result == 404) errx(1,"S3: %s not found",key.c_str()); + fwrite(b->base,1,b->len,stdout); + delete b; + argv++; + argc--; + } +} + +/* s3 doesn't give an error if you try to delete an object that doesn't exist */ +void s3_rm(int argc,char **argv) +{ + argc--;argv++; + while(*argv){ + printf("s3 rm %s\n",*argv); + int r = object_rm(opt_bucket,*argv); + if(r) errx(1,"HTTP transport error"); + argv++; + argc--; + } + exit(0); +} + +/* del prefix */ +void s3_delp(int argc,char **argv) +{ + argc--;argv++; + + FILE *f = stdout; + char line[80]; + cvector cv; + if(!strcmp(argv[0],"-q")){ + f = fopen("/dev/null","w"); + argv++; + argc--; + } + + + if(argc!=1) errx(1,"delp requires a single argument"); + s3_ls(f,*argv,&cv); + if(cv.size()==0) errx(0,"No items to delete"); + printf("Really delete %d item%s?\n",cv.size(),cv.size()==1 ? "" : "s"); + fgets(line,sizeof(line),stdin); + if(line[0]!='y' && line[0]!='Y') errx(1,"Aborted"); + for(cvector::iterator i=cv.begin(); + i!=cv.end(); + i++){ + printf("s3 rm %s\n",i->Key.c_str()); + if(object_rm(opt_bucket,i->Key.c_str())){ + warn("HTTP error"); + } + } +} + +void s3_cp(int argc, char **argv) +{ + argc--; + argv++; + char * fname = argv[0]; + char * key = argv[1]; + struct s3headers meta[2] = {{0,0},{0,0}}; + char buf[64]; + + if(opt_flag){ + snprintf(buf,sizeof(buf),"%d",opt_flag); + meta[0].name = AMAZON_METADATA_PREFIX "arg"; + meta[0].value = buf; + } + + /* Read from fname into a buffer. + * Note that we do this with read, so that we can read from stdin + */ + FILE *f = fopen(fname,"r"); + if(!f) err(1,"%s",fname); + class response_buffer inbuf; + while(!feof(f)){ + char buf[65536]; + int count; + count = fread(buf,1,sizeof(buf),f); + if(count>0){ + inbuf.write(buf,count); + } + } + if(object_put(opt_bucket,key,inbuf.base,inbuf.len,meta)){ + errx(1,"%s: ",fname); + } + exit(0); +} + +void s3_mkdir(int argc,char **argv) +{ + argc--;argv++; + while(argc>0){ + if(bucket_mkdir(*argv)) err(1,"%s",*argv); + argc--; + argv++; + } + exit(0); +} + +void s3_rmdir(int argc, char **argv) +{ + argc--;argv++; + while(argc>0){ + string bucket(*argv); + if(bucket_rmdir(bucket)) errx(1,"%s",bucket.c_str()); + argc--; + argv++; + } + exit(0); +} + + + +void usage() +{ + printf("s3 testing program.\n\n"); + printf("Usage:\n"); + printf("s3 ls (or dir) [prefix] - list the bucket's contents\n"); + printf("s3 mkdir - make a bucket\n"); + printf("s3 rmdir - delete a bucket\n"); + printf("s3 df - display all of the buckets\n"); + printf("s3 rm key - delete key from the bucket\n"); + printf("s3 cat key - send the contents of the key to stdout\n"); + printf("s3 cp fname key - copy local file fname to key\n"); + printf("s3 delp [-q] prefix - Delete all keys with 'prefix' as a prefix\n"); + printf("\n"); + printf("Debugging commands:\n"); + printf("s3 regress - run regression tests\n"); + printf("s3 bandwidth [-m] nn - measure bandwidth \n"); + printf(" Use -m to make the files that are needed for read testing.\n"); + printf("\n"); + printf("Options:\n"); + printf(" -d = enable HTTP debugging\n"); + printf(" -b = Specifies bucket\n"); + printf(" -fnn = specify a 32-bit metadata 'flag'\n"); + printf(" -m = just report the metadata (for cat)\n"); + printf(" -O = set bandwidth offset (default 0)\n"); + printf(" -u url = go to url instead of %s\n",aws_base_url); + printf(" -o fn = append output to file fn\n"); + printf(" -V verbose\n"); + printf(" -v print version number and exit\n"); + printf(" -t tag = tag to append to output\n"); + exit(0); +} + +/****************************************************************/ + +/* Regression testing */ +int regress() +{ + /* Make some data */ + for(int i=0;i<100;i++){ + char name[1024]; + char value[1024]; + snprintf(name,sizeof(name),"bucket%d",i); + snprintf(value,sizeof(value),"This is the contents of bucket %d\n",i); + object_put(opt_bucket,name,value,strlen(value),0); + } + + for(int i=0;i<100;i++){ + if(i%10==0) printf("\n"); + class s3_result *e = list_bucket(opt_bucket,"","",0); + if(!e->lbr) err(1,"Error loading bucket pass %d\n",i); + delete e; + printf("%d ",i); + fflush(stdout); + } + printf("Done. Check for memory leaks, then press any key...\n"); + getchar(); + exit(0); +} + +void s3_bandwidth(int argc,char **argv) +{ + int opt_make = opt_meta; // in case it was set + int read_retry=0; + int write_retry=0; + int write_err=0; + int opt_write_test=1; + const char *opt_url = 0; + + argc--;argv++; + + if(argc>0 && strcmp(argv[0],"-m")==0){ + if(verbose) fprintf(stderr,"opt_make\n"); + opt_make = 1; + argc--; + argv++; + } + + /* Bandwidth testing requires bandwidth_test0 through 9. + * If they don't exist, make them. + */ +#if defined(HAVE_SRANDOMDEV) + srandomdev(); +#endif +#if !defined(HAVE_SRANDOMDEV) && defined(HAVE_SRANDOM) + srandom(time(0)); +#endif + + size_t size = BANDWIDTH_DEFAULT_SIZE; + if(argc>0) size=atoi(argv[0]); + if(verbose) fprintf(stderr,"size=%d\n",size); + if(size==0) err(1,"size=0"); + char *buf = (char *)malloc(size); + if(!buf) err(1,"malloc"); + memset(buf,'E',size); + + if(argc>0) size=atoi(argv[0]); + + if(strncmp(argv[0],"http://",7)==0){ + opt_url = argv[0]; + opt_write_test = 0; + } + + char base[1024]; + char randp[1024]; + snprintf(base,sizeof(base),"%s.%d",BANDWIDTH_PREFIX,size); + snprintf(randp,sizeof(randp),"%s.%d.%d",BANDWIDTH_PREFIX,size,random() % 1000); + + + aftimer twrite; + + if(opt_write_test || opt_make){ + /* Measure the write bandwidth */ + twrite.start(); + + if(opt_make) strcpy(randp,base); // just write to this one + char wkey[1024]; + snprintf(wkey,sizeof(wkey),"%s.%d",randp,bandwidth_offset); + object_put(opt_bucket,wkey,buf,size,0); + if(verbose) fprintf(stderr," wrote %s/%s\n",opt_bucket,wkey); + write_retry += s3_request_retry_count; + write_err += s3_object_put_retry_count; + + twrite.stop(); + + if(opt_make) exit(0); // file made + + /* Delete the writes */ + if(object_rm(opt_bucket,wkey)){ + err(1,"object_rm failed\n"); + } + if(verbose) fprintf(stderr," deleted %s/%s\n",opt_bucket,wkey); + + } + + /* Now measure the read bandwidth */ + aftimer tread; + tread.start(); + + char rkey[1024]; + snprintf(rkey,sizeof(rkey),"%s.%d",base,bandwidth_offset); + response_buffer *r=0; + + if(opt_url==0){ + r = object_get(opt_bucket,rkey,0); + if(!r || r->result==404){ + err(1,"object_get(%s/%s) failed (%d)",opt_bucket,rkey,errno); + } + } + else { + r = s3::get_url(opt_url); + if(r) size = r->len; + } + read_retry += s3_request_retry_count; + tread.stop(); + if(verbose) fprintf(stderr," read %s/%s\n",opt_bucket,rkey); + + char line[1024]; + memset(line,0,sizeof(line)); + time_t t = time(0); + struct tm *tm = gmtime(&t); + char tbuf[64]; + strftime(tbuf,sizeof(tbuf),"%F %T",tm); + + char hextag[64]; + memset(hextag,0,sizeof(hextag)); + for(int i=0;i<16;i++){ + sprintf(hextag+i*2,"%02x",r->ETag[i]); + } + + FILE *out = outfile ? fopen(outfile,"a") : stdout; + + //snprintf(line,sizeof(line),"v3: %s\t%d\t%d\t%f\t%f\t%d\t%d\t%d\t%s + fprintf(out,"v3: %s\t",tbuf); + + fprintf(out,"%d\t%d\t",size,bandwidth_offset); + fprintf(out,"%f\t",twrite.elapsed_seconds()); + fprintf(out,"%f\t",tread.elapsed_seconds()); + fprintf(out,"%d\t%d\t%d\t",write_retry,write_err,read_retry); + fprintf(out," "); + /* Now verify the MD5 */ + unsigned char md5[16]; + memset(md5,0,sizeof(md5)); + MD5((const unsigned char *)r->base,r->len,md5); + if(memcmp(r->ETag,md5,16)!=0){ + fprintf(out," FAILED != "); + for(int i=0;i<16;i++) fprintf(out,"%02x",md5[i]); + } + + if(tag) fprintf(out,"\t%d ",tag); + fprintf(out,"\n"); + fclose(out); + if(r) delete r; + exit(0); +} + + +/****************************************************************/ + + +int main(int argc,char **argv) +{ + int bflag, ch; + + bflag = 0; + opt_bucket = getenv(S3_DEFAULT_BUCKET); + while ((ch = getopt(argc, argv, "b:dVh?f:mc:O:u:o:vt:")) != -1) { + switch (ch) { + case 'O': bandwidth_offset = atoi(optarg);break; + case 'd': s3_debug++;break; + case 'b': opt_bucket = optarg;break; + case 'f': opt_flag = atoi(optarg);break; + case 'm': opt_meta = 1;break; + case 'v': + printf("%s version %s\n",argv[0],PACKAGE_VERSION); + exit(0); + case 'u': aws_base_url = optarg;break; + case 'o': outfile = optarg;break; + case 'V': verbose++;break; + case 't': tag = atoi(optarg);break; + case 'h': + case '?': + default: + usage(); + } + } + argc -= optind; + argv += optind; + + if(argc<1){ + usage(); + } + + if(getenv(S3_DEBUG)){ + s3_debug = atoi(getenv(S3_DEBUG)); + fprintf(stderr,"s3_debug set to %d\n",s3_debug); +#ifdef HAVE_ERR_SET_EXIT + err_set_exit(s3_audit); +#endif + } + + aws_access_key_id = getenv(AWS_ACCESS_KEY_ID); + aws_secret_access_key = getenv(AWS_SECRET_ACCESS_KEY); + if(!aws_access_key_id) fprintf(stderr,"s3: AWS_ACCESS_KEY_ID not defined\n"); + if(!aws_secret_access_key) fprintf(stderr,"s3: AWS_SECRET_ACCESS_KEY not defined\n"); + if(!aws_access_key_id || !aws_secret_access_key) return -1; /* can't open */ + if(!opt_bucket) { + fprintf(stderr,"No bucket. Please setenv S3_DEFAULT_BUCKET or specify -b option.\n"); + exit(1); + } + + + char *cmd = argv[0]; + + if(!strcmp(cmd,"ls") || !strcmp(cmd,"dir")){ + const char *prefix = argc>1 ? argv[1] : ""; + s3_ls(stdout,prefix,0);exit(0); + } + if(!strcmp(cmd,"df")) {s3_df();exit(0);} + if(!strcmp(cmd,"cat")) {s3_cat(argc,argv);exit(0);} + if(!strcmp(cmd,"rm")) {s3_rm(argc,argv);exit(0);} + if(!strcmp(cmd,"cp") || !strcmp(cmd,"put") || !strcmp(cmd,"send")) { + if ( argc != 3 ) {usage();exit(1);} + s3_cp(argc,argv);exit(0); + } + if(!strcmp(cmd,"mkdir")) {s3_mkdir(argc,argv);exit(0);} + if(!strcmp(cmd,"rmdir")) {s3_rmdir(argc,argv);exit(0);} + if(!strcmp(cmd,"regress")) {regress();exit(0);} + if(!strcmp(cmd,"delp")) {s3_delp(argc,argv);exit(0);} + if(!strcmp(cmd,"bandwidth")) { + s3_bandwidth(argc,argv);exit(0); + } + usage(); + exit(0); +} +#else +int main(int argc,char **argv) +{ + fprintf(stderr,"S3 is not compiled in.\n"); + exit(0); +} +#endif diff --git a/lib/s3_glue.cpp b/lib/s3_glue.cpp new file mode 100644 index 0000000..71adab5 --- /dev/null +++ b/lib/s3_glue.cpp @@ -0,0 +1,733 @@ +#include "affconfig.h" + +/* + * Distributed under the Berkeley 4-part license + */ + +#include +#include +#include + +#ifdef USE_S3 + +#ifdef HAVE_SYS_PARAM_H +#include +#endif + +#if (defined(__FreeBSD_version)) && (__FreeBSD_version<500000) && (!defined(BAD_STL)) +#define BAD_STL +#endif + + +#include "s3_glue.h" +#include "curl/curl.h" +#include "base64.h" + +#include +#include +#include + +#ifdef HAVE_OPENSSL_MD5_H +#include +#endif + +#ifdef HAVE_OPENSSL_HMAC_H +#include +#endif + +#ifdef HAVE_ERR_H +#include +#endif + +#ifdef HAVE_CTYPE_H +#include +#endif + +#if !defined(HAVE_OPENSSL_MD5_H) +#error S3 support requires MD5 support +#endif + +int s3_debug = 0; +int s3_retry_max = 5; // read by the code + +/* debug levels: + * 1 - print retries + * 2- print queries + * 3- print full results + */ + +/* Counters that are used; they aren't threadsafe, but they are never referenced */ +int s3_request_retry_count = 0; +int s3_object_put_retry_count = 0; +long long s3_bytes_written=0; +long long s3_bytes_read=0; + +using namespace std; +using namespace s3; + +const char *aws_access_key_id; +const char *aws_secret_access_key; +const char *aws_base_url = "http://s3.amazonaws.com/"; + +/* Simson's S3 implementation in C++. + * Note that libcurl and expat will both handle data in chunks, so + * technically we don't need to create a single buffer with the entire response + * from AWS. For AFFLIB, though, we want to work on data as buffers. + * As a result, we create and use a buffer for all work. + */ + +namespace s3 { + +static string itos(int i) +{ + char buf[64]; + snprintf(buf,sizeof(buf),"%d",i); + return string(buf); +} + +size_t buffer::write(const char *b,size_t count){ + if(!writable) return false; + base = (char *)realloc(base,len+count); + if(base){ + memcpy(base+len,b,count); // copy the memory over + len += count; + return count; + } + return 0; +} +size_t buffer::read(char *b,size_t count){ + if(base){ + if(count>len-ptr) count=len-ptr; + memcpy(b,base+ptr,count); + ptr += count; + return count; + } + return 0; +} + +void buffer::print() { + fwrite(base,1,len,stdout); +} + +void buffer::clear(){ + if(base){ + free(base); + base = 0; + } + len = 0; +} + + +static size_t buffer_write(void *buffer, size_t size, size_t nmemb, void *userp) +{ + return ((class buffer *)userp)->write((const char *)buffer,size * nmemb); +} + +static size_t buffer_read(void *buffer, size_t size, size_t nmemb, void *userp) +{ + return ((class buffer *)userp)->read((char *)buffer,size * nmemb); +} + + +static void startElement(void *userData, const char *name, const char **atts) +{ + class s3_result *einfo = (class s3_result *)userData; + einfo->depth++; + switch(einfo->depth){ + case 1: + if(!strcmp(name,"ListBucketResult")) {einfo->lbr = new ListBucketResult();break;} + if(!strcmp(name,"ListAllMyBucketsResult")) {einfo->lambr = new ListAllMyBucketsResult();break;} + fprintf(stderr,"\ns3 buffer:\n%s",einfo->buf->base); + errx(1,"Unknown XML element from S3: '%s'",name); + break; + case 2: + if(einfo->lbr && !strcmp(name,"Contents")){ einfo->lbr->contents.push_back(new Contents());break;} + break; + case 3: + if(einfo->lambr && !strcmp(name,"Bucket")){ einfo->lambr->Buckets.push_back(new Bucket());break;} + break; + } +} + +static void endElement(void *userData, const char *name) +{ + class s3_result *einfo = (class s3_result *)userData; + if(einfo->lambr){ + switch(einfo->depth){ + case 3: + if(!strcmp(name,"ID")){ einfo->lambr->OwnerID = einfo->cbuf;break;} + if(!strcmp(name,"DisplayName")){ einfo->lambr->OwnerDisplayName = einfo->cbuf;break;} + break; + case 4: + if(!strcmp(name,"Name")) { einfo->lambr->Buckets.back()->Name = einfo->cbuf;break;} + if(!strcmp(name,"CreationDate")) { einfo->lambr->Buckets.back()->CreationDate = einfo->cbuf;break;} + } + } + if(einfo->lbr){ + switch(einfo->depth){ + case 2: + if(!strcmp(name,"Name")){ einfo->lbr->Name = einfo->cbuf; break;} + if(!strcmp(name,"Prefix")){ einfo->lbr->Prefix = einfo->cbuf;break;} + if(!strcmp(name,"Marker")){ einfo->lbr->Marker = einfo->cbuf;break;} + if(!strcmp(name,"MaxKeys")){ einfo->lbr->MaxKeys = atoi(einfo->cbuf.c_str());break;} + if(!strcmp(name,"IsTruncated")){ einfo->lbr->IsTruncated = tolower(einfo->cbuf[0]) == 't';break;} + break; + case 3: + if(!strcmp(name,"Key")){ einfo->lbr->contents.back()->Key = einfo->cbuf; break;} + if(!strcmp(name,"LastModified")){einfo->lbr->contents.back()->LastModified = einfo->cbuf;break;} + if(!strcmp(name,"ETag")){ einfo->lbr->contents.back()->ETag = einfo->cbuf;break;} + if(!strcmp(name,"Size")){ einfo->lbr->contents.back()->Size = atoi(einfo->cbuf.c_str());break;} + break; + case 4: + if(!strcmp(name,"ID")){ einfo->lbr->contents.back()->OwnerID = einfo->cbuf;break;} + if(!strcmp(name,"DisplayName")){ einfo->lbr->contents.back()->OwnerDisplayName = einfo->cbuf;break;} + break; + default:; + } + } +#ifdef BAD_STL + einfo->cbuf = ""; +#else + einfo->cbuf.clear(); +#endif + einfo->depth--; +} + +static void characterDataHandler(void *userData,const XML_Char *s,int len) +{ + class s3_result *einfo = (class s3_result *)userData; + einfo->cbuf.append((const char *)s,len); +} + + +static class s3_result *xml_extract_response(const class buffer *buf) +{ + class s3_result *e = new s3_result(); + + e->buf = buf; + XML_Parser parser = XML_ParserCreate(NULL); + XML_SetUserData(parser, e); + XML_SetElementHandler(parser, startElement, endElement); + XML_SetCharacterDataHandler(parser,characterDataHandler); + + if (!XML_Parse(parser, (const char *)buf->base, buf->len, 1)) { + char buf2[2048]; + snprintf(buf2,sizeof(buf2), + "XML Error: %s at line %d", + XML_ErrorString(XML_GetErrorCode(parser)),(int)XML_GetCurrentLineNumber(parser)); + fprintf(stderr,"%s:\n",buf2); + XML_ParserFree(parser); + return 0; + } + XML_ParserFree(parser); + return e; +} + + + +/* Create the cannonical string for the headers */ +static string canonical_string(string method,string path,curl_slist *headers, time_t expires) +{ + /* Iterate through the headers a line at a time */ + map interesting_headers; + + for(;headers;headers = headers->next){ + char *line = strdup(headers->data); + char *word; + char *brk2; + word = strtok_r(line,": ",&brk2); + if(word){ + if(strcasecmp(word,"Date")==0 || + strcasecmp(word,"Range")==0 || + strncmp(word,AMAZON_METADATA_PREFIX,strlen(AMAZON_METADATA_PREFIX))==0){ + char *value = strtok_r(NULL,"",&brk2); + while(value && isspace(*value)) value++; + interesting_headers[word] = value; + } + } + free(line); + } + /* Add the headers that we don't have */ + + /* handle the date */ + + /* Get the sorted headers */ + vector sorted_header_keys; + for(map::const_iterator i = interesting_headers.begin(); + i!=interesting_headers.end(); + i++){ + sorted_header_keys.push_back(i->first); + } + +#ifndef BAD_STL + sort(sorted_header_keys.begin(),sorted_header_keys.end()); +#endif + + string buf = method + "\n"; + buf += "\n"; // content-md5 value + buf += "\n"; // content-type value + + /* Either put in a date header or else do the expires */ + if(expires){ + char b[64]; + snprintf(b,sizeof(b),"%d\n",(int)expires); + buf += b; + } + else { + buf += interesting_headers["Date"] + "\n"; // date + } + + /* AMAON_HEADER_PREFIX headers only... */ + for(vector::const_iterator i = sorted_header_keys.begin(); + i != sorted_header_keys.end(); + i++){ + if(i->substr(0,strlen(AMAZON_METADATA_PREFIX))==AMAZON_METADATA_PREFIX){ + buf += *i + ":" + interesting_headers[*i] + "\n"; + } + } + buf += "/" + path; // the resource + + //printf("canonical: \n===========\n%s\n=========\n",buf.c_str()); + + return buf; + +} + +static string encode(const char *aws_secret_access_key,string str) +{ + unsigned char md[20]; + uint32_t md_len = sizeof(md); + + /* Note: This MUST be sha1() */ + HMAC(EVP_sha1(),aws_secret_access_key,strlen(aws_secret_access_key), + (const unsigned char *)str.c_str(),str.size(), + md,&md_len); + /* Now encode this to base64 */ + char b64str[64]; + memset(b64str,0,sizeof(b64str)); + b64_ntop(md,md_len,b64str,sizeof(b64str)); + return string(b64str); +} + + + +static string quote_plus(string &url) +{ + /* encode the URL */ + string eurl; + char buf[6]; + for(string::const_iterator c = url.begin(); c != url.end(); c++){ + switch(*c){ + case '%': + case ';': + case '/': + case '?': + case '@': + case '&': + case '=': + case '+': + case '$': + case ',': + sprintf(buf,"%%%02X",*c); + eurl += buf; + continue; + case ' ': + eurl += "+"; + continue; + default: + eurl += *c; + continue; + } + } + return eurl; +} + +#ifndef HAVE_ISDIGIT +static int isdigit(char ch) +{ + return ch>='0' && ch<='9'; +} +#endif + +static int hexval(int ch) { return (isdigit(ch) ? ch-'0' : ch-'a'+10);} + +/* + * Execute an S3 request: + * method - method to execute. + * path - path for the object. + * query - anything optional after the "?" in the path + * expires - When the authorization URL should expire. + * sendbuf - if we are sending something ,this is what is being sent. + * sendbuflen - how long that buffer is + * extraheaders - any additional headers that should be sent; useful for metadata + * + * Returns a response buffer + */ + +/* CURLINFO_RESPONSE_CODE is the new name for the option previously known as + * CURLINFO_HTTP_CODE. + */ + + +#ifndef CURLINFO_RESPONSE_CODE +#define CURLINFO_RESPONSE_CODE CURLINFO_HTTP_CODE +#endif + +class response_buffer *request(string method,string path,string query,time_t expires, + const char *sendbuf,size_t sendbuflen, + const s3headers *extraheaders) +{ + /* Note: this function is not threadsafe */ + static bool curl_initted = false; + if(!curl_initted){ + curl_global_init(CURL_GLOBAL_ALL); + curl_initted=true; + } + + int retry_count=0; + class response_buffer *b = 0; + class buffer *h = 0; + do { + + if(s3_debug>1) printf("==================================================\n"); + if(s3_debug && retry_count>0) printf("=== S3 RETRY %d ===\n",retry_count); + + CURL *c = curl_easy_init(); + struct curl_slist *headers=NULL; + + if(expires==0){ + /* Add the Date: field to the header */ + struct tm tm; + time_t t = time(0); + char date[64]; + strftime(date,sizeof(date),"Date: %a, %d %b %Y %X GMT",gmtime_r(&t,&tm)); + headers = curl_slist_append(headers, date); + } + + /* Add the extra headers */ + while(extraheaders && extraheaders[0].name){ + int len = strlen(extraheaders[0].name)+strlen(extraheaders[0].value)+4; + char *buf = (char *)alloca(len); + snprintf(buf,len,"%s: %s",extraheaders[0].name,extraheaders[0].value); + headers = curl_slist_append(headers, buf); + extraheaders++; + } + + string url = aws_base_url + path; + string canonical_str = canonical_string(method,path,headers,expires); + string encoded_canonical = encode(aws_secret_access_key,canonical_str); + + if(expires==0){ + /* Create an Authorization header */ + + char authorization[96]; + + snprintf(authorization,sizeof(authorization),"Authorization: AWS %s:%s", + aws_access_key_id,encoded_canonical.c_str()); + headers = curl_slist_append(headers, authorization); + curl_easy_setopt(c, CURLOPT_HTTPHEADER, headers); + } + + if(expires){ + /* Add authorization to the URL*/ + if(query.size()>0) query += "&"; + query += "Signature=" + quote_plus(encoded_canonical); + query += "&Expires=" + itos(expires); + query += "&AWSAccessKeyId=" + string(aws_access_key_id); + } + + if(query.size()>0){ + url += "?" + query; + } + + if(b) delete b; + b = new response_buffer(); + memset(b->ETag,0,sizeof(b->ETag)); + if(s3_debug>1) curl_easy_setopt(c,CURLOPT_VERBOSE,1); + if(method != "GET"){ + curl_easy_setopt(c,CURLOPT_CUSTOMREQUEST,method.c_str()); + } + + if(method == "HEAD"){ + curl_easy_setopt(c,CURLOPT_NOBODY,1); + } + + /* Queries that take longer than an hour should timeout */ + curl_easy_setopt(c,CURLOPT_TIMEOUT,60*60); + + /* Disable DNS cache */ + curl_easy_setopt(c,CURLOPT_DNS_CACHE_TIMEOUT,0); // per amazon specification + curl_easy_setopt(c,CURLOPT_WRITEFUNCTION,buffer_write); + curl_easy_setopt(c,CURLOPT_WRITEDATA,b); // fourth argument + curl_easy_setopt(c,CURLOPT_URL,url.c_str()); + + /* Are we sending data */ + class buffer *sendbuffer = 0; + if(sendbuf){ + sendbuffer = new buffer(sendbuf,sendbuflen); + curl_easy_setopt(c,CURLOPT_READFUNCTION,buffer_read); + curl_easy_setopt(c,CURLOPT_READDATA,sendbuffer); + curl_easy_setopt(c,CURLOPT_UPLOAD,1); + curl_easy_setopt(c,CURLOPT_INFILESIZE,sendbuflen); + //fprintf(stderr,"***** sendbuflen= %d %qd\n",sizeof(sendbuflen),sendbuflen); + } + + /* Make provisions to get the response headers */ + if(h) delete h; + h = new buffer(); + curl_easy_setopt(c,CURLOPT_HEADERFUNCTION,buffer_write); + curl_easy_setopt(c,CURLOPT_WRITEHEADER,h); // fourth argument + + /* Make provisions for getting the headers */ + + int success = curl_easy_perform(c); + + if(sendbuffer){ + delete sendbuffer; + sendbuffer = 0; + if(success==0) s3_bytes_written += sendbuflen; + } + + s3_bytes_read += h->len; + s3_bytes_read += b->len; + + // CURL API says do not assume NULL terminate, so terminate it + h->write("\000",1); + curl_easy_getinfo(c,CURLINFO_RESPONSE_CODE,&b->result); + + /* Now clean up */ + s3_request_retry_count = retry_count; + if(headers) curl_slist_free_all(headers); + curl_easy_cleanup(c); + + /* Process the results */ + if(success!=0){ + delete h; + delete b; + s3_request_retry_count = retry_count; + return 0; // internal CURL error + } + if(s3_debug>2){ + printf("Header results:\n"); + h->print(); + printf("Data results:\n"); + b->print(); + printf("\n"); + } + } while(b->result==500 && ++retry_countresult==404) errno=ENOENT; + + /* Pull out the headers */ + char *line,*brkt; + for(line = strtok_r(h->base,"\r\n",&brkt); + line; + line = strtok_r(NULL,"\r\n",&brkt)){ + char *cc = strchr(line,':'); + if(cc){ + *cc++ = '\000'; + while(*cc && isspace(*cc)) cc++; + b->rheaders[line] = cc; + } + } + + /* Find the ETag in the header and put in the buffer */ + const char *e = b->rheaders["ETag"].c_str(); + if(strlen(e)==34){ + for(int i=0;i<16;i++){ + b->ETag[i] = (hexval(e[i*2+1])<<4) + hexval(e[i*2+2]); + } + } + + delete h; // we don't care about it + if(s3_debug>1) printf(".\n\n"); + return b; +} + +response_buffer *get_url(const char *url) +{ + int retry_count = 0; + response_buffer *b = new response_buffer(); + do { + CURL *c = curl_easy_init(); + curl_easy_setopt(c,CURLOPT_WRITEFUNCTION,buffer_write); + curl_easy_setopt(c,CURLOPT_WRITEDATA,b); + curl_easy_setopt(c,CURLOPT_URL,url); + int success = curl_easy_perform(c); + curl_easy_getinfo(c,CURLINFO_RESPONSE_CODE,&b->result); + curl_easy_cleanup(c); + } while(b->result!=200 && ++retry_count0) query += "prefix=" + prefix; + if(marker.size()>0){ + if(query.size()>0) query += "&"; + query += "marker=" + marker; + } + if(max_keys>0){ + if(query.size()>0) query += "&";; + query += "max-keys=" + itos(max_keys); + } + class response_buffer *b = request("GET",bucket,query,0,0,0,0); + if(!b) return 0; + class s3_result *r = xml_extract_response(b); + delete b; + return r; +} + +/* + * af_hexbuf: + * Turn a binay string into a hex string, optionally with spaces. + */ + +#define HEXBUF_NO_SPACES 0 +#define HEXBUF_SPACE2 0x0001 // space every 2 characters +#define HEXBUF_SPACE4 0x0002 // space every 4 characters +#define HEXBUF_UPPERCASE 0x1000 // uppercase +static const char *hexbuf(char *dst,int dst_len,const unsigned char *bin,int bytes,int flag) +{ + int charcount = 0; + const char *start = dst; // remember where the start of the string is + const char *fmt = (flag & HEXBUF_UPPERCASE) ? "%02X" : "%02x"; + + *dst = 0; // begin with null termination + while(bytes>0 && dst_len > 3){ + sprintf(dst,fmt,*bin); // convert the next byte + dst += 2; + bin += 1; + dst_len -= 2; + bytes--; + charcount++; // how many characters + + bool add_spaces = false; + if(flag & HEXBUF_SPACE2) add_spaces = true; + if((flag & HEXBUF_SPACE4) && charcount%2==0){ + *dst++ = ' '; + *dst = '\000'; + dst_len -= 1; + } + } + return start; // return the start +} + + +/* object_put: + * Put an object. Make sure that the MD5 of the response matches.. + * Makes a few retry attempts + * Return 0 if success, -1 if failure. + */ +int object_put(string bucket,string path, + const char *buf,size_t buflen, + const struct s3headers *extraheaders) +{ + unsigned char md5[16]; + memset(md5,0,sizeof(md5)); + MD5((const unsigned char *)buf,buflen,md5); + for(int i=0;i0){ + fprintf(stderr,"S3: Attempt to write object '%s' failed. Retrying...\n", + path.c_str()); + } + + response_buffer *res = request("PUT",bucket + "/" + path,"",0,buf,buflen,extraheaders); + if(!res) { + fprintf(stderr,"S3 request: No response.\n"); + continue; + } + if(memcmp(res->ETag,md5,16)==0){ /* Check the MD5 of the response */ + delete res; + return 0; + } + char buf0[64],buf1[64]; + fprintf(stderr,"S3: Expected ETag '%s' got '%s'\n", + hexbuf(buf0,sizeof(buf0),md5,16,HEXBUF_SPACE4), + hexbuf(buf1,sizeof(buf1),res->ETag,16,HEXBUF_SPACE4)); + delete res; + } + /* Write failed. Delete the written object and return */ + response_buffer *res = request("DELETE",bucket + "/" + path,"",0,0,0,0); + if(res) delete res; + errno = EIO; + return -1; +} + +int bucket_mkdir(string bucket) +{ + class response_buffer *b = request("PUT",bucket,"",0,0,0,0); + int result = b->result; + delete b; + switch(result){ + case 409:errno=EEXIST; return -1; + case 200:errno=0;return 0; + } + return -1; // some unknown error +} + +int bucket_rmdir(string bucket) +{ + class response_buffer *b = request("DELETE",bucket,"",0,0,0,0); + int result = b->result; + delete b; + switch(result){ + case 403:errno=EACCES; return -1; + case 404:errno=ENOENT; return -1; + case 409:errno=ENOTEMPTY; return -1; + case 204:errno=0;return 0; // no content is actually what it gives + case 200:errno=0;return 0; // doesn't seem to give this one + } + return -1; // some unknown error +} + +class response_buffer *object_get(string bucket,string path,const s3headers *extra_headers) +{ + return request("GET",bucket + "/" + path,"",0,0,0,extra_headers); +} + +class response_buffer *object_head(string bucket,string path,const s3headers *extra_headers) +{ + return request("HEAD",bucket + "/" + path,"",0,0,0,extra_headers); +} + +int object_rm(string bucket,string path) +{ + class response_buffer *b = request("DELETE",bucket + "/" + path,"",0,0,0,0); + if(b){ + delete b; + return 0; + } + return -1; +} + +} + +void s3_audit(int i) +{ + if(i>0 || s3_bytes_written>0 || s3_bytes_read>0){ + fprintf(stderr,"\n"); + fprintf(stderr,"S3 bytes written: %qu\n",s3_bytes_written); + fprintf(stderr,"S3 bytes read: %qu\n",s3_bytes_read); + } +} +#endif /* USE_S3 */ diff --git a/lib/s3_glue.h b/lib/s3_glue.h new file mode 100644 index 0000000..d59dedb --- /dev/null +++ b/lib/s3_glue.h @@ -0,0 +1,158 @@ +/* + * s3_glue.h: + * + * Glue logic to make AFFLIB work with Amazon's Simple Storage Service. + * This can also be compiled stand-alone by #define STAND. + * That's useful for testing + * + * Requires: expat, openssl (for base64 coding) + * Distributed under the Berkeley 4-part license + */ + + +#ifndef S3_GLUE_H +#define S3_GLUE_H + +#include +#include + +#include +#include +#include +#include // memcpy, strcmp, strlen +#include // sort + +#define S3_DEFAULT_BUCKET "S3_DEFAULT_BUCKET" +#define S3_DEBUG "S3_DEBUG" +#define AWS_ACCESS_KEY_ID "AWS_ACCESS_KEY_ID" +#define AWS_SECRET_ACCESS_KEY "AWS_SECRET_ACCESS_KEY" + +extern int s3_debug; +extern int s3_retry_max; // default 5; you can set however you wish +extern int s3_request_retry_count; +extern int s3_object_put_retry_count; +extern const char *aws_access_key_id; +extern const char *aws_secret_access_key; +extern const char *aws_base_url; +extern long long s3_total_written; +extern long long s3_total_read; + +#define AMAZON_METADATA_PREFIX "x-amz-meta-" +#define S3_CONTENT_LENGTH "Content-Length" + +void s3_audit(int x); + +namespace s3 { + using namespace std; + + struct s3headers { + const char *name; // do not include x-amz-meta- + const char *value; + }; + + class buffer { + public: + char *base; // array + size_t len; // length + int ptr; // for reading + bool writable; + buffer() : base(0),len(0),ptr(0),writable(true) {} + buffer(const char *base_,int len_) : + base((char *)base_),len(len_),ptr(0),writable(false) {} + ~buffer() { if(base && writable) free(base);} + /* Append bytes; return number of bytes appended */ + size_t write(const char *b,size_t count); + size_t read(char *b,size_t count); + void print(); + void clear(); + }; + + class response_buffer : public buffer { + public: + long result; // HTTP result code + map rheaders; // response headers + unsigned char ETag[16]; // if provided, in binary + }; + + /* S3 XML Objects */ + class Contents { + public: + string Key; + string LastModified; + string ETag; + size_t Size; + string OwnerID; + string OwnerDisplayName; + string StorageClass; + }; + + class Bucket { + public: + string Name; + string CreationDate; + }; + + class ListAllMyBucketsResult { + public: + ~ListAllMyBucketsResult(){ + for(vector::iterator i = Buckets.begin(); + i != Buckets.end(); + i++){ + delete *i; + } + } + string OwnerID; + string OwnerDisplayName; + vector Buckets; + }; + + class ListBucketResult { + public: + ~ListBucketResult(){ + for(vector::iterator i = contents.begin(); + i != contents.end(); + i++){ + delete *i; + } + } + string Name; + string Prefix; + string Marker; + int MaxKeys; + bool IsTruncated; + vector contents; // list of objects + }; + + class s3_result { + public: + s3_result() : depth(0),lambr(0),lbr(0){}; + ~s3_result() { + if(lambr) delete lambr; + if(lbr) delete lbr; + } + int depth; + string cbuf; // buffer of these characters + class ListAllMyBucketsResult *lambr; + class ListBucketResult *lbr; // list bucket results + const class buffer *buf; // what we are parsing + }; + + response_buffer *request(string method,string path,string query,time_t expires, + const char *sendbuf,size_t sendbuflen, + const s3headers *extra_headers); + response_buffer *get_url(const char *url); + s3_result *list_buckets(); + s3_result *list_bucket(string bucket,string prefix,string marker,int max_keys); + int object_put(string bucket,string path, + const char *buf,size_t buflen, + const struct s3headers *meta); + int bucket_mkdir(string bucket); + int bucket_rmdir(string bucket); + response_buffer *object_get(string bucket,string path, + const s3headers *extra_headers); + response_buffer *object_head(string bucket,string path, + const s3headers *extra_headers); + int object_rm(string bucket,string path); +} + +#endif diff --git a/lib/utils.cpp b/lib/utils.cpp new file mode 100644 index 0000000..0843f21 --- /dev/null +++ b/lib/utils.cpp @@ -0,0 +1,96 @@ +/* + * utils.cpp: + * + * Some handy utilities for working with AFF + * + * Distributed under the Berkeley 4-part license + */ + + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" +#include "utils.h" +#ifdef HAVE_ERR_H +#include "err.h" +#endif + +#include +#ifdef HAVE_CSTRING +#include +#endif + +#ifdef HAVE_READLINE_READLINE_H +#include +#endif + +namespace aff { + + bool ends_with(const char *buf,const char *with) + { + if(buf && with){ + size_t buflen = strlen(buf); + size_t withlen = strlen(with); + if(buflen>withlen && strcmp(buf+buflen-withlen,with)==0) return 1; + } + return 0; + } + +#ifdef HAVE_STL + /** Given argc and argv, return a string with the command line */ + std::string command_line(int argc,char **argv) { + std::string command = ""; + for(int i=0;i0) command += " "; + command += argv[i]; + } + return command; + } + bool ends_with(const std::string &buf,const std::string &with) + { + return ends_with(buf.c_str(),with.c_str()); + } + + + /* Given an AFFILE, return a seglist. + * Returns -1 if failure, 0 if success. + */ + int seglist::get_seglist(AFFILE *af) + { + if(af_rewind_seg(af)) return -1; + char name_[AF_MAX_NAME_LEN]; + size_t len_=0; + uint32_t arg_=0; + while(af_get_next_seg(af,name_,sizeof(name_),&arg_,0,&len_)==0){ + // We shouldn't have 0-len segment names, but we do in some files. + // Don't copy these segments. + if(strlen(name_)>0){ + seginfo si(name_,len_,arg_); + push_back(si); + } + } + return 0; + } + + bool seglist::has_signed_segments() + { + for(seglist::const_iterator seg = begin(); seg!=end() ;seg++){ + if(ends_with(seg->name.c_str(),AF_SIG256_SUFFIX)){ + return true; + } + } + return false; + } + + + bool seglist::contains(std::string segname) + { + for(std::vector::const_iterator i = begin(); i!=end(); i++){ + if(i->name == segname) return true; + } + return false; + } + +#endif +} + diff --git a/lib/utils.h b/lib/utils.h new file mode 100644 index 0000000..a2335c4 --- /dev/null +++ b/lib/utils.h @@ -0,0 +1,89 @@ +/* + * utils.h: + * Some useful utilities for building AFF-aware programs. + * Distributed under the Berkeley 4-part license + */ + +#ifndef AFF_UTILS_H +#define AFF_UTILS_H + +#ifdef __cplusplus +#ifdef HAVE_STL +#include +#include +#include +#include +#include +#include +#include +#endif + +#ifdef HAVE_OPENSSL_PEM_H +#include +#include +#else +typedef void X509; +typedef void EVP_PKEY; +typedef void BIO; +#define BIO_free free +#endif + +namespace aff { + +#ifdef HAVE_STL + std::string command_line(int argc,char **argv); + bool ends_with(std::string str,std::string ending); +#endif + bool ends_with(const char *buf,const char *with); + + /* Structure for hash map */ + struct less_c_str + { + inline bool operator()( const char* x, const char* y) const + { return ( strcmp( x,y ) < 0 ); + } + }; + + struct md5blob { + unsigned char buf[16]; + }; + +#ifdef HAVE_STL + typedef std::map< const char*, struct md5blob, less_c_str > hashMapT; + + /* The seginfo stores information about a segment other than its data*/ + class seginfo { + public: + seginfo(std::string n1,size_t l1,unsigned int a1): name(n1),len(l1),arg(a1) {} + std::string name; + size_t len; + unsigned long arg; + /** pagenumber returns <0 for invalid pages, >= for a page */ + int64_t pagenumber() const {return af_segname_page_number(name.c_str());} + bool inline operator==(const class seginfo &b) const { + return name == b.name; + } + virtual ~seginfo(){}; + }; + + /* the seglist provides AFF internal functions and tools an easy way to get + * a list of all of the segments in the currently open AFF file. + * Use the seglist(af) constructor to populate it with all the segments + * when you create. Each element is populated with the name, length and arg. + */ + class seglist : public std::vector { + public: + bool contains(std::string segname); + bool has_signed_segments(); + int get_seglist(AFFILE *af); + seglist(){} + virtual ~seglist(){} + seglist(AFFILE *af){ + get_seglist(af); + } + }; +#endif +} +#endif + +#endif diff --git a/lib/version.h.in b/lib/version.h.in new file mode 100644 index 0000000..02aa75f --- /dev/null +++ b/lib/version.h.in @@ -0,0 +1,3 @@ +@SET_MAKE@ +#define AFFLIB_VERSION "@PACKAGE_VERSION@" + diff --git a/lib/vnode_afd.cpp b/lib/vnode_afd.cpp new file mode 100644 index 0000000..70decd2 --- /dev/null +++ b/lib/vnode_afd.cpp @@ -0,0 +1,557 @@ +/* + * vnode_aff.cpp: + * + * Functions for the manipulation of AFF files... + * Distributed under the Berkeley 4-part license + */ + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" +#include "vnode_afd.h" +#include "aff_db.h" + +#ifndef F_OK +#define F_OK 00 +#endif + +#ifndef R_OK +#define R_OK 04 +#endif + + +#if defined(WIN32) and !defined(HAVE__MINGW_H) +/********************************************************************** + * Implement dirent-style opendir/readdir/rewinddir/closedir on Win32 + * + * Functions defined are opendir(), readdir(), rewinddir() and + * closedir() with the same prototypes as the normal dirent.h + * implementation. + * + * Does not implement telldir(), seekdir(), or scandir(). The dirent + * struct is compatible with Unix, except that d_ino is always 1 and + * d_off is made up as we go along. + * + * The DIR typedef is not compatible with Unix. + **********************************************************************/ + +extern "C" DIR *opendir(const char *dir) +{ + DIR *dp; + char *filespec; + long handle; + int index; + + filespec = (char *)malloc(strlen(dir) + 2 + 1); + strcpy(filespec, dir); + index = strlen(filespec) - 1; + if (index >= 0 && (filespec[index] == '/' || (filespec[index] == '\\' ))) + filespec[index] = '\0'; + strcat(filespec, "\\*"); + + dp = (DIR *) malloc(sizeof(DIR)); + dp->offset = 0; + dp->finished = 0; + + if ((handle = _findfirst(filespec, &(dp->fileinfo))) < 0) { + if (errno == ENOENT) { + dp->finished = 1; + } else { + free(dp); + free(filespec); + return NULL; + } + } + dp->dir = strdup(dir); + dp->handle = handle; + free(filespec); + + return dp; +} + +extern "C" struct dirent *readdir(DIR *dp) +{ + if (!dp || dp->finished) + return NULL; + + if (dp->offset != 0) { + if (_findnext(dp->handle, &(dp->fileinfo)) < 0) { + dp->finished = 1; + return NULL; + } + } + dp->offset++; + + strlcpy(dp->dent.d_name, dp->fileinfo.name, _MAX_FNAME+1); + dp->dent.d_ino = 1; + dp->dent.d_reclen = strlen(dp->dent.d_name); + dp->dent.d_off = dp->offset; + + return &(dp->dent); +} + +extern "C" int readdir_r(DIR *dp, struct dirent *entry, struct dirent **result) +{ + if (!dp || dp->finished) { + *result = NULL; + return 0; + } + + if (dp->offset != 0) { + if (_findnext(dp->handle, &(dp->fileinfo)) < 0) { + dp->finished = 1; + *result = NULL; + return 0; + } + } + dp->offset++; + + strlcpy(dp->dent.d_name, dp->fileinfo.name, _MAX_FNAME+1); + dp->dent.d_ino = 1; + dp->dent.d_reclen = strlen(dp->dent.d_name); + dp->dent.d_off = dp->offset; + + memcpy(entry, &dp->dent, sizeof(*entry)); + + *result = &dp->dent; + + return 0; +} + +extern "C" int closedir(DIR *dp) +{ + if (!dp) + return 0; + _findclose(dp->handle); + if (dp->dir) + free(dp->dir); + if (dp) + free(dp); + + return 0; +} + +extern "C" int rewinddir(DIR *dp) +{ + /* Re-set to the beginning */ + char *filespec; + long handle; + int index; + + _findclose(dp->handle); + + dp->offset = 0; + dp->finished = 0; + + filespec = (char *)malloc(strlen(dp->dir) + 2 + 1); + strcpy(filespec, dp->dir); + index = strlen(filespec) - 1; + if (index >= 0 && (filespec[index] == '/' || filespec[index] == '\\')) + filespec[index] = '\0'; + strcat(filespec, "/*"); + + if ((handle = _findfirst(filespec, &(dp->fileinfo))) < 0) { + if (errno == ENOENT) + dp->finished = 1; + } + dp->handle = handle; + free(filespec); + return 0; +} +#endif + + +/**************************************************************** + *** Service routines + ****************************************************************/ + +struct afd_private { + AFFILE **afs; // list of AFFILEs... + int num_afs; // number of them + int cur_file; // current segment number... +}; + +static inline struct afd_private *AFD_PRIVATE(AFFILE *af) +{ + assert(af->v == &vnode_afd); + return (struct afd_private *)(af->vnodeprivate); +} + + +/* afd_file_with_seg: + * Returns the AFFILE for a given segment, or 0 if it isn't found. + */ + +static AFFILE *afd_file_with_seg(AFFILE *af,const char *name) +{ + struct afd_private *ap = AFD_PRIVATE(af); + + for(int i=0;inum_afs;i++){ + if(af_get_seg(ap->afs[i],name,0,0,0)==0){ + return ap->afs[i]; + } + } + errno = ENOTDIR; // get ready for error return + return 0; +} + +static void aff_filename(AFFILE *afd,char *buf,int buflen,int num) +{ + snprintf(buf,buflen,"%s/file_%03d.aff",afd->fname,num); +} + +/* Return 1 if a file is an AFF file */ +static int afd_identify_file(const char *filename,int exists) +{ + if(filename==0 || strlen(filename)==0) return 0; // zero-length filenames aren't welcome + if(strncmp(filename,"file://",7)==0){ + /* Move file pointer past file:// then find a '/' and take the next character */ + filename += 7; + while(*filename && *filename!='/'){ + filename++; + } + /* At this point if *filename==0 then we never found the end of the URL. + * return 0, since it's not an AFF file. + */ + if(*filename==0) return 0; + + /* So *filename must == '/' */ + assert(*filename == '/'); + filename++; + } + if(exists && access(filename,R_OK)!=0) return 0; // needs to exist and it doesn't + + /* If it ends with a '/', remove it */ + char *fn = (char *)malloc(strlen(filename)+1); + strcpy(fn,filename); + char *lastc = fn + strlen(fn) - 1; + if(*lastc=='/') *lastc = '\000'; + + /* If filename exists and it is a dir, it needs to end afd */ + struct stat sb; + if(stat(fn,&sb)==0){ + if((sb.st_mode & S_IFMT)==S_IFDIR){ + if(af_ext_is(fn,"afd")){ + free(fn); + return 1; + } + } + free(fn); + return 0; // + } + /* Doesn't exist. Does it end .afd ? */ + if(af_ext_is(fn,"afd")){ + free(fn); + return 1; + } + free(fn); + return 0; +} + + + + +/* Add a file to the AFF system. + * if fname==0, create a new one and copy over the relevant metadata... + */ +static int afd_add_file(AFFILE *af,const char *fname_) +{ + struct afd_private *ap = AFD_PRIVATE(af); + const char *segs_to_copy[] = {AF_BADFLAG, + AF_CASE_NUM, + AF_IMAGE_GID, + AF_ACQUISITION_ISO_COUNTRY, + AF_ACQUISITION_COMMAND_LINE, + AF_ACQUISITION_DATE, + AF_ACQUISITION_NOTES, + AF_ACQUISITION_DEVICE, + AF_ACQUISITION_TECHNICIAN, + AF_DEVICE_MANUFACTURER, + AF_DEVICE_MODEL, + AF_DEVICE_SN, + AF_DEVICE_FIRMWARE, + AF_DEVICE_SOURCE, + AF_CYLINDERS, + AF_HEADS, + AF_SECTORS_PER_TRACK, + AF_LBA_SIZE, + AF_HPA_PRESENT, + AF_DCO_PRESENT, + AF_LOCATION_IN_COMPUTER, + AF_DEVICE_CAPABILITIES, + 0}; + + char fname[MAXPATHLEN+1]; + memset(fname,0,sizeof(fname)); + if(fname_){ + strlcpy(fname,fname_,sizeof(fname)); + } + else { + aff_filename(af,fname,sizeof(fname),ap->num_afs); + } + + int new_file = access(fname,F_OK)!=0; // Is this a new file? + + AFFILE *af2 = af_open(fname,af->openflags|AF_NO_CRYPTO,af->openmode); + if(af2==0){ + (*af->error_reporter)("open(%s,%d,%d) failed: %s\n", + fname,af->openflags,af->openmode,strerror(errno)); + return -1; // this is bad + } + + ap->num_afs += 1; + ap->afs = (AFFILE **)realloc(ap->afs,sizeof(AFFILE *) * ap->num_afs); + ap->afs[ap->num_afs-1] = af2; + + if(new_file){ + /* Copy over configuration from AFD vnode*/ + af_enable_compression(af2,af->compression_type,af->compression_level); + af_set_pagesize(af2,af->image_pagesize); // + af_set_sectorsize(af2,af->image_sectorsize); + af_update_seg(af,AF_AFF_FILE_TYPE,0,(const u_char *)"AFD",3); + + /* If this is the second file, copy over additional metadata from first... */ + if(ap->num_afs>1){ + AFFILE *af0 = ap->afs[0]; + memcpy(af2->badflag,af0->badflag,af->image_sectorsize); + af2->bytes_memcpy += af->image_sectorsize; + + for(const char **segname=segs_to_copy;*segname;segname++){ + unsigned char data[65536]; // big enough for most metadata + size_t datalen = sizeof(data); + uint32_t arg=0; + + if(af_get_seg(af0,*segname,&arg,data,&datalen)==0){ + int r = af_update_seg(af2,*segname,arg,data,datalen); + if(r!=0){ + (*af->error_reporter)("afd_add_file: could not update %s in %s (r=%d)", + *segname,af_filename(af2),r); + } + } + } + } + } + + return 0; +} + + + +/**************************************************************** + *** User-visible functions. + ****************************************************************/ + +static int afd_open(AFFILE *af) +{ + if(af->fname==0 || strlen(af->fname)==0) return -1; // zero-length filenames aren't welcome + + /* If the name ends with a '/', remove it */ + char *lastc = af->fname + strlen(af->fname) - 1; + if(*lastc=='/') *lastc = '\000'; + + + /* If the directory doesn't exist, make it (if we are O_CREAT) */ + struct stat sb; + af->exists = 1; // assume that the directory eixsts + if(stat(af->fname,&sb)!=0){ + if((af->openflags & O_CREAT) == 0){ // flag not set + errno = ENOTDIR; + return -1; + } + mode_t cmask = umask(0); // get the current umask + umask(cmask & 077); // make sure we will be able to write the file + mkdir(af->fname,af->openmode|0111); // make the directory + umask(cmask); // put back the old mask + af->exists = 0; // directory doesn't exist; we had to make it. + if(stat(af->fname,&sb)) return -1; // error if we can't stat it + } + /* If this is a regular file, don't open it */ + if(!S_ISDIR(sb.st_mode)){ + errno = ENOTDIR; // needs to be a directory + return -1; + } + + + af->maxsize = AFD_DEFAULT_MAXSIZE; + af->vnodeprivate = (void *)calloc(1,sizeof(struct afd_private)); + struct afd_private *ap = AFD_PRIVATE(af); + ap->afs = (AFFILE **)malloc(sizeof(AFFILE *)); + + /* Open the directory and read all of the AFF files */ + DIR *dirp = opendir(af->fname); + if(!dirp){ + return -1; // something is wrong... + } + struct dirent *dp; + while ((dp = readdir(dirp)) != NULL){ + if (af_ext_is(dp->d_name,"aff")){ + char path[MAXPATHLEN+1]; + strlcpy(path,af->fname,sizeof(path)); + strlcat(path,"/",sizeof(path)); + strlcat(path,dp->d_name,sizeof(path)); + if(afd_add_file(af,path)){ + return -1; + } + } + } + closedir(dirp); + if(ap->num_afs==0 && af->exists){ + snprintf(af->error_str,sizeof(af->error_str),".afd directory contains no .aff files!"); + return -1; + } + return 0; // "we were successful" +} + + +static int afd_close(AFFILE *af) +{ + struct afd_private *ap = AFD_PRIVATE(af); + + /* Close all of the subfiles, then free the memory, then close this file */ + for(int i=0;inum_afs;i++){ + ap->afs[i]->image_size = af->image_size; // set each to have correct imagesize + af_close(ap->afs[i]); // and close each file + } + free(ap->afs); + memset(ap,0,sizeof(*ap)); // clean object reuse + free(ap); // won't need it again + return 0; +} + + +#if !defined(WIN32) || defined(HAVE__MINGW_H) +static uint64_t max(uint64_t a,uint64_t b) +{ + return a > b ? a : b; +} +#endif + +static int afd_vstat(AFFILE *af,struct af_vnode_info *vni) +{ + struct afd_private *ap = AFD_PRIVATE(af); + memset(vni,0,sizeof(*vni)); // clear it + + /* See if there is some device that knows how big the disk is */ + if(ap->num_afs>0){ + af_vstat(ap->afs[0],vni); // get disk free bytes + } + + /* Get the file with the largest imagesize from either the + * AFD or any of the sub AFDs... + */ + vni->imagesize = af->image_size; + for(int i=0;inum_afs;i++){ + vni->imagesize = max(vni->imagesize,ap->afs[i]->image_size); + } + vni->has_pages = 1; + vni->supports_metadata = 1; + return 0; +} + +static int afd_get_seg(AFFILE *af,const char *name,uint32_t *arg,unsigned char *data, + size_t *datalen) +{ + AFFILE *af2 = afd_file_with_seg(af,name); + if(af2){ + return af_get_seg(af2,name,arg,data,datalen); // use this one + } + return -1; // not found +} + + +static int afd_get_next_seg(AFFILE *af,char *segname,size_t segname_len,uint32_t *arg, + unsigned char *data,size_t *datalen_) +{ + /* See if there are any more in the current segment */ + struct afd_private *ap = AFD_PRIVATE(af); + while (ap->cur_file < ap->num_afs) { + int r = af_get_next_seg(ap->afs[ap->cur_file],segname,segname_len,arg,data,datalen_); + if(r!=AF_ERROR_EOF){ // if it is not EOF + return r; + } + ap->cur_file++; // advance to the next file + if(ap->cur_file < ap->num_afs){ // rewind it to the beginning + af_rewind_seg(ap->afs[ap->cur_file]); + } + } while(ap->cur_file < ap->num_afs); + return AF_ERROR_EOF; // really made it to the end +} + + +/* Rewind all of the segments */ +static int afd_rewind_seg(AFFILE *af) +{ + struct afd_private *ap = AFD_PRIVATE(af); + ap->cur_file = 0; + for(int i=0;inum_afs;i++){ + af_rewind_seg(ap->afs[i]); + } + return 0; +} + + + +/* Update: + * If this segment is in any of the existing files, update it there. + * Otherwise, if the last file isn't too big, add it there. + * Otherwise, ada a new file. + */ +static int afd_update_seg(AFFILE *af, const char *name, + uint32_t arg,const u_char *value,uint32_t vallen) + +{ + struct afd_private *ap = AFD_PRIVATE(af); + AFFILE *af2 = afd_file_with_seg(af,name); + if(af2){ + return af_update_seg(af2,name,arg,value,vallen); // update where it was found + } + /* Segment doesn't exist anywhere... */ + /* Append to the last file if there is space and a space limitation... */ + if(ap->num_afs>0){ + AFFILE *af3 = ap->afs[ap->num_afs-1]; + FILE *aseg = af3->aseg; + + uint64_t offset = ftello(aseg); + fseeko(aseg,0,SEEK_END); + + uint64_t len = ftello(aseg); + fseeko(aseg,offset,SEEK_SET); + + if((len + vallen + 1024 < af->maxsize) && (af->maxsize!=0)){ + /* It should fit with room left over! */ + return af_update_seg(af3,name,arg,value,vallen); + } + } + + /* Create a new file and add the segment to it.*/ + if(afd_add_file(af,0)) return -1; + AFFILE *af4 = ap->afs[ap->num_afs-1]; // this is the one just added + return af_update_seg(af4,name,arg,value,vallen); +} + +int afd_del_seg(AFFILE *af,const char *segname) +{ + AFFILE *af2 = afd_file_with_seg(af,segname); + if(af2){ + return af_del_seg(af2,segname); + } + return -1; // not found +} + + +struct af_vnode vnode_afd = { + AF_IDENTIFY_AFD, // + AF_VNODE_TYPE_COMPOUND|AF_VNODE_TYPE_RELIABLE, // + "AFF Directory", + afd_identify_file, + afd_open, // open + afd_close, // close + afd_vstat, // vstat + afd_get_seg, // get_seg + afd_get_next_seg, // get_next_seg + afd_rewind_seg, // rewind_seg + afd_update_seg, // update_seg + afd_del_seg, // del_seg + 0, // read + 0 // write +}; diff --git a/lib/vnode_afd.h b/lib/vnode_afd.h new file mode 100644 index 0000000..448734c --- /dev/null +++ b/lib/vnode_afd.h @@ -0,0 +1,8 @@ +/* + * Distributed under the Berkeley 4-part license + */ + +extern struct af_vnode vnode_afd; /* vnode_afd.cpp */ + +#define AFD_DEFAULT_MAXSIZE 1024*1024*608 + diff --git a/lib/vnode_aff.cpp b/lib/vnode_aff.cpp new file mode 100644 index 0000000..9d51ce7 --- /dev/null +++ b/lib/vnode_aff.cpp @@ -0,0 +1,712 @@ +/* + * vnode_aff.cpp: + * + * Functions for the manipulation of AFF files... + * Distributed under the Berkeley 4-part license + */ + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" +#include "vnode_aff.h" +#include "aff_db.h" + +#ifdef HAVE_SYS_FILE_H +#include +#endif + +#define xstr(s) str(s) +#define str(s) #s + + + +static int aff_write_ignore(AFFILE *af,size_t bytes); +static int aff_write_seg(AFFILE *af,const char *name,uint32_t arg, + const u_char *value,size_t vallen); +static int aff_get_seg(AFFILE *af,const char *name,uint32_t *arg, + unsigned char *data,size_t *datalen); +#ifdef KERNEL_LIBRARY +static int aff_write_seg_no_data(AFFILE *af,const char *name,uint32_t arg, size_t vallen); +#endif +static int aff_get_next_seg(AFFILE *af,char *segname,size_t segname_len, + uint32_t *arg, unsigned char *data, size_t *datalen); + +/** aff_segment_overhead: + * @param segname - the name of a segment + * @return The number of bytes in the AFF file that the segment takes up without the data. + */ + +int aff_segment_overhead(const char *segname) +{ + return sizeof(struct af_segment_head)+sizeof(struct af_segment_tail)+(segname?strlen(segname):0); +} + +static int aff_write_ignore2(AFFILE *af,size_t bytes) +{ +#ifdef KERNEL_LIBRARY + aff_write_seg_no_data(af,AF_IGNORE,0,bytes); +#else + if(af_trace) fprintf(af_trace,"aff_write_ignore2(%p,%d)\n",af,(int)bytes); + unsigned char *invalidate_data = (unsigned char *)calloc(bytes,1); + aff_write_seg(af,AF_IGNORE,0,invalidate_data,bytes); // overwrite with NULLs + free(invalidate_data); +#endif + return 0; +} + +static int aff_write_ignore(AFFILE *af,size_t bytes) +{ + int64_t startpos = ftello(af->aseg); // remember start position + int r = 0; + + if(af_trace) fprintf(af_trace,"aff_write_ignore(%p,%d)\n",af,(int)bytes); + + /* First write the ignore */ + r = aff_write_ignore2(af,bytes); + + /* If the next one is also an ignore, + * then we should go back and make the ignore_size bigger. + * We could do this recursively, + * but it's probably not worth the added complexity. + */ + char next[AF_MAX_NAME_LEN]; + size_t segsize2=0; + int count=0; + while(af_probe_next_seg(af,next,sizeof(next),0,0,&segsize2,1)==0 && next[0]==0 && segsize2>=0){ + count++; + if(count>10) break; // something is wrong; just get out. + //printf("*** next %d segment at %qd len=%d will be deleted\n",count,ftello(af->aseg),segsize2); + bytes += segsize2; + fseeko(af->aseg,startpos,SEEK_SET); + r = aff_write_ignore2(af,bytes); + if(r!=0) return r; + } + + /* See if the previous segment is also blank; if so, collapse them */ + fseeko(af->aseg,startpos,SEEK_SET); + if(af_backspace(af)==0){ + uint64_t prev_segment_loc = ftello(af->aseg); // remember where we are + char prev_segment_name[AF_MAX_NAME_LEN]; + size_t prev_segment_size=0; + if(af_probe_next_seg(af,prev_segment_name,sizeof(prev_segment_name),0,0,&prev_segment_size,1)==0){ + //printf("** prev segment name='%s' len=%d\n",prev_segment_name,prev_segment_size); + if(prev_segment_name[0]==0){ + bytes += prev_segment_size; + fseeko(af->aseg,prev_segment_loc,SEEK_SET); + r = aff_write_ignore2(af,bytes); + fseeko(af->aseg,prev_segment_loc,SEEK_SET); + } + } + } + + return(r); +} + + +/* aff_write_seg: + * put the given named segment at the current position in the file. + * Return 0 for success, -1 for failure (probably disk full?) + * This is the only place where a segment actually gets written + */ + +int aff_write_seg(AFFILE *af, const char *segname,uint32_t arg,const u_char *data,size_t datalen) +{ + if(af_trace) fprintf(af_trace,"aff_write_seg(%p,%s,%"PRIu32",%p,len=%u)\n", + af,segname,arg,data,(int)datalen); + + struct af_segment_head segh; + struct af_segment_tail segt; + + if(af->debug){ + (*af->error_reporter)("aff_write_seg(" POINTER_FMT ",'%s',%lu,data=" POINTER_FMT ",datalen=%u)", + af,segname,arg,data,datalen); + } + + assert(sizeof(segh)==16); + assert(sizeof(segt)==8); + + /* If the last command was not a probe (so we know where we are), and + * we are not at the end of the file, something is very wrong. + */ + + uint32_t segname_len = strlen(segname); + + strcpy(segh.magic,AF_SEGHEAD); + segh.name_len = htonl(segname_len); + segh.data_len = htonl(datalen); + segh.flag = htonl(arg); + + strcpy(segt.magic,AF_SEGTAIL); + segt.segment_len = htonl(sizeof(segh)+segname_len + datalen + sizeof(segt)); + aff_toc_update(af,segname,ftello(af->aseg),datalen); + + + if(af_trace) fprintf(af_trace,"aff_write_seg: putting segment %s (datalen=%d) offset=%"PRId64"\n", + segname,(int)datalen,ftello(af->aseg)); + + if(fwrite(&segh,sizeof(segh),1,af->aseg)!=1) return -10; + if(fwrite(segname,1,segname_len,af->aseg)!=segname_len) return -11; + if(fwrite(data,1,datalen,af->aseg)!=datalen) return -12; + if(fwrite(&segt,sizeof(segt),1,af->aseg)!=1) return -13; + fflush(af->aseg); // make sure it is on the disk + return 0; +} + + +#ifdef KERNEL_LIBRARY +/* aff_write_seg_no_data: + * put the given named segment at the current position in the file but don't write any data. + * this is an attempt at optimizing the write performance + * Return 0 for success, -1 for failure (probably disk full?) + */ + +int aff_write_seg_no_data(AFFILE *af, const char *segname,uint32_t arg,size_t datalen) +{ + struct af_segment_head segh; + struct af_segment_tail segt; + + assert(sizeof(segh)==16); + assert(sizeof(segt)==8); + + /* If the last command was not a probe (so we know where we are), and + * we are not at the end of the file, something is very wrong. + */ + + uint32_t segname_len = strlen(segname); + + strcpy(segh.magic,AF_SEGHEAD); + segh.name_len = htonl(segname_len); + segh.data_len = htonl(datalen); + segh.flag = htonl(arg); + + strcpy(segt.magic,AF_SEGTAIL); + segt.segment_len = htonl(sizeof(segh)+segname_len + datalen + sizeof(segt)); + aff_toc_update(af,segname,ftello(af->aseg),datalen); + + + if(af_trace) fprintf(af_trace,"aff_write_seg: putting segment %s (datalen=%zd) offset=%"PRId64"\n", + segname,datalen,ftello(af->aseg)); + + if(fwrite(&segh,sizeof(segh),1,af->aseg)!=1) return -10; + if(fwrite(segname,1,segname_len,af->aseg)!=segname_len) return -11; + //if(fwrite(data,1,datalen,af->aseg)!=datalen) return -12; + if(fseeko(af->aseg,datalen,SEEK_CUR)!=0) return -12; + if(fwrite(&segt,sizeof(segt),1,af->aseg)!=1) return -13; + fflush(af->aseg); // make sure it is on the disk + return 0; +} +#endif + + +/**************************************************************** + *** low-level routines for reading + ****************************************************************/ + +/* aff_get_segment: + * Get the named segment, using the toc cache. + */ + +static int aff_get_seg(AFFILE *af,const char *name, + uint32_t *arg,unsigned char *data,size_t *datalen) +{ + if(af_trace) fprintf(af_trace,"aff_get_seg(%p,%s,arg=%p,data=%p,datalen=%p)\n",af,name,arg,data,datalen); + + char next[AF_MAX_NAME_LEN]; + + /* If the segment is in the directory, then seek the file to that location. + * Otherwise, we'll probe the next segment, and if it is not there, + * we will rewind to the beginning and go to the end. + */ + struct aff_toc_mem *adm = aff_toc(af,name); + if(!adm) return -1; + + fseeko(af->aseg,adm->offset,SEEK_SET); + int ret = aff_get_next_seg(af,next,sizeof(next),arg,data,datalen); + assert(ret!=0 || strcmp(next,name)==0); // hopefully this is what they asked for + return ret; +} + + + +/** + * Get the next segment. + * @param af - The AFF file pointer + * @param segname - Array to hold the name of the segment. + * @param segname_len - Available space in the segname array. + * @param arg - pointer to the arg + * @param data - pointer to the data + * @param datalen_ - length of the data_ array. If *datalen_==0, set to the length of the data. + * + * @return + * 0 = success. + * -1 = end of file. (AF_ERROR_EOF) + * -2 = *data is not large enough to hold the segment (AF_ERROR_DATASMALL) + * -3 = af file is corrupt; no tail (AF_ERROR_TAIL) + */ +static int aff_get_next_seg(AFFILE *af,char *segname,size_t segname_len,uint32_t *arg, + unsigned char *data,size_t *datalen_) +{ + if(af_trace) fprintf(af_trace,"aff_get_next_seg()\n"); + if(!af->aseg){ + snprintf(af->error_str,sizeof(af->error_str),"af_get_next_segv only works with aff files"); + return AF_ERROR_INVALID_ARG; + } + + uint64_t start = ftello(af->aseg); + size_t data_len; + + int r = af_probe_next_seg(af,segname,segname_len,arg,&data_len,0,0); + if(r<0) return r; // propigate error code + if(data){ /* Read the data? */ + if(datalen_ == 0){ + snprintf(af->error_str,sizeof(af->error_str),"af_get_next_seg: data provided but datalen is NULL"); + return AF_ERROR_INVALID_ARG; + } + size_t read_size = data_len<=*datalen_ ? data_len : *datalen_; + + if(fread(data,1,read_size,af->aseg)!=read_size){ + snprintf(af->error_str,sizeof(af->error_str),"af_get_next_segv: EOF on reading segment? File is corrupt."); + return AF_ERROR_SEGH; + } + if(data_len > *datalen_){ + /* Read was incomplete; + * go back to the beginning of the segment and return + * the incomplete code. + */ + fseeko(af->aseg,start,SEEK_SET); // go back + errno = E2BIG; + return AF_ERROR_DATASMALL; + } + } else { + fseeko(af->aseg,data_len,SEEK_CUR); // skip past the data + } + if(datalen_) *datalen_ = data_len; + + /* Now read the tail */ + struct af_segment_tail segt; + memset(&segt,0,sizeof(segt)); // zero before reading + if(fread(&segt,sizeof(segt),1,af->aseg)!=1){ + snprintf(af->error_str,sizeof(af->error_str), + "af_get_next_segv: end of file reading segment tail; AFF file is truncated (AF_ERROR_TAIL)"); + return AF_ERROR_TAIL; + } + /* Validate tail */ + uint32_t stl = ntohl(segt.segment_len); + uint32_t calculated_segment_len = + sizeof(struct af_segment_head) + + strlen(segname) + + data_len + sizeof(struct af_segment_tail); + + if(strcmp(segt.magic,AF_SEGTAIL)!=0){ + snprintf(af->error_str,sizeof(af->error_str),"af_get_next_segv: AF file is truncated (AF_ERROR_TAIL)."); + fseeko(af->aseg,start,SEEK_SET); // go back to last good position + return AF_ERROR_TAIL; + } + if(stl != calculated_segment_len){ + snprintf(af->error_str,sizeof(af->error_str),"af_get_next_segv: AF file corrupt (%"PRIu32"!=%"PRIu32")/!", + stl,calculated_segment_len); + fseeko(af->aseg,start,SEEK_SET); // go back to last good position + return AF_ERROR_TAIL; + } + return 0; +} + + +static int aff_rewind_seg(AFFILE *af) +{ + if(af_trace) fprintf(af_trace,"aff_rewind_seg()\n"); + fseeko(af->aseg,sizeof(struct af_head),SEEK_SET); // go to the beginning + return 0; +} + + +/* Removes the last segment of an AFF file if it is blank. + * @return 0 for success, -1 for error */ +int af_truncate_blank(AFFILE *af) +{ + uint64_t last_loc = ftello(af->aseg); // remember where we are + if(af_backspace(af)==0){ + uint64_t backspace_loc = ftello(af->aseg); // remember where we are + char next_segment_name[AF_MAX_NAME_LEN]; + if(af_probe_next_seg(af,next_segment_name,sizeof(next_segment_name),0,0,0,1)==0){ + if(next_segment_name[0]==0){ + /* Remove it */ + fflush(af->aseg); + if(ftruncate(fileno(af->aseg),backspace_loc)<0) return -1; + return 0; + } + } + } + fseeko(af->aseg,last_loc,SEEK_SET); // return to where we were + return -1; // say that we couldn't do it. +} + + + + +/**************************************************************** + *** Update functions + ****************************************************************/ + +/* + * af_update_seg: + * Update the given named segment with the new value. + */ + +static int aff_update_seg(AFFILE *af, const char *name, + uint32_t arg,const u_char *value,uint32_t vallen) +{ + char next_segment_name[AF_MAX_NAME_LEN]; + size_t next_segsize = 0; + size_t next_datasize = 0; + + /* if we are updating with a different size, + * remember the location and size of the AF_IGNORE segment that + * has the smallest size that is >= strlen(name)+vallen + */ + size_t size_needed = vallen+aff_segment_overhead(name); + size_t size_closest = 0; + uint64_t loc_closest = 0; + struct aff_toc_mem *adm = aff_toc(af,name); + + if(af_trace) fprintf(af_trace,"aff_update_seg(name=%s,arg=%"PRIu32",vallen=%u)\n",name,arg,vallen); + + + if(adm){ + /* Segment is in the TOC; seek to it */ + fseeko(af->aseg,adm->offset,SEEK_SET); + } + else { + /* Otherwise, go to the beginning of the file and try to find a suitable hole + * TK: This could be made significantly faster by just scanning the TOC for a hole. + */ + af_rewind_seg(af); // start at the beginning + } + + while(af_probe_next_seg(af,next_segment_name,sizeof(next_segment_name),0,&next_datasize,&next_segsize,1)==0){ + /* Remember this information */ + uint64_t next_segment_loc = ftello(af->aseg); +#ifdef DEBUG2 + fprintf(stderr," next_segment_name=%s next_datasize=%d next_segsize=%d next_segment_loc=%qd\n", + next_segment_name, next_datasize, next_segsize,next_segment_loc); +#endif + if(strcmp(next_segment_name,name)==0){ // found the segment + if(next_datasize == vallen){ // Does it exactly fit? + int r = aff_write_seg(af,name,arg,value,vallen); // Yes, just write in place! + return r; + } + + //printf("** Segment '%s' doesn't fit at %qd; invalidating.\n",name,ftello(af->aseg)); + aff_write_ignore(af,next_datasize+strlen(name)); + + /* If we are in random mode, jump back to the beginning of the file. + * This does a good job filling in the holes. + */ + if(af->random_access){ + af_rewind_seg(af); + continue; + } + + /* Otherwise just go to the end. Experience has shown that sequential access + * tends not to generate holes. + */ + fseeko(af->aseg,(uint64_t)0,SEEK_END); // go to the end of the file + break; // and exit this loop + + } + + if((next_segment_name[0]==0) && (next_datasize>=size_needed)){ + //printf(" >> %d byte blank\n",next_datasize); + } + + /* If this is an AF_IGNORE, see if it is a close match */ + if((next_segment_name[0]==AF_IGNORE[0]) && + (next_datasize>=size_needed) && + ((next_datasize=1024 && size_needed>=1024))){ + size_closest = next_datasize; + loc_closest = next_segment_loc; + } + fseeko(af->aseg,next_segsize,SEEK_CUR); // skip this segment + } + + /* Ready to write */ + if(size_closest>0){ + /* Yes. Put it here and put a new AF_IGNORE in the space left-over + * TODO: If the following space is also an AF_IGNORE, then combine the two. + */ + //printf("*** Squeezing it in at %qd. name=%s. vallen=%d size_closest=%d\n",loc_closest,name,vallen,size_closest); + + fseeko(af->aseg,loc_closest,SEEK_SET); // move to the location + aff_write_seg(af,name,arg,value,vallen); // write the new segment + + size_t newsize = size_closest - vallen - aff_segment_overhead(0) - strlen(name); + aff_write_ignore(af,newsize); // write the smaller ignore + return 0; + } + /* If we reach here we are positioned at the end of the file. */ + /* If the last segment is an ignore, truncate the file before writing */ + while(af_truncate_blank(af)==0){ + /* Keep truncating until there is nothing left */ + } + //printf("*** appending '%s' bytes=%d to the end\n",name,vallen); + fseeko(af->aseg,0L,SEEK_END); // move back to the end of the file + return aff_write_seg(af,name,arg,value,vallen); // just write at the end +} + + + +/* Delete the first occurance of the named segment. + * Special case code: See if the segment being deleted + * is the last segment. If it is, truncate the file... + * This handles the case of AF_DIRECTORY and possibly other cases + * as well... + */ + +static int aff_del_seg(AFFILE *af,const char *segname) +{ + if(af_trace) fprintf(af_trace,"aff_del_seg(%p,%s)\n",af,segname); + + if(aff_toc_del(af,segname)){ // if del fails + return 0; // it's not present. + } + + /* Find out if the last segment is the one we are deleting; + * If so, we can just truncate the file. + */ + char last_segname[AF_MAX_NAME_LEN]; + int64_t last_pos; + af_last_seg(af,last_segname,sizeof(last_segname),&last_pos); + if(strcmp(segname,last_segname)==0){ + fflush(af->aseg); // flush any ouput + if(ftruncate(fileno(af->aseg),last_pos)) return -1; // make the file shorter + return 0; + } + + size_t datasize=0,segsize=0; + if(aff_find_seg(af,segname,0,&datasize,&segsize)!=0){ + return -1; // nothing to delete? + } + /* Now wipe it out */ + size_t ignore_size = datasize+strlen(segname); + aff_write_ignore(af,ignore_size); + + return 0; +} + + + +#ifdef HAVE_OPENSSL_RAND_H +#include +#endif + +/* aff_create: + * af is an empty file that is being set up. + */ +static int aff_create(AFFILE *af) +{ + fwrite(AF_HEADER,1,8,af->aseg); // writes the header + aff_toc_build(af); // build the toc (will be pretty small) + af_make_badflag(af); // writes the flag for bad blocks + + const char *version = xstr(PACKAGE_VERSION); + aff_update_seg(af,AF_AFFLIB_VERSION,0,(const u_char *)version,strlen(version)); + +#ifdef HAVE_GETPROGNAME + const char *progname = getprogname(); + if(aff_update_seg(af,AF_CREATOR,0,(const u_char *)progname,strlen(progname))) return -1; +#endif + if(aff_update_seg(af,AF_AFF_FILE_TYPE,0,(const u_char *)"AFF",3)) return -1; + + return 0; +} + + +/**************************************************************** + *** VNODE implementation functions + ****************************************************************/ + +/* Return 1 if a file is an AFF file */ +static int aff_identify_file(const char *filename,int exists) +{ + if(af_is_filestream(filename)==0) return 0; // not a file stream + if(strncmp(filename,"file://",7)==0){ + /* Move file pointer past file:// then find a '/' and take the next character */ + filename += 7; + while(*filename && *filename!='/'){ + filename++; + } + /* At this point if *filename==0 then we never found the end of the URL. + * return 0, since it's not an AFF file. + */ + if(*filename==0) return 0; + + /* So *filename must == '/' */ + assert(*filename == '/'); + filename++; + } + + if(exists && access(filename,R_OK)!=0) return 0; // needs to exist and it doesn't + int fd = open(filename,O_RDONLY | O_BINARY); + if(fd<0){ + /* File doesn't exist. Is this an AFF name? */ + if(af_ext_is(filename,"aff")) return 1; + return 0; + } + + if(fd>0){ + int len = strlen(AF_HEADER)+1; + char buf[64]; + int r = read(fd,buf,len); + close(fd); + if(r==len){ // if I could read the header + if(strcmp(buf,AF_HEADER)==0) return 1; // must be an AFF file + return 0; // not an AFF file + } + /* If it is a zero-length file and the file extension ends AFF, + * then let it be an AFF file... + */ + if(r==0 && af_ext_is(filename,"aff")) return 1; + return 0; // must not be an aff file + } + return 0; +} + + + +static int aff_open(AFFILE *af) +{ + if(af_is_filestream(af->fname)==0) return -1; // not a file stream + + /* Open the raw file */ + int fd = open(af->fname,af->openflags | O_BINARY,af->openmode); + if(fd<0){ // couldn't open + return -1; + } + + /* Lock the file if writing */ +#ifdef HAVE_FLOCK + if(af->openflags & O_RDWR){ + int lockmode = LOCK_SH; // default + if((af->openflags & O_ACCMODE)==O_RDWR) lockmode = LOCK_EX; // there can be only one + if(flock(fd,lockmode)){ + warn("Cannot exclusively lock %s:",af->fname); + } + } +#endif + + /* Set defaults */ + + af->compression_type = AF_COMPRESSION_ALG_ZLIB; + af->compression_level = Z_DEFAULT_COMPRESSION; + + /* Open the FILE for the AFFILE */ + char strflag[8]; + strcpy(strflag,"rb"); // we have to be able to read + if(af->openflags & O_RDWR) strcpy(strflag,"w+b"); + + af->aseg = fdopen(fd,strflag); + if(!af->aseg){ + (*af->error_reporter)("fdopen(%d,%s)",fd,strflag); + return -1; + } + + /* Get file size */ + struct stat sb; + if(fstat(fd,&sb)){ + (*af->error_reporter)("aff_open: fstat(%s): ",af->fname); // this should not happen + return -1; + } + + /* If file is empty, then put out an AFF header, badflag, and AFF version */ + if(sb.st_size==0){ + return aff_create(af); + } + + /* We are opening an existing file. Verify once more than it is an AFF file + * and skip past the header... + */ + + char buf[8]; + if(fread(buf,sizeof(buf),1,af->aseg)!=1){ + /* Hm. End of file. That shouldn't happen here. */ + (*af->error_reporter)("aff_open: couldn't read AFF header on existing file?"); + return -1; // should not happen + } + + if(strcmp(buf,AF_HEADER)!=0){ + buf[7] = 0; + (*af->error_reporter)("aff_open: %s is not an AFF file (header=%s)\n", + af->fname,buf); + return -1; + } + + /* File has been validated */ + if(aff_toc_build(af)) return -1; // build the TOC + return 0; // everything must be okay. +} + + +/* + * aff_close: + * If the imagesize changed, write out a new value. + */ +static int aff_close(AFFILE *af) +{ + aff_toc_free(af); + fclose(af->aseg); + return 0; +} + + +static int aff_vstat(AFFILE *af,struct af_vnode_info *vni) +{ + memset(vni,0,sizeof(*vni)); // clear it + vni->imagesize = af->image_size; // we can just return this + vni->pagesize = af->image_pagesize; + vni->supports_compression = 1; + vni->has_pages = 1; + vni->supports_metadata = 1; + vni->cannot_decrypt = af_cannot_decrypt(af) ? 1 : 0; + + /* Check for an encrypted page */ + if(af->toc){ + for(int i=0;itoc_count;i++){ + if(af->toc[i].name){ + bool is_page = false; + vni->segment_count_total++; + if(af_segname_page_number(af->toc[i].name)>=0){ + vni->page_count_total++; + is_page = true; + } + if(af_is_encrypted_segment(af->toc[i].name)){ + vni->segment_count_encrypted++; + if(is_page) vni->page_count_encrypted++; + } + if(af_is_signature_segment(af->toc[i].name)){ + vni->segment_count_signed++; + } + } + } + } + return 0; +} + + +struct af_vnode vnode_aff = { + AF_IDENTIFY_AFF, + AF_VNODE_TYPE_PRIMITIVE|AF_VNODE_TYPE_RELIABLE, + "AFF", + aff_identify_file, + aff_open, + aff_close, + aff_vstat, + aff_get_seg, + aff_get_next_seg, + aff_rewind_seg, + aff_update_seg, + aff_del_seg, + 0, // read; keep 0 + 0 // write +}; + diff --git a/lib/vnode_aff.h b/lib/vnode_aff.h new file mode 100644 index 0000000..3f8f1fe --- /dev/null +++ b/lib/vnode_aff.h @@ -0,0 +1,7 @@ +/* + * Distributed under the Berkeley 4-part license + */ + +extern struct af_vnode vnode_aff; /* vnode_aff.cpp */ + + diff --git a/lib/vnode_afm.cpp b/lib/vnode_afm.cpp new file mode 100644 index 0000000..b65c051 --- /dev/null +++ b/lib/vnode_afm.cpp @@ -0,0 +1,368 @@ +/* vnode_afm: afm raw file implementation with optional metadata support + * Distributed under the Berkeley 4-part license + */ + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" +#include "vnode_afm.h" +#include "vnode_aff.h" +#include "vnode_split_raw.h" + +/* + * Notes on implementation: + * AF_PAGES_PER_IMAGE_FILE is a segment read as a 64-bit number. + * If its value is 0, then do not split the files. + */ + +struct afm_private { + AFFILE *aff; // the AFFILE we use for the actual metadata + AFFILE *sr; // the AFFILE we use for the splitraw + int sr_initialized; // has the split-raw been setup from AFM? +}; + + +static inline struct afm_private *AFM_PRIVATE(AFFILE *af) +{ + assert(af->v == &vnode_afm); + return (struct afm_private *)(af->vnodeprivate); +} + +/* Return 1 if a file has the AFF header or if the file doesn't + * exist and its extension is .afm + */ +static int afm_identify_file(const char *filename,int exists) +{ + if(exists && access(filename,R_OK)!=0) return 0; // needs to exist and it doesn't + return af_ext_is(filename,"afm"); +} + + +static int afm_close(AFFILE *af) +{ + struct afm_private *ap = AFM_PRIVATE(af); + if(ap){ + if(ap->sr) af_close(ap->sr); // close the split files + if(ap->aff) af_close(ap->aff); // and close the AFF file + memset(ap,0,sizeof(*ap)); + free(ap); + } + return 0; +} + + +static int invalid_extension_char(const char *extension,int ext_len) +{ + for(int i=0;ierror_reporter)("split_raw_read_write_setup: %s: failed to write %s\n", + af->fname, AF_RAW_IMAGE_FILE_EXTENSION); + afm_close(af); // close the sub-file + return -1; // we failed + } + af_set_pagesize(af,AFM_DEFAULT_PAGESIZE); + af_update_seg(af,AF_AFF_FILE_TYPE,0,(const u_char *)"AFM",3); + return 0; +} + +/* + * The AFM is actually a shell that opens two + * sub-implementations in their own files: an AFF file + * and a split_raw file. + */ + +static int afm_open(AFFILE *af) +{ + af->vnodeprivate = (void *)calloc(sizeof(struct afm_private),1); + struct afm_private *ap = AFM_PRIVATE(af); + ap->aff = af_open_with(af_filename(af),af->openflags,af->openmode,&vnode_aff); + + if(ap->aff==0){ // open failed? + afm_close(af); + return -1; + } + ap->aff->parent = af; + + /* If this is a new file, write out the default split raw extension */ + if(af->exists == 0){ + if(afm_create(af)) return -1; + } + + /* If this is an old file, read the image_pagesize */ + if(af->exists){ + af->image_pagesize = ap->aff->image_pagesize; + } + + /* Read the split raw extension */ + char raw_file_extension[4]; + size_t len=3; // don't overwrite the NUL + + memset(raw_file_extension,0,sizeof(raw_file_extension)); + if (af_get_seg(ap->aff,AF_RAW_IMAGE_FILE_EXTENSION,0,(unsigned char *)raw_file_extension,&len)) { + (*af->error_reporter)("afm_open: %s: %s segment missing or too large\n", + af_filename(af),AF_RAW_IMAGE_FILE_EXTENSION); + afm_close(af); + return -1; + } + if(invalid_extension_char(raw_file_extension,len)){ + (*af->error_reporter)("afm_open: file extension contains invalid character\n", + af->fname, AF_RAW_IMAGE_FILE_EXTENSION); + afm_close(af); + return -1; + } + + /* Now open the splitraw file */ + char *sr_filename = strdup(af_filename(af)); + char *ext = strrchr(sr_filename,'.'); + if(!ext){ + (*af->error_reporter)("afm_open: cannot find extension in '%s'",sr_filename); + free(sr_filename); + afm_close(af); + return -1; + } + ext++; // skip past '.' + if(strlen(ext) != strlen(raw_file_extension)){ + (*af->error_reporter)("afm_open: file extension in '%s' too short",sr_filename); + free(sr_filename); + afm_close(af); + return -1; + } + + strcpy(ext,raw_file_extension); + ap->sr = af_open_with(sr_filename,af->openflags,af->openmode,&vnode_split_raw); + if(ap->sr==0){ + (*af->error_reporter)("afm_open: could not open '%s'",sr_filename); + free(sr_filename); + afm_close(af); + return -1; + } + ap->sr->parent = af; + free(sr_filename); + + /* Additional setup will happen first time a data read/write call is made + * by the function afm_read_write_setup(). + * This allows a new file to be created with af_open() and then to have + * the parameters set with af_update_seg() calls, yet the split_raw + * implementation gets the proper settings + */ + return 0; +} + + + +/* afm_split_raw_setup: + * Sets up the parameters of the split-raw by reading from the metadata file. + * The advantage of doing it this way is that a new file can be opened, + * the metadata parmeters set, and then an af_write() call made, and the values. + * get copied. + */ +static int afm_split_raw_setup(AFFILE *af) +{ + struct afm_private *ap = AFM_PRIVATE(af); + if(ap->sr_initialized) return 0; // already setup + + /* The size of AF_PAGES_PER_RAW_IMAGE_FILE indicates whether the file is split. + * If it is not present, or if it is 0-length, assume that the file is not split. + */ + uint64_t pages_per_file = 0; + size_t len = 0; + if (af_get_seg(ap->aff,AF_PAGES_PER_RAW_IMAGE_FILE,0,0,&len)) { + + /* Not in file; put it there based on maxsize and image_pagesize, + * both of which better be set at this point + */ + if (af->image_pagesize < 1) { + (*af->error_reporter)("afm_split_raw_setup: image_pagesize==0\n"); + return -1; + } + if (af->maxsize % af->image_pagesize) { + (*af->error_reporter)("afm_split_raw_setup: maxsize (%"I64d") " + "not a multiple of image_pagesize (%d)\n", + af->maxsize,af->image_pagesize); + return -1; + } + pages_per_file = af->maxsize / af->image_pagesize; + if (af_update_segq (af, AF_PAGES_PER_RAW_IMAGE_FILE, pages_per_file)) { + (*af->error_reporter)("split_raw_read_write_setup: %s: failed to write %s\n", + af_filename(af), AF_PAGES_PER_RAW_IMAGE_FILE); + return -1; + } + } + + /* Now, read the segment (which might have just been put there) and set up the split_raw file */ + if(af_get_segq(af,AF_PAGES_PER_RAW_IMAGE_FILE,(int64_t *)&pages_per_file)){ + (*af->error_reporter)("split_raw_read_write_setup: %s: failed to write %s\n", + af_filename(af), AF_PAGES_PER_RAW_IMAGE_FILE); + return -1; + } + + /* Verify that splitraw's notion of the rawfilesize is the same as the + * metadata's notion if the AFF image_size has been set + */ + if (ap->aff->image_size && ap->aff->image_size != ap->sr->image_size) { + (*af->error_reporter)("afm_split_raw_setup: internal error. " + "AFF image_size %"I64d" != SR image_size %"I64d"\n", + ap->aff->image_size,ap->sr->image_size); + return -1; + } + + /* Uses pages_per_file to set the maxsize of the split_raw if it hasn't been set yet*/ + if(ap->sr->maxsize==0){ + ap->sr->maxsize = pages_per_file * af->image_pagesize; + } + + /* Verify that the parameters make sense */ + if (ap->sr->maxsize != (pages_per_file * af->image_pagesize) && pages_per_file>0) { + (*af->error_reporter)("afm_split_raw_setup: %s: per size indicated by metadata (%d * %d) " + "doesn't match maxsize (%"I64d")\n", + af_filename(af),pages_per_file,af->image_pagesize,ap->sr->maxsize); + return -1; + } + + /* Push down the image_pagesize from the AFM to the split_raw */ + uint32_t image_pagesize = af->image_pagesize; // default to what's in memory + af_get_seg(af,AF_PAGESIZE,&image_pagesize,0,0); // get from the AFF file if possible + ap->sr->image_pagesize = af->image_pagesize; // overwrite the default with what the AFM file + + ap->sr_initialized = 1; + return 0; +} + +/* + * the stat of the afm is the stat of the metafile, + * but we don't do compression. + */ +static int afm_raw_vstat(AFFILE *af,struct af_vnode_info *vni) +{ + memset(vni,0,sizeof(*vni)); // clear it + struct afm_private *ap = AFM_PRIVATE(af); + af_vstat(ap->aff,vni); + vni->supports_compression = 0; + vni->supports_metadata = 1; + return 0; +} + + +/* + * afm_get_seg: + * If it is a page segment, satisfy it from the splitraw, + * otherwise from the aff file. + */ +static int afm_get_seg(AFFILE *af,const char *name,uint32_t *arg,unsigned char *data,size_t *datalen) +{ + struct afm_private *ap = AFM_PRIVATE(af); + int64_t page_num = af_segname_page_number(name); + if(page_num>=0) return af_get_seg(ap->sr,name,arg,data,datalen); + return af_get_seg(ap->aff,name,arg,data,datalen); + +} + + +/* + * afm_del_seg: + * If it is a page segment, generate an error. + * otherwise from the aff file. + */ +static int afm_del_seg(AFFILE *af,const char *segname) +{ + struct afm_private *ap = AFM_PRIVATE(af); + int64_t page_num = af_segname_page_number(segname); + if(page_num>=0){ + errno = ENOTSUP; + return -1; + } + return af_del_seg(ap->aff,segname); +} + + +/* + * afm_get_next_seg: + * Try get_next_seg on the AFF file first until it has none left. + * Then call get_next_seg of the splitraw until it has noneleft. + */ + +static int afm_get_next_seg(AFFILE *af,char *segname,size_t segname_len,uint32_t *arg, + unsigned char *data,size_t *datalen_) +{ + struct afm_private *ap = AFM_PRIVATE(af); + int r = af_get_next_seg(ap->aff,segname,segname_len,arg,data,datalen_); + if(r==-1) return af_get_next_seg(ap->sr,segname,segname_len,arg,data,datalen_); + return r; +} + +/* afm_rewind_seg: + * Rewind both the AFF file and the split_raw file(s) + */ +static int afm_rewind_seg(AFFILE *af) +{ + struct afm_private *ap = AFM_PRIVATE(af); + if ( af_rewind_seg(ap->aff) ) return -1; // that's bad + return af_rewind_seg(ap->sr); // and rewind the splitraw +} + + +/* For afm_update_seg, hand off page updates to the split_raw implementation + * and metadata updates to the AFF implementation. + */ +static int afm_update_seg(AFFILE *af, const char *name, + uint32_t arg,const u_char *value,uint32_t vallen) + +{ + struct afm_private *ap = AFM_PRIVATE(af); + int64_t page_num = af_segname_page_number(name); // <0 means update metadata + if(page_num<0){ + return af_update_seg(ap->aff,name,arg,value,vallen); + } + return af_update_seg(ap->sr,name,arg,value,vallen); +} + + +static int afm_read(AFFILE *af, unsigned char *buf, uint64_t pos,size_t count) +{ + struct afm_private *ap = AFM_PRIVATE(af); + if(ap->sr_initialized==0 && afm_split_raw_setup(af)) return -1; + return (*ap->sr->v->read)(ap->sr,buf,pos,count); +} + +static int afm_write(AFFILE *af, unsigned char *buf, uint64_t pos,size_t count) +{ + struct afm_private *ap = AFM_PRIVATE(af); + if(ap->sr_initialized==0 && afm_split_raw_setup(af)) return -1; + af_set_callback(ap->sr,af->w_callback); // update the callback + int r = (*ap->sr->v->write)(ap->sr,buf,pos,count); // call split_raw's write + if(ap->sr->image_size > af->image_size){ + af->image_size = ap->sr->image_size; // image was extended; note this in parent & AFF file + ap->aff->image_size = ap->sr->image_size; // copy over the image size + } + return r; +} + + +struct af_vnode vnode_afm = { + AF_IDENTIFY_AFM, + AF_VNODE_TYPE_COMPOUND|AF_VNODE_TYPE_RELIABLE|AF_VNODE_MAXSIZE_MULTIPLE|AF_VNODE_NO_SEALING, + "AFM (AFF metadata with split raw file)", + afm_identify_file, + afm_open, + afm_close, + afm_raw_vstat, // nothing aff specific here. + afm_get_seg, // get seg + afm_get_next_seg, // get_next_seg + afm_rewind_seg, // rewind_seg + afm_update_seg, // update_seg + afm_del_seg, // del_seg (afm_open fills in aff_del_seg) + afm_read, // read + afm_write // write +}; + diff --git a/lib/vnode_afm.h b/lib/vnode_afm.h new file mode 100644 index 0000000..9070f63 --- /dev/null +++ b/lib/vnode_afm.h @@ -0,0 +1,6 @@ +/* + * Distributed under the Berkeley 4-part license + */ + +extern struct af_vnode vnode_afm; +#define AFM_DEFAULT_PAGESIZE 1024*1024*16 diff --git a/lib/vnode_qemu.cpp b/lib/vnode_qemu.cpp new file mode 100644 index 0000000..d40b231 --- /dev/null +++ b/lib/vnode_qemu.cpp @@ -0,0 +1,261 @@ +/* + * AFF/qemu glue + * + * 2008 by Simson L. Garfinkel + * + * This file is a work of a US government employee and as such is in the Public domain. + * Simson L. Garfinkel, March 12, 2012 + * + */ + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" + +#ifdef USE_QEMU + +extern "C" { +#include "qemu/qemu-common.h" +#include "qemu/block_int.h" +} + + +/* Return 1 if a file is a qemu file... */ +static int vmdk_identify_file(const char *filename,int exists) +{ + return af_ext_is(filename,"vmdk"); +} + +/* Return 1 if a file is a qemu file... */ +static int dmg_identify_file(const char *filename,int exists) +{ + return af_ext_is(filename,"dmg"); +} + +/* Return 1 if a file is a qemu file... */ +static int sparseimage_identify_file(const char *filename,int exists) +{ + return af_ext_is(filename,"sparseimage"); +} + +#define QEMU_HANDLE(af) ((BlockDriverState *)af->vnodeprivate) + +static int qemu_open(AFFILE *af) +{ + BlockDriverState *bs; + BlockDriver *drv=NULL; + uint64_t total_sectors=0; + static int bdrv_init_called = 0; + + if(bdrv_init_called==0){ // DO NOT CALL MORE THAN ONCE + bdrv_init(); + bdrv_init_called = 1; + } + bs = bdrv_new(""); + if (bs == NULL) return -1; + if(bdrv_open2(bs,af_filename(af),0,drv)!=0){ + bdrv_delete(bs); + return -1; + } + bdrv_get_geometry(bs, &total_sectors); + + af->image_pagesize = 1024*1024*1; // megabyte for now + af->image_size = total_sectors * 512; + + af->vnodeprivate = (void *)bs; + return 0; +} + + +static int qemu_vstat(AFFILE *af,struct af_vnode_info *vni) +{ + vni->imagesize = af->image_size; + vni->pagesize = af->image_pagesize; + vni->has_pages = 1; // use the AFF page system + return 0; +} + +static int qemu_close(AFFILE *af) +{ + bdrv_delete(QEMU_HANDLE(af)); + return 0; +} + + +static int qemu_rewind_seg(AFFILE *af) +{ + af->cur_page = -1; // starts at the metadata + return 0; +} + + +static int qemu_get_seg(AFFILE *af,const char *name, uint32_t *arg, + unsigned char *data,size_t *datalen) +{ + /* Is the user asking for a page? */ + int64_t segnum = af_segname_page_number(name); + if(segnum>=0){ + /* Get the segment number */ + if(data==0){ + /* Need to make sure that the segment exists */ + if(segnum * (af->image_pagesize+1) > (int64_t) af->image_size ){ + return -1; // this segment does not exist + } + if(datalen) *datalen =af->image_pagesize; // just return the chunk size + return 0; + } + int64_t sector_start = segnum * af->image_pagesize / 512; + u_int sector_count = af->image_pagesize/512; + if(datalen==0) return -1; + if(sector_count*512 > *datalen) return -1; // no room + return bdrv_read(QEMU_HANDLE(af),sector_start,data,sector_count); + } + + /* See if it is a page name we understand */ + if(strcmp(name,AF_PAGESIZE)==0){ + if(arg) *arg = af->image_pagesize; + return 0; + } + if(strcmp(name,AF_IMAGESIZE)==0){ + if(arg) *arg = 0; + if(datalen==0) return 0; + if(*datalen==0){ + *datalen = 8; // the structure is 8 bytes long + return 0; + } + if(*datalen<8) return -2; + + struct aff_quad q; + q.low = htonl((uint32_t)(af->image_size & 0xffffffff)); + q.high = htonl((uint32_t)(af->image_size >> 32)); + memcpy(data,&q,8); + return 0; + } + if(strcmp(name,AF_SECTORSIZE)==0){ + if(arg) *arg=512; // seems to be what QEMU uses + if(datalen) *datalen = 0; + return 0; + } + if(strcmp(name,AF_DEVICE_SECTORS)==0){ + /* Is this in flag or a quad word? */ + if(arg) *arg = af->image_size / 512; + if(datalen) *datalen = 0; + return 0; + } + + /* They are asking for a metdata segment. If we have wide character type + * compiled in for libqemu, just ignore it, because afflib doesn't do wide characters + * at the moment... + */ + + return -1; // don't know this header +} + +static const char *emap[] = { + AF_PAGESIZE, + AF_IMAGESIZE, + AF_SECTORSIZE, + AF_DEVICE_SECTORS, + 0 +}; + + +static int qemu_get_next_seg(AFFILE *af,char *segname,size_t segname_len,uint32_t *arg, + unsigned char *data,size_t *datalen) +{ + /* Figure out what the next segment would be, then get it */ + /* Metadata first */ + if(af->cur_page<0){ + /* Find out how many mapped segments there are */ + int mapped=0; + for(mapped=0;emap[mapped];mapped++){ + } + if(-af->cur_page >= mapped ){ + af->cur_page = 0; + goto get_next_data_seg; + } + int which = 0 - af->cur_page; // which one to get + af->cur_page--; // go to the next one + if(segname_len < strlen(emap[which])) return -2; // not enough room for segname + strlcpy(segname,emap[which],segname_len); // give caller the name of the mapped segment. + return qemu_get_seg(af,segname,arg,data,datalen); + } + + get_next_data_seg: + if(af->cur_page * af->image_pagesize >= (int64_t)af->image_size) return -1; // end of list + /* Make the segment name */ + char pagename[AF_MAX_NAME_LEN]; // + memset(pagename,0,sizeof(pagename)); + snprintf(pagename,sizeof(pagename),AF_PAGE,af->cur_page++); + + int r = 0; + /* Get the segment, if it is wanted */ + if(data) r = qemu_get_seg(af,pagename,arg,data,datalen); + + /* If r==0 and there is room for copying in the segment name, return it */ + if(r==0){ + if(strlen(pagename)+1 < segname_len){ + strlcpy(segname,pagename,segname_len); + return 0; + } + /* segname wasn't big enough */ + return -2; + } + return r; // some other error +} + +struct af_vnode vnode_vmdk = { + AF_IDENTIFY_VMDK, + AF_VNODE_TYPE_PRIMITIVE|AF_VNODE_NO_SIGNING|AF_VNODE_NO_SEALING, + "VMDK(LIBQEMU)", + vmdk_identify_file, + qemu_open, + qemu_close, + qemu_vstat, + qemu_get_seg, // get seg + qemu_get_next_seg, // get_next_seg + qemu_rewind_seg, // rewind_seg + 0, // update_seg + 0, // del_seg + 0, // read + 0 // write +}; + + +struct af_vnode vnode_dmg = { + AF_IDENTIFY_DMG, + AF_VNODE_TYPE_PRIMITIVE|AF_VNODE_NO_SIGNING|AF_VNODE_NO_SEALING, + "DMG(LIBQEMU)", + dmg_identify_file, + qemu_open, + qemu_close, + qemu_vstat, + qemu_get_seg, // get seg + qemu_get_next_seg, // get_next_seg + qemu_rewind_seg, // rewind_seg + 0, // update_seg + 0, // del_seg + 0, // read + 0 // write +}; + + +struct af_vnode vnode_sparseimage = { + AF_IDENTIFY_SPARSEIMAGE, + AF_VNODE_TYPE_PRIMITIVE|AF_VNODE_NO_SIGNING|AF_VNODE_NO_SEALING, + "SPARSEIMAGE(LIBQEMU)", + sparseimage_identify_file, + qemu_open, + qemu_close, + qemu_vstat, + qemu_get_seg, // get seg + qemu_get_next_seg, // get_next_seg + qemu_rewind_seg, // rewind_seg + 0, // update_seg + 0, // del_seg + 0, // read + 0 // write +}; + + +#endif diff --git a/lib/vnode_qemu.h b/lib/vnode_qemu.h new file mode 100644 index 0000000..b20f2a5 --- /dev/null +++ b/lib/vnode_qemu.h @@ -0,0 +1,9 @@ +/* + * This file is a work of a US government employee and as such is in the Public domain. + * Simson L. Garfinkel, March 12, 2012 + */ + +extern struct af_vnode vnode_vmdk; +extern struct af_vnode vnode_dmg; +extern struct af_vnode vnode_sparseimage; + diff --git a/lib/vnode_raw.cpp b/lib/vnode_raw.cpp new file mode 100644 index 0000000..fab306e --- /dev/null +++ b/lib/vnode_raw.cpp @@ -0,0 +1,317 @@ +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" +#include "vnode_raw.h" + +/* + * Distributed under the Berkeley 4-part license + */ + +/* the RAW_PAGESIZE is visible outside the module, but it's kind of irrevellant */ +#define RAW_PAGESIZE 16*1024*1024 + +/* raw file implementation */ +struct raw_private { + /* For Raw files */ + FILE *raw; // if it is a raw file + int raw_popen; // opened with popen +}; + +#define RAW_PRIVATE(af) ((struct raw_private *)(af->vnodeprivate)) + +/* Return 1 if a file is a raw file... */ +static int raw_identify_file(const char *filename,int exists) +{ + if(exists && access(filename,R_OK)!=0) return 0; // needs to exist and it doesn't + return access(filename,R_OK)==0; // if we can read it, it's raw... +} + + +/* Return the size of the raw file */ +static int64_t raw_filesize(AFFILE *af) +{ + struct raw_private *rp = RAW_PRIVATE(af); + + struct stat sb; + if(fstat(fileno(rp->raw),&sb)==0){ + if(sb.st_mode & S_IFREG){ // only do this for regular files + return sb.st_size; + } + + /* See if this is a device that we can figure */ + struct af_figure_media_buf afb; + if(af_figure_media(fileno(rp->raw),&afb)==0){ + if(afb.total_sectors>0 && afb.sector_size>0){ + return afb.total_sectors * afb.sector_size; + } + } + } + return 0; // no clue +} + +static int raw_open(AFFILE *af) +{ + /* Raw is the passthrough system. + * Right now, it is read only... + */ + const char *mode = "rb"; + if(af->openflags && (O_RDWR | O_WRONLY)) mode = "r+b"; + + af->vnodeprivate = (void *)calloc(1,sizeof(struct raw_private)); + struct raw_private *rp = RAW_PRIVATE(af); + + if(af->fname) rp->raw=fopen(af->fname,mode); + if(rp->raw==0) return -1; // raw open failed + af->image_size = raw_filesize(af); + af->image_pagesize = RAW_PAGESIZE; + af->cur_page = 0; + return 0; +} + +int raw_freopen(AFFILE *af,FILE *file) +{ + af->fname = 0; + af->vnodeprivate = (void *)calloc(1,sizeof(struct raw_private)); + struct raw_private *rp = RAW_PRIVATE(af); + rp->raw = file; + af->image_size = raw_filesize(af); + af->image_pagesize = RAW_PAGESIZE; + af->cur_page = 0; + return 0; +} + + +int raw_popen(AFFILE *af,const char *command,const char *type) +{ +#ifdef HAVE_POPEN + if(strcmp(type,"r")!=0){ + (*af->error_reporter)("af_popen: only type 'r' supported"); + return -1; + } + /* If shell metacharacters exist in command, don't open it */ + if(af_hasmeta(command)){ + (*af->error_reporter)("raw_popen: invalid shell metacharacters in command '%s'", + command); + return -1; + } + af->fname = 0; + af->vnodeprivate = (void *)calloc(1,sizeof(struct raw_private)); + struct raw_private *rp = RAW_PRIVATE(af); + rp->raw = popen(command,"r"); + rp->raw_popen = 1; + return 0; +#else + (*af->error_reporter)("af_popen: popen not supported on this platform."); + return -1; +#endif +} + + +static int raw_close(AFFILE *af) +{ + struct raw_private *rp = RAW_PRIVATE(af); + + if(rp->raw_popen){ +#ifdef HAVE_POPEN + pclose(rp->raw); +#endif + } + else { + fclose(rp->raw); + } + memset(rp,0,sizeof(*rp)); // clean object reuse + free(rp); // won't need it again + return 0; +} + +static int raw_get_seg(AFFILE *af,const char *name, + uint32_t *arg,unsigned char *data,size_t *datalen) +{ + struct raw_private *rp = RAW_PRIVATE(af); + + int64_t segnum = af_segname_page_number(name); + if(segnum<0){ + /* See if PAGESIZE or IMAGESIZE is being requested; we can fake those */ + if(strcmp(name,AF_PAGESIZE)==0){ + if(arg) *arg = af->image_pagesize; + if(datalen) *datalen = 0; + return 0; + } + if(strcmp(name,AF_IMAGESIZE)==0){ + struct aff_quad q; + if(data && *datalen>=8){ + q.low = htonl((uint32_t)(af->image_size & 0xffffffff)); + q.high = htonl((uint32_t)(af->image_size >> 32)); + memcpy(data,&q,8); + *datalen = 8; + } + return 0; + } + if(strcmp(name,AF_SECTORSIZE)==0){ + if(arg) *arg = af->image_sectorsize; + if(datalen) *datalen = 0; + return 0; + } + if(strcmp(name,AF_DEVICE_SECTORS)==0){ + int64_t devicesectors = af->image_size / af->image_sectorsize; + struct aff_quad q; + if(data && *datalen>=8){ + q.low = htonl((uint32_t)(devicesectors & 0xffffffff)); + q.high = htonl((uint32_t)(devicesectors >> 32)); + memcpy(data,&q,8); + *datalen = 8; + } + return 0; + } + + return -1; // don't know how to fake this + } + + fflush(rp->raw); // make sure that any buffers are flushed + + int64_t pos = (int64_t)segnum * af->image_pagesize; // where we are to start reading + int64_t bytes_left = af->image_size - pos; // how many bytes left in the file + + if(bytes_left<0) bytes_left = 0; + + int bytes_to_read = af->image_pagesize; // copy this many bytes, unless + if(bytes_to_read > bytes_left) bytes_to_read = bytes_left; // only this much is left + + if(arg) *arg = 0; // arg is always 0 + if(datalen){ + if(data==0){ // asked for 0 bytes, so give the actual size + *datalen = bytes_to_read; + return 0; + } + if(*datalen < (unsigned)bytes_to_read){ + *datalen = bytes_to_read; + return AF_ERROR_DATASMALL; + } + } + if(data){ + fseeko(rp->raw,pos,SEEK_SET); + int bytes_read = fread(data,1,bytes_to_read,rp->raw); + if(bytes_read==bytes_to_read){ + if(datalen) *datalen = bytes_read; + return 0; + } + return -1; // some kind of EOF? + } + return 0; // no problems! +} + + +int raw_update_seg(AFFILE *af, const char *name, + uint32_t arg,const u_char *value,uint32_t vallen) +{ + struct raw_private *rp = RAW_PRIVATE(af); + + /* Simple implementation; only updates data segments */ + int64_t pagenum = af_segname_page_number(name); + if(pagenum<0){ + errno = ENOTSUP; + return -1; // not a segment number + } + int64_t pos = pagenum * af->image_pagesize; // where we are to start reading + fseeko(rp->raw,pos,SEEK_SET); + + if(fwrite(value,vallen,1,rp->raw)==1){ + return 0; + } + return -1; // some kind of error... +} + + +static int raw_vstat(AFFILE *af,struct af_vnode_info *vni) +{ + struct raw_private *rp = RAW_PRIVATE(af); + + vni->imagesize = -1; + vni->pagesize = RAW_PAGESIZE; // decent page size + vni->supports_metadata = 0; + vni->is_raw = 1; + vni->changable_pagesize = 1; // change it at any time + vni->changable_sectorsize = 1; // change it at any time + + /* If we can stat the file, use that. */ + fflush(rp->raw); + vni->imagesize = raw_filesize(af); + vni->supports_compression = 0; + vni->has_pages = 1; + + if(rp->raw_popen){ + /* popen files require special handling */ + vni->has_pages = 0; + vni->use_eof = 1; + vni->at_eof = feof(rp->raw); // are we there yet? + } + return 0; +} + +static int raw_rewind_seg(AFFILE *af) +{ + af->cur_page = 0; + return 0; +} + + +static int raw_get_next_seg(AFFILE *af,char *segname,size_t segname_len,uint32_t *arg, + unsigned char *data,size_t *datalen) +{ + + /* See if we are at the end of the "virtual" segment list */ + if((uint64_t)af->cur_page * af->image_pagesize >= af->image_size) return -1; + + /* Make the segment name */ + char pagename[AF_MAX_NAME_LEN]; // + memset(pagename,0,sizeof(pagename)); + snprintf(pagename,sizeof(pagename),AF_PAGE,af->cur_page++); + + /* Get the segment, if we can */ + int r = raw_get_seg(af,pagename,arg,data,datalen); + + /* If r==0 and there is room for copying in the segment name, return it */ + if(r==0){ + if(strlen(pagename)+1 < segname_len){ + strcpy(segname,pagename); + return 0; + } + /* segname wasn't big enough */ + return -2; + } + return r; // some other error +} + +static int raw_read(AFFILE *af, unsigned char *buf, uint64_t pos,size_t count) +{ + struct raw_private *rp = RAW_PRIVATE(af); + fseeko(rp->raw,pos,SEEK_SET); + return fread(buf,1,count,rp->raw); +} + +static int raw_write(AFFILE *af, unsigned char *buf, uint64_t pos,size_t count) +{ + struct raw_private *rp = RAW_PRIVATE(af); + if(fseeko(rp->raw,pos,SEEK_SET)<0) return -1; + return fwrite(buf,1,count,rp->raw); +} + + + +struct af_vnode vnode_raw = { + AF_IDENTIFY_RAW, + AF_VNODE_TYPE_PRIMITIVE|AF_VNODE_TYPE_RELIABLE|AF_VNODE_NO_SIGNING|AF_VNODE_NO_SEALING, + "Raw", + raw_identify_file, + raw_open, + raw_close, + raw_vstat, + raw_get_seg, // get seg + raw_get_next_seg, // get_next_seg + raw_rewind_seg, // rewind_seg + raw_update_seg, // update_seg + 0, // del_seg + raw_read, // read + raw_write // write +}; + diff --git a/lib/vnode_raw.h b/lib/vnode_raw.h new file mode 100644 index 0000000..5c9927d --- /dev/null +++ b/lib/vnode_raw.h @@ -0,0 +1,9 @@ +/* + * Distributed under the Berkeley 4-part license + */ + + +extern struct af_vnode vnode_raw; /* vnode_raw.cpp */ + +int raw_freopen(AFFILE *af,FILE *f); +int raw_popen(AFFILE *af,const char *command,const char *type); diff --git a/lib/vnode_s3.cpp b/lib/vnode_s3.cpp new file mode 100644 index 0000000..b7bf23c --- /dev/null +++ b/lib/vnode_s3.cpp @@ -0,0 +1,329 @@ +/* + * vnode_aff.cpp: + * + * Functions for the manipulation of AFF files... + * + * Distributed under the Berkeley 4-part license + */ + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" +#include "vnode_s3.h" +#include "s3_glue.h" + +static int s3_close(AFFILE *af); + +/* Return 1 if a file is an S3 URL */ +static int s3_identify_file(const char *filename,int exists) +{ + if(strlen(filename)<5 || strncmp(filename,"s3://",5)!=0) return 0; // not a valid URL +#ifdef USE_S3 + if(exists==0) return 1; // don't need to check for existence; just return true + + /* See if it exists */ + AFFILE *af = af_open_with(filename,O_RDONLY,0,&vnode_s3); // can we open it with s3? + if(!af) return 0; // can't open it + s3_close(af); +#endif + return 1; // it's legit (or S3 support is not compiled in) +} + +#ifdef USE_S3 +#define S3_HEADER_ARG AMAZON_METADATA_PREFIX "arg" // XML metadata where the arg is stored + + +using namespace std; +using namespace s3; + +/**************************************************************** + *** Service routines + ****************************************************************/ + +class s3_private { +public: + s3_private():lbr(0) {} + ~s3_private(){ + if(lbr) delete lbr; + } + string bucket; + string path; // of the S3 root object + string current_seg; // the segment we are currently on + ListBucketResult *lbr; // if we have one +}; + +static inline struct s3_private *S3_PRIVATE(AFFILE *af) +{ + assert(af->v == &vnode_s3); + return (s3_private *)(af->vnodeprivate); +} + + +/**************************************************************** + *** User-visible functions. + ****************************************************************/ + +#include +static int s3_open(AFFILE *af) +{ + /* Set debug variable */ + if(getenv(S3_DEBUG)){ + s3_debug = atoi(getenv(S3_DEBUG)); +#ifdef HAVE_ERR_SET_EXIT + err_set_exit(s3_audit); +#endif + } + + /* Create the bucket if it doesn't exist */ + aws_access_key_id = getenv(AWS_ACCESS_KEY_ID); + aws_secret_access_key = getenv(AWS_SECRET_ACCESS_KEY); + + if(!aws_access_key_id) fprintf(stderr,"s3: AWS_ACCESS_KEY_ID not defined\n"); + if(!aws_secret_access_key) fprintf(stderr,"s3: AWS_SECRET_ACCESS_KEY not defined\n"); + if(!aws_access_key_id || !aws_secret_access_key) return -1; /* can't open */ + + /* URL host becomes bucket */ + char bucket[1024]; memset(bucket,0,sizeof(bucket)); + strcpy(bucket,af->hostname); + + if(strlen(bucket)==0){ + const char *b = getenv(S3_DEFAULT_BUCKET); + if(!b){ + fprintf(stderr,"s3: S3_DEFAULT_BUCKET not defined and no bucket in URL.\n"); + return -1; + } + strlcpy(bucket,b,sizeof(bucket)); + } + if(strlen(af->fname)==0){ + fprintf(stderr,"s3: No path specified in URL '%s'\n",af->fname); + return -1; + } + + af->vnodeprivate = (void *)new s3_private(); + struct s3_private *sp =S3_PRIVATE(af); + sp->bucket = bucket; + sp->path = string(af->fname) + "/"; + + /* If we are opening with O_CREAT and O_EXCL and the pagesize exists, then the + * file was already created. Return an error. + */ + bool exists = af_get_seg(af,AF_PAGESIZE,0,0,0)==0; + + if((af->openflags & O_CREAT) && (af->openflags & O_EXCL) && exists){ + errno = EEXIST; + return -1; + } + + /* If we are opening without O_CREAT and the pagesize does not exist, then the + * file was not created. Return an error. + */ + if((af->openflags & O_CREAT)==0 && !exists){ + errno = ENOENT; + return -1; + } + return 0; // "we were successful" +} + + +static int s3_close(AFFILE *af) +{ + struct s3_private *sp =S3_PRIVATE(af); + if(sp) delete sp; + return 0; +} + + +static int s3_vstat(AFFILE *af,struct af_vnode_info *vni) +{ + memset(vni,0,sizeof(*vni)); // clear it + + vni->has_pages = 1; + vni->supports_metadata = 1; + if(af->image_size==0) af_read_sizes(af); // wasn't set? + vni->imagesize = af->image_size; + return 0; +} + +static int s3_get_seg(AFFILE *af,const char *name,uint32_t *arg,unsigned char *data, + size_t *datalen) +{ + /* TK: Don't get the whole object if we just want the size or the argument. + * Use Content-Range: as documented at http://docs.amazonwebservices.com/AmazonS3/2006-03-01/ + **/ + + struct s3_private *sp =S3_PRIVATE(af); + sp->current_seg = name; + uint content_length = 0; + + response_buffer *r = 0; + + if(data) r = object_get(sp->bucket,sp->path + sp->current_seg,0); + else r = object_head(sp->bucket,sp->path + sp->current_seg,0); + + if(r==0) return -1; // no response was returned? + + if(r->result!=200){ // segment not found + delete r; + return -1; + } + + /* Check for metadata headers */ + if(arg) *arg=0; // default + for(map::const_iterator i = r->rheaders.begin(); + i != r->rheaders.end(); + i++){ + if( i->first == S3_HEADER_ARG && arg){ + *arg = atoi(i->second.c_str()); + continue; + } + if( i->first == S3_CONTENT_LENGTH){ + content_length = atoi(i->second.c_str()); + } + } + + if(datalen==0) { // no clue the size of the data... + delete r; + return 0; + } + if(*datalen==0){ + *datalen = data ? r->len : content_length; // use content_length if not getting data + delete r; + return 0; // datalen didn't have enough room + } + if(*datalen < r->len){ + delete r; + return -2; // datalen not being enough + } + if(data) memcpy(data,r->base,r->len); + *datalen = r->len; + delete r; + return 0; +} + + +static int s3_get_next_seg(AFFILE *af,char *segname,size_t segname_len,uint32_t *arg, + unsigned char *data,size_t *datalen) +{ + memset(segname,0,segname_len); + + struct s3_private *sp =S3_PRIVATE(af); + + if(sp->lbr && sp->lbr->contents.size()==0){ // this one is empty + delete sp->lbr; + sp->lbr = 0; + } + if(sp->lbr==0){ // need to get a new lbr.. + s3_result *r = list_bucket(sp->bucket,sp->path,sp->path + sp->current_seg,0); + if(r->lbr==0){delete r;return -1;} // hm... didn't get the right response? + sp->lbr = r->lbr; // grab the lbr + r->lbr = 0; // and we won't let it be freed here. + delete r; + } + + if(sp->lbr->contents.size()==0){ + delete sp->lbr; + sp->lbr = 0; + return -1; // nothing left + } + + sp->current_seg= sp->lbr->contents[0]->Key.substr(sp->path.size()); + + /* Set up the fields */ + memset(segname,0,segname_len); + if(segname_len > sp->current_seg.size()){ + strcpy(segname,sp->current_seg.c_str()); + } + if(datalen) *datalen = sp->lbr->contents[0]->Size; + + sp->lbr->contents.erase(sp->lbr->contents.begin()); // remove the first item + + /* If the user has asked for either the arg or the data, we need to get the object */ + if(arg || data) return s3_get_seg(af,segname,arg,data,datalen); + return 0; // otherwise, return success +} + + +/* Rewind all of the segments */ +static int s3_rewind_seg(AFFILE *af) +{ + struct s3_private *sp =S3_PRIVATE(af); + sp->current_seg = ""; + if(sp->lbr){ + delete sp->lbr; + sp->lbr = 0; + } + return 0; +} + + + +/* Update: + * S3 implementation ignores append + */ +static int s3_update_seg(AFFILE *af, const char *name, + uint32_t arg,const u_char *value,uint32_t vallen) + +{ + struct s3_private *sp =S3_PRIVATE(af); + char metabuf[64]; + snprintf(metabuf,sizeof(metabuf),"%lu",arg); // get the arg + struct s3headers meta[] = {{S3_HEADER_ARG,metabuf},{0,0}}; + + sp->current_seg = name; + if(vallen==0){ + value=(const u_char *)""; // point to a null string, so object_put knows to put + } + return object_put(sp->bucket,sp->path + sp->current_seg,(const char *)value,vallen,meta); +} + +int s3_del_seg(AFFILE *af,const char *segname) +{ + struct s3_private *sp =S3_PRIVATE(af); + sp->current_seg = segname; + return object_rm(sp->bucket,sp->path + sp->current_seg); +} + + +struct af_vnode vnode_s3 = { + AF_IDENTIFY_S3, // + AF_VNODE_TYPE_RELIABLE, // + "s3.amazonaws.com", + s3_identify_file, + s3_open, // open + s3_close, // close + s3_vstat, // vstat + s3_get_seg, // get_seg + s3_get_next_seg, // get_next_seg + s3_rewind_seg, // rewind_seg + s3_update_seg, // update_seg + s3_del_seg, // del_seg + 0, // read + 0 // write +}; +#else +static int s3_cantopen(AFFILE *af) +{ + err(1,"AFFLIB s3: Request to open %s, but S3 support is not compiled in.",af_filename(af)); + return -1; +} + +struct af_vnode vnode_s3 = { + AF_IDENTIFY_S3, // + AF_VNODE_TYPE_RELIABLE, // + "s3.amazonaws.com", + s3_identify_file, + s3_cantopen, // open + 0, // close + 0, // vstat + 0, // get_seg + 0, // get_next_seg + 0, // rewind_seg + 0, // update_seg + 0, // del_seg + 0, // read + 0 // write +}; + + +#endif // USE_S3 + diff --git a/lib/vnode_s3.h b/lib/vnode_s3.h new file mode 100644 index 0000000..4742e4b --- /dev/null +++ b/lib/vnode_s3.h @@ -0,0 +1,6 @@ +/* + * Distributed under the Berkeley 4-part license + */ + + +extern struct af_vnode vnode_s3; /* vnode_s3.cpp */ diff --git a/lib/vnode_split_raw.cpp b/lib/vnode_split_raw.cpp new file mode 100644 index 0000000..05d8d16 --- /dev/null +++ b/lib/vnode_split_raw.cpp @@ -0,0 +1,639 @@ +/* + * AFFLIB(tm) + * + * AFF and AFFLIB is a trademark of Simson Garfinkel and Basis Technology Corp. + * + * Distributed under the Berkeley 4-part license + */ +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" +#include "vnode_split_raw.h" + +#ifdef HAVE_CTYPE_H +#include +#endif + +#ifndef HAVE_ISDIGIT +static int isdigit(char ch) +{ + return ch>='0' && ch<='9'; +} +#endif + + + +/* split raw file implementation with optional metadata support */ +struct split_raw_private { + u_int num_raw_files; // number of raw files + int *fds; // array of file descriptors for each open raw file + uint64_t *pos; // where we are in each file + char *first_raw_fname; /* The filename of the first raw file. */ + char *next_raw_fname; /* The filename of the next raw file, or 0 + when one big file is used. */ + int64_t cur_page; // current page number, used for split_raw_get_next_seg +}; + +static inline struct split_raw_private *SPLIT_RAW_PRIVATE(AFFILE *af) +{ + assert(af->v == &vnode_split_raw); + return (struct split_raw_private *)(af->vnodeprivate); +} + +/* Return 1 if a file is the first of a split-raw series*/ +static int split_raw_identify_file(const char *filename,int exists) +{ + if(exists && access(filename,R_OK)!=0) return 0; // needs to exist and it doesn't + return af_ext_is(filename,"000") || af_ext_is(filename,"001") || + af_ext_is(filename,"aaa") || af_ext_is(filename,"AAA"); +} + +/* split_raw_close: + * Close each of the split files. + */ + +static int split_raw_close(AFFILE *af) +{ + struct split_raw_private *srp = SPLIT_RAW_PRIVATE(af); + + for (uint64_t i = 0; i < srp->num_raw_files; i++){ + if(srp->fds[i]){ + close(srp->fds[i]); + srp->fds[i] = 0; + } + } + if (srp->fds) free (srp->fds); + if (srp->pos) free (srp->pos); + if (srp->first_raw_fname) free (srp->first_raw_fname); + if (srp->next_raw_fname) free (srp->next_raw_fname); + free(srp); + af->vnodeprivate = 0; + return 0; +} + + +/** + * increment_fname(filename): + * "filename.000" => "filename.001" + * "filename.123" => "filename.124" + * "filename.999" => "filename.AAA" + * "filename.AZZ" => "filename.BAA" + * "filename.aaa" => "filename.aab" (legacy support) + * fn must be at least 4 characters long and must have a 3-character extension + * + * @param fn filename to increment (modified in place) + * @return 0 if successful. + * -1 for invalid filename or no more namespace. + */ +/** increase the character and return true if carry */ +static bool incval(char &ch,int base) +{ + if(base==10){ + if(ch=='9'){ + ch='0'; + return true; + } + ch++; + return false; + } + + /* Assume base 36 */ + switch(ch){ + case 'Z': + ch='0'; // go back to 0 + return true; // and carry + case '9': + ch='A'; + return false; + default: + ch++; // normal increment + return false; + } +} + +int split_raw_increment_fname (char *fn) +{ + size_t len = strlen(fn); + if(len<4 || fn[len-4]!='.') return -1; + char *ext = fn+len-3; + + /* See if it is a number */ + if(isdigit(ext[0]) && isdigit(ext[1]) && isdigit(ext[2])){ + int num = atoi(ext); + if(num==999){ + strcpy(ext,"A00"); + return 0; + } + snprintf(ext,4,"%03d",num+1); + return 0; + } + + /* First digit goes A-Z, second and third go 0-9A-Z */ + + /* Get the case */ + int lower = islower(ext[0]); + + /* Convert to all uppercase */ + for(int i=0;i<3;i++){ + if(isalpha(ext[i])) ext[i] = toupper(ext[i]); + } + + /* Increment */ + if(incval(ext[2],10)){ + if(incval(ext[1],36)){ + if(incval(ext[0],36)){ + return EINVAL; + } + } + } + + /* Convert back to lowercase if necessary */ + for(int i=0;i<3;i++){ + if(isalpha(ext[i]) && lower) ext[i] = tolower(ext[i]); + } + return 0; +} + + +void srp_validate(AFFILE *af) +{ + struct split_raw_private *srp = SPLIT_RAW_PRIVATE(af); + for(uint32_t i=0;inum_raw_files;i++){ + assert(srp->fds[i]!=0); + } +} + +/** Debugging routine. + */ +void srp_dump(AFFILE *af) +{ + struct split_raw_private *srp = SPLIT_RAW_PRIVATE(af); + for(uint32_t i=0;inum_raw_files;i++){ + fprintf(stderr," fds[%d]=%d pos[%d]=%"I64d"\n",i,srp->fds[i],i,srp->pos[i]); + } + srp_validate(af); + fprintf(stderr,"===================\n"); +} + +static void srp_add_fd(AFFILE *af,int fd) +{ + struct split_raw_private *srp = SPLIT_RAW_PRIVATE(af); + srp->num_raw_files++; + srp->fds = (int *)realloc (srp->fds, sizeof (int) * (srp->num_raw_files)); + srp->fds[srp->num_raw_files - 1] = fd; + srp->pos = (uint64_t *)realloc (srp->pos, sizeof (uint64_t) * (srp->num_raw_files)); + srp->pos[srp->num_raw_files - 1] = 0; +} + + +static int split_raw_open_internal(AFFILE *af, uint64_t *image_size) +{ + struct split_raw_private *srp = SPLIT_RAW_PRIVATE(af); + int fd; + struct stat sb; + + fd = open(srp->first_raw_fname, af->openflags|O_BINARY, af->openmode); + if (fd < 0) { + (*af->error_reporter)("split_raw_open_internal: open(%s): ",af->fname); + return -1; + } + + srp->num_raw_files = 1; + srp->fds = (int *)malloc (sizeof (int)); + srp->fds[0] = fd; + srp->pos = (uint64_t *)malloc (sizeof (uint64_t)); + if (fstat (fd, &sb) != 0) { + (*af->error_reporter)("split_raw_open_internal: fstat(%s): ",af->fname); + close (fd); + return -1; + } + + af->maxsize = 0; + + /* If there's a next_raw_fname set by the caller of this function, we + * have a split file; otherwise we have one big file. + */ + if (srp->next_raw_fname==0) { + (*image_size) = sb.st_size; + return 0; + } + + /* This gets set to 1 the first time we find a file whose size doesn't + match the size of the first file. If we successfully open a file + when this flag is already 1, then our sanity checks fail. */ + int current_file_must_be_last = 0; + + do { + if (split_raw_increment_fname (srp->next_raw_fname) != 0) { + (*af->error_reporter)("split_raw_open_internal: too many files\n"); + errno = EINVAL; + return -1; + } + fd = open(srp->next_raw_fname, + af->openflags & O_RDWR ? (O_RDWR|O_BINARY) : (O_RDONLY|O_BINARY)); + + if (fd < 0) { + if (errno != ENOENT) { + (af->error_reporter)("split_raw_open_internal errno=%d",errno); + return -1; + } + (*image_size) = sb.st_size + af->maxsize * (srp->num_raw_files - 1); + errno = 0; // reset errno + return 0; // end of files + } + srp_add_fd(af,fd); + if (current_file_must_be_last) { + (*af->error_reporter)("split_raw_open_internal: %s exists, " + "but previous file didn't match expected file size\n",af->fname); + return -1; + } + /* Set af->maxsize to the size of the first file, but only + if a second file exists. If no second file exists, then we want + to use af->maxsize, which cannot be set until after + af_open returns. */ + if (!af->maxsize) + af->maxsize = sb.st_size; + if (fstat (fd, &sb) != 0) { + (*af->error_reporter)("split_raw_open_internal: fstat(%s): ",af->fname); + return -1; + } + if ((uint64_t)sb.st_size != af->maxsize){ + current_file_must_be_last = 1; + } + } while (1); + return -1; +} + +static int split_raw_open(AFFILE *af) +{ + int ret; + + af->vnodeprivate = (void *)calloc(sizeof(struct split_raw_private),1); + struct split_raw_private *srp = SPLIT_RAW_PRIVATE(af); + + srp->first_raw_fname = strdup (af->fname); + srp->next_raw_fname = strdup (af->fname); + ret = split_raw_open_internal (af, &(af->image_size)); + + if (ret != 0) { + split_raw_close (af); + return ret; + } + + /* Adaptively find the largest pagesize we can use that fits within maxsize */ + af->image_pagesize = 512; + while ((af->image_pagesize < (16 * 1024 * 1024)) + && !(af->maxsize % (af->image_pagesize * 2))) + af->image_pagesize *= 2; + + if ((ret == 0) && (af->maxsize % af->image_pagesize!=0)) { + (*af->error_reporter)("split_raw_open: %s: raw_file_size (%"I64d" not a multiple of pagesize %lu\n", + af->fname, af->maxsize,af->image_pagesize); + split_raw_close (af); + return -1; + } + + return 0; +} + +static int split_raw_vstat(AFFILE *af,struct af_vnode_info *vni) +{ + memset(vni,0,sizeof(*vni)); // clear it + vni->imagesize = af->image_size; + vni->pagesize = af->image_pagesize; + vni->supports_compression = 0; + vni->supports_metadata = 0; + vni->is_raw = 1; + vni->changable_pagesize = 1; // change it at any time + vni->changable_sectorsize = 1; // change it at any time + return 0; +} + +static int split_raw_read(AFFILE *af, unsigned char *buf, uint64_t pos,size_t count) +{ + struct split_raw_private *srp = SPLIT_RAW_PRIVATE(af); + off_t c3; + int ret = 0; // how many bytes read + + if ((af->image_size - pos) < (unsigned)count){ + count = (off_t)(af->image_size - pos); + } + + while (count > 0) { + int filenum = -1; + off_t file_offset = 0; + + if (af->maxsize) { // if we do file segments + filenum = (int)(pos / af->maxsize); + file_offset = (off_t)(pos % af->maxsize); + } else { + filenum = 0; + file_offset = (off_t)pos; + } + if (file_offset != (off_t) srp->pos[filenum]) { + off_t c2 = lseek (srp->fds[filenum], file_offset, SEEK_SET); + if (file_offset != c2) { // seek failed; return work to date + if (ret) return ret; // some bytes were read; return that + else return -1; // no bytes read; return error + } + srp->pos[filenum] = c2; // this file starts here + } + if (af->maxsize && ((af->maxsize - file_offset) < (unsigned) count)) + c3 = (off_t)(af->maxsize - file_offset); + else + c3 = count; + off_t c4 = read (srp->fds[filenum], buf, c3); + if (c4 <= 0) { // got an error + if (ret) return ret; // return how many bytes we read + else return -1; // otherwise, return -1 + } + buf += c4; + count -= c4; + ret += c4; + pos += c4; + srp->pos[filenum] += c4; // position of this file pointer + if (c3 != c4) return ret; // incomplete? + } + return ret; +} + +/* + * split_raw_write_internal2: + * If buf==0, assume we are writing zeros to the end of the file, + * and just seek to the last character and write a single NUL. + */ + +int split_raw_write_internal2(AFFILE *af, unsigned char *buf, uint64_t pos,size_t count) +{ + struct split_raw_private *srp = SPLIT_RAW_PRIVATE(af); + off_t c1, c3; + int i; + int ret = 0; + struct affcallback_info acbi; + + /* Setup the callback structure */ + memset(&acbi,0,sizeof(acbi)); + acbi.info_version = 1; + acbi.af = af->parent ? af->parent : af; + acbi.pagenum = af->image_pagesize ? pos / af->image_pagesize : 0; + acbi.bytes_to_write = count; + + while (count > 0) { + if (af->maxsize) { // do we need to possibly split into multiple file writes? + /* Figure out which file number we will need to write to... */ + if (pos >= (af->maxsize * srp->num_raw_files)) { + int fd = open(srp->next_raw_fname, O_RDWR | O_CREAT | O_EXCL | O_BINARY, af->openmode); + if (fd < 0) { + (*af->error_reporter)("split_raw_write: open(%s): ",af->fname); + if (ret) return ret; + else return -1; + } + srp_add_fd(af,fd); + if (split_raw_increment_fname (srp->next_raw_fname) != 0) { + (*af->error_reporter)("split_raw_write: too many files\n"); + if (ret) + return ret; + else + return -1; + } + } + i = (int)(pos / af->maxsize); + c1 = (off_t)(pos % af->maxsize); + } else { + i = 0; + c1 = (off_t)pos; + } + if (c1 != (off_t)srp->pos[i]) { // do we need to seek this file? + off_t c2 = lseek (srp->fds[i], c1, SEEK_SET); // try to seek + if (c1 != c2) { // hm. Ended up in the wrong place. That's an error + if (ret>0) { // return how many bytes we got + return ret; + } + else { + return -1; + } + } + srp->pos[i] = c2; + } + if (af->maxsize && ((af->maxsize - c1) < (unsigned)count)) + c3 = (off_t)(af->maxsize - c1); + else + c3 = count; + if(af->w_callback) {acbi.phase = 3;(*af->w_callback)(&acbi);} + + /* WRITE THE DATA! */ + off_t c4 = 0; + + if(buf){ + c4 = write (srp->fds[i], buf, c3); + } + else { + /* Extend with lseek() and write a single byte */ + char z = 0; + + lseek(srp->fds[i],c3-1,SEEK_CUR); + if(write(srp->fds[i],&z,1)!=1) return -1; // failure + c4 = c3; + } + + /* DONE! */ + + acbi.bytes_written = c4; + if(af->w_callback) {acbi.phase = 4;(*af->w_callback)(&acbi);} + if (c4 <= 0) { // some error writing? + if (ret) + return ret; + else + return -1; + } + buf += c4; + count -= c4; + ret += c4; + pos += c4; + srp->pos[i] += c4; + if (af->image_size < pos) af->image_size = pos; // image was extended + if (c3 != c4){ // amount written doesn't equal request; return + return ret; + } + } + return ret; +} + +int split_raw_write(AFFILE *af, unsigned char *buf, uint64_t pos,size_t count) +{ + /* If we are being asked to start writing beyond the end of the file + * pad out the file (and possibly create one or more new image files.) + */ + + if (af->maxsize) { + if (pos > af->image_size) { // writing beyond the end... + while(pos > af->image_size){ + + /* repeat until file is as big as where we should be writing */ + int64_t bytes_left = pos - af->image_size; + int bytes_to_write = (int)(af->maxsize - (af->image_size % af->maxsize)); + if(bytes_to_write > bytes_left) bytes_to_write = (int)bytes_left; + int bytes_written = split_raw_write_internal2(af,0,af->image_size,bytes_to_write); + if(bytes_to_write != bytes_written){ + return -1; // some kind of internal error + } + } + } + } + + return split_raw_write_internal2 (af, buf, pos,count); +} + + + +/* Get a segment; if a data page is being asked for, then fake it. + * Otherwise, return an error. + */ + +static int split_raw_get_seg(AFFILE *af,const char *name,uint32_t *arg,unsigned char *data, + size_t *datalen) +{ + int64_t page_num = af_segname_page_number(name); + if(page_num<0){ + /* See if PAGESIZE or IMAGESIZE is being requested; we can fake those */ + if(strcmp(name,AF_PAGESIZE)==0){ + if(arg) *arg = af->image_pagesize; + if(datalen) *datalen = 0; + return 0; + } + if(strcmp(name,AF_IMAGESIZE)==0){ + struct aff_quad q; + if(data && *datalen>=8){ + q.low = htonl((uint32_t)(af->image_size & 0xffffffff)); + q.high = htonl((uint32_t)(af->image_size >> 32)); + memcpy(data,&q,8); + *datalen = 8; + } + return 0; + } + if(strcmp(name,AF_SECTORSIZE)==0){ + if(arg) *arg = af->image_sectorsize; + if(datalen) *datalen = 0; + return 0; + } + if(strcmp(name,AF_DEVICE_SECTORS)==0){ + int64_t devicesectors = af->image_size / af->image_sectorsize; + struct aff_quad q; + if(data && *datalen>=8){ + q.low = htonl((uint32_t)(devicesectors & 0xffffffff)); + q.high = htonl((uint32_t)(devicesectors >> 32)); + memcpy(data,&q,8); + *datalen = 8; + } + return 0; + } + errno = ENOTSUP; // sorry! We don't store metadata + return -1; + } + + uint64_t pos = page_num * af->image_pagesize; // where we are to start reading + uint64_t bytes_left = af->image_size - pos; // how many bytes left in the file + + uint32_t bytes_to_read = af->image_pagesize; // copy this many bytes, unless + if(bytes_to_read > bytes_left) bytes_to_read = (uint32_t)bytes_left; // only this much is left + + if(arg) *arg = 0; // arg is always 0 + if(datalen){ + if(data==0){ // asked for 0 bytes, so give the actual size + *datalen = bytes_to_read; + return 0; + } + if(*datalen < (unsigned)bytes_to_read){ + *datalen = bytes_to_read; + return AF_ERROR_DATASMALL; + } + } + if(data){ + int bytes_read = split_raw_read(af,data,pos,bytes_to_read); + if(bytes_read>=0){ + if(datalen) *datalen = bytes_read; + return 0; + } + return -1; // some kind of EOF? + } + return 0; // no problems! +} + +/* + * split_raw_get_next_seg: + * Try get_next_seg on the AFF file first. If that fails, + * create the next virtual segment + */ + +static int split_raw_get_next_seg(AFFILE *af,char *segname,size_t segname_len,uint32_t *arg, + unsigned char *data,size_t *datalen_) +{ + struct split_raw_private *srp = SPLIT_RAW_PRIVATE(af); + + int64_t total_pages = (af->image_size + af->image_pagesize - 1) / af->image_pagesize; + if(srp->cur_page >= total_pages) return -1; // that's all there are + + /* Make the segment name */ + char pagename[AF_MAX_NAME_LEN]; + memset(pagename,0,sizeof(pagename)); + snprintf(pagename,sizeof(pagename),AF_PAGE,srp->cur_page++); + + /* Get the segment, if we can */ + int r = split_raw_get_seg(af,pagename,arg,data,datalen_); + + /* If r==0 and there is room for copying in the segment name, return it */ + if(r==0){ + if(strlen(pagename)+1 < segname_len){ + strcpy(segname,pagename); + return 0; + } + /* segname wasn't big enough */ + return -2; + } + return r; // some other error +} + + +/* Rewind all of the segments */ +static int split_raw_rewind_seg(AFFILE *af) +{ + struct split_raw_private *srp = SPLIT_RAW_PRIVATE(af); + srp->cur_page = 0; + return 0; +} + +static int split_raw_update_seg(AFFILE *af, const char *name, + uint32_t arg,const u_char *value,uint32_t vallen) + +{ + int64_t page_num = af_segname_page_number(name); + if(page_num<0){ + errno = ENOTSUP; // sorry! We don't store metadata + return -1; + } + + uint64_t pos = page_num * af->image_pagesize; // where we are to start reading + int written = split_raw_write(af, (unsigned char *)value, pos,vallen); + if(written==(int)vallen) return 0; // success + return -1; +} + + +struct af_vnode vnode_split_raw = { + AF_IDENTIFY_SPLIT_RAW, + AF_VNODE_TYPE_COMPOUND|AF_VNODE_TYPE_RELIABLE|AF_VNODE_MAXSIZE_MULTIPLE|AF_VNODE_NO_SIGNING|AF_VNODE_NO_SEALING, + "Split Raw", + split_raw_identify_file, + split_raw_open, + split_raw_close, + split_raw_vstat, + split_raw_get_seg, // get seg + split_raw_get_next_seg, // get_next_seg + split_raw_rewind_seg, // rewind_seg + split_raw_update_seg, // update_seg + 0, // del_seg + split_raw_read, // read + split_raw_write // write +}; + + diff --git a/lib/vnode_split_raw.h b/lib/vnode_split_raw.h new file mode 100644 index 0000000..49376bd --- /dev/null +++ b/lib/vnode_split_raw.h @@ -0,0 +1,10 @@ +/* + * Distributed under the Berkeley 4-part license + */ + + +#define SPLITRAW_DEFAULT_EXTENSION "000" + +extern struct af_vnode vnode_split_raw; + + diff --git a/lzma443/7zC.txt b/lzma443/7zC.txt new file mode 100644 index 0000000..88f039c --- /dev/null +++ b/lzma443/7zC.txt @@ -0,0 +1,235 @@ +7z ANSI-C Decoder 4.43 +---------------------- + +7z ANSI-C Decoder 4.43 Copyright (C) 1999-2006 Igor Pavlov + +7z ANSI-C provides 7z/LZMA decoding. +7z ANSI-C version is simplified version ported from C++ code. + +LZMA is default and general compression method of 7z format +in 7-Zip compression program (www.7-zip.org). LZMA provides high +compression ratio and very fast decompression. + + +LICENSE +------- + +Read lzma.txt for information about license. + + +Files +--------------------- + +7zAlloc.* - Allocate and Free +7zBuffer.* - Buffer structure +7zCrc.* - CRC32 code +7zDecode.* - Low level memory->memory decoding +7zExtract.* - High level stream->memory decoding +7zHeader.* - .7z format constants +7zIn.* - .7z archive opening +7zItem.* - .7z structures +7zMain.c - Test application +7zMethodID.* - MethodID structure +7zTypes.h - Base types and constants + + +How To Use +---------- + +You must download 7-Zip program from www.7-zip.org. + +You can create .7z archive with 7z.exe or 7za.exe: + + 7za.exe a archive.7z *.htm -r -mx -m0fb=255 -mf=off + +If you have big number of files in archive, and you need fast extracting, +you can use partly-solid archives: + + 7za.exe a archive.7z *.htm -ms=512K -r -mx -m0fb=255 -m0d=512K -mf=off + +In that example 7-Zip will use 512KB solid blocks. So it needs to decompress only +512KB for extracting one file from such archive. + + +Limitations of current version of 7z ANSI-C Decoder +--------------------------------------------------- + + - It reads only "FileName", "Size", and "CRC" information for each file in archive. + - It supports only LZMA and Copy (no compression) methods. + - It converts original UTF-16 Unicode file names to UTF-8 Unicode file names. + +These limitations will be fixed in future versions. + + +Using 7z ANSI-C Decoder Test application: +----------------------------------------- + +Usage: 7zDec + +: + e: Extract files from archive + l: List contents of archive + t: Test integrity of archive + +Example: + + 7zDec l archive.7z + +lists contents of archive.7z + + 7zDec e archive.7z + +extracts files from archive.7z to current folder. + + +How to use .7z Decoder +---------------------- + +.7z Decoder can be compiled in one of two modes: + +1) Default mode. In that mode 7z Decoder will read full compressed + block to RAM before decompressing. + +2) Mode with defined _LZMA_IN_CB. In that mode 7z Decoder can read + compressed block by parts. And you can specify desired buffer size. + So memory requirements can be reduced. But decompressing speed will + be 5-10% lower and code size is slightly larger. + + +Memory allocation +~~~~~~~~~~~~~~~~~ + +7z Decoder uses two memory pools: +1) Temporary pool +2) Main pool +Such scheme can allow you to avoid fragmentation of allocated blocks. + +Steps for using 7z decoder +-------------------------- + +Use code at 7zMain.c as example. + +1) Declare variables: + inStream /* implements ISzInStream interface */ + CArchiveDatabaseEx db; /* 7z archive database structure */ + ISzAlloc allocImp; /* memory functions for main pool */ + ISzAlloc allocTempImp; /* memory functions for temporary pool */ + +2) call InitCrcTable(); function to initialize CRC structures. + +3) call SzArDbExInit(&db); function to initialize db structures. + +4) call SzArchiveOpen(inStream, &db, &allocMain, &allocTemp) to open archive + +This function opens archive "inStream" and reads headers to "db". +All items in "db" will be allocated with "allocMain" functions. +SzArchiveOpen function allocates and frees temporary structures by "allocTemp" functions. + +5) List items or Extract items + + Listing code: + ~~~~~~~~~~~~~ + { + UInt32 i; + for (i = 0; i < db.Database.NumFiles; i++) + { + CFileItem *f = db.Database.Files + i; + printf("%10d %s\n", (int)f->Size, f->Name); + } + } + + Extracting code: + ~~~~~~~~~~~~~~~~ + + SZ_RESULT SzExtract( + ISzInStream *inStream, + CArchiveDatabaseEx *db, + UInt32 fileIndex, /* index of file */ + UInt32 *blockIndex, /* index of solid block */ + Byte **outBuffer, /* pointer to pointer to output buffer (allocated with allocMain) */ + size_t *outBufferSize, /* buffer size for output buffer */ + size_t *offset, /* offset of stream for required file in *outBuffer */ + size_t *outSizeProcessed, /* size of file in *outBuffer */ + ISzAlloc *allocMain, + ISzAlloc *allocTemp); + + If you need to decompress more than one file, you can send these values from previous call: + blockIndex, + outBuffer, + outBufferSize, + You can consider "outBuffer" as cache of solid block. If your archive is solid, + it will increase decompression speed. + + After decompressing you must free "outBuffer": + allocImp.Free(outBuffer); + +6) call SzArDbExFree(&db, allocImp.Free) to free allocated items in "db". + + + + +Memory requirements for .7z decoding +------------------------------------ + +Memory usage for Archive opening: + - Temporary pool: + - Memory for compressed .7z headers (if _LZMA_IN_CB is not defined) + - Memory for uncompressed .7z headers + - some other temporary blocks + - Main pool: + - Memory for database: + Estimated size of one file structures in solid archive: + - Size (4 or 8 Bytes) + - CRC32 (4 bytes) + - Some file information (4 bytes) + - File Name (variable length) + pointer + allocation structures + +Memory usage for archive Decompressing: + - Temporary pool: + - Memory for compressed solid block (if _LZMA_IN_CB is not defined) + - Memory for LZMA decompressing structures + - Main pool: + - Memory for decompressed solid block + + +If _LZMA_IN_CB is defined, 7z Decoder will not allocate memory for +compressed blocks. Instead of this, you must allocate buffer with desired +size before calling 7z Decoder. Use 7zMain.c as example. + + + +EXIT codes +----------- + +7z Decoder functions can return one of the following codes: + +#define SZ_OK (0) +#define SZE_DATA_ERROR (1) +#define SZE_OUTOFMEMORY (2) +#define SZE_CRC_ERROR (3) + +#define SZE_NOTIMPL (4) +#define SZE_FAIL (5) + +#define SZE_ARCHIVE_ERROR (6) + + + +LZMA Defines +------------ + +_LZMA_IN_CB - Use special callback mode for input stream to reduce memory requirements + +_SZ_FILE_SIZE_64 - define it if you need support for files larger than 4 GB +_SZ_NO_INT_64 - define it if your compiler doesn't support long long int + +_LZMA_PROB32 - it can increase LZMA decompressing speed on some 32-bit CPUs. + +_SZ_ONE_DIRECTORY - define it if you want to locate all source files to one directory +_SZ_ALLOC_DEBUG - define it if you want to debug alloc/free operations to stderr. + + +--- + +http://www.7-zip.org +http://www.7-zip.org/support.html diff --git a/lzma443/7zFormat.txt b/lzma443/7zFormat.txt new file mode 100644 index 0000000..56ff817 --- /dev/null +++ b/lzma443/7zFormat.txt @@ -0,0 +1,471 @@ +7z Format description (2.30 Beta 25) +----------------------------------- + +This file contains description of 7z archive format. +7z archive can contain files compressed with any method. +See "Methods.txt" for description for defined compressing methods. + + +Format structure Overview +------------------------- + +Some fields can be optional. + +Archive structure +~~~~~~~~~~~~~~~~~ +SignatureHeader +[PackedStreams] +[PackedStreamsForHeaders] +[ + Header + or + { + Packed Header + HeaderInfo + } +] + + + +Header structure +~~~~~~~~~~~~~~~~ +{ + ArchiveProperties + AdditionalStreams + { + PackInfo + { + PackPos + NumPackStreams + Sizes[NumPackStreams] + CRCs[NumPackStreams] + } + CodersInfo + { + NumFolders + Folders[NumFolders] + { + NumCoders + CodersInfo[NumCoders] + { + ID + NumInStreams; + NumOutStreams; + PropertiesSize + Properties[PropertiesSize] + } + NumBindPairs + BindPairsInfo[NumBindPairs] + { + InIndex; + OutIndex; + } + PackedIndices + } + UnPackSize[Folders][Folders.NumOutstreams] + CRCs[NumFolders] + } + SubStreamsInfo + { + NumUnPackStreamsInFolders[NumFolders]; + UnPackSizes[] + CRCs[] + } + } + MainStreamsInfo + { + (Same as in AdditionalStreams) + } + FilesInfo + { + NumFiles + Properties[] + { + ID + Size + Data + } + } +} + +HeaderInfo structure +~~~~~~~~~~~~~~~~~~~~ +{ + (Same as in AdditionalStreams) +} + + + +Notes about Notation and encoding +--------------------------------- + +7z uses little endian encoding. + +7z archive format has optional headers that are marked as +[] +Header +[] + +REAL_UINT64 means real UINT64. + +UINT64 means real UINT64 encoded with the following scheme: + + Size of encoding sequence depends from first byte: + First_Byte Extra_Bytes Value + (binary) + 0xxxxxxx : ( xxxxxxx ) + 10xxxxxx BYTE y[1] : ( xxxxxx << (8 * 1)) + y + 110xxxxx BYTE y[2] : ( xxxxx << (8 * 2)) + y + ... + 1111110x BYTE y[6] : ( x << (8 * 6)) + y + 11111110 BYTE y[7] : y + 11111111 BYTE y[8] : y + + + +Property IDs +------------ + +0x00 = kEnd, + +0x01 = kHeader, + +0x02 = kArchiveProperties, + +0x03 = kAdditionalStreamsInfo, +0x04 = kMainStreamsInfo, +0x05 = kFilesInfo, + +0x06 = kPackInfo, +0x07 = kUnPackInfo, +0x08 = kSubStreamsInfo, + +0x09 = kSize, +0x0A = kCRC, + +0x0B = kFolder, + +0x0C = kCodersUnPackSize, +0x0D = kNumUnPackStream, + +0x0E = kEmptyStream, +0x0F = kEmptyFile, +0x10 = kAnti, + +0x11 = kName, +0x12 = kCreationTime, +0x13 = kLastAccessTime, +0x14 = kLastWriteTime, +0x15 = kWinAttributes, +0x16 = kComment, + +0x17 = kEncodedHeader, + + +7z format headers +----------------- + +SignatureHeader +~~~~~~~~~~~~~~~ + BYTE kSignature[6] = {'7', 'z', 0xBC, 0xAF, 0x27, 0x1C}; + + ArchiveVersion + { + BYTE Major; // now = 0 + BYTE Minor; // now = 2 + }; + + UINT32 StartHeaderCRC; + + StartHeader + { + REAL_UINT64 NextHeaderOffset + REAL_UINT64 NextHeaderSize + UINT32 NextHeaderCRC + } + + +........................... + + +ArchiveProperties +~~~~~~~~~~~~~~~~~ +BYTE NID::kArchiveProperties (0x02) +while(true) +{ + BYTE PropertyType; + if (aType == 0) + break; + UINT64 PropertySize; + BYTE PropertyData[PropertySize]; +} + + +Digests (NumStreams) +~~~~~~~~~~~~~~~~~~~~~ + BYTE AllAreDefined + if (AllAreDefined == 0) + { + for(NumStreams) + BIT Defined + } + UINT32 CRCs[NumDefined] + + +PackInfo +~~~~~~~~~~~~ + BYTE NID::kPackInfo (0x06) + UINT64 PackPos + UINT64 NumPackStreams + + [] + BYTE NID::kSize (0x09) + UINT64 PackSizes[NumPackStreams] + [] + + [] + BYTE NID::kCRC (0x0A) + PackStreamDigests[NumPackStreams] + [] + + BYTE NID::kEnd + + +Folder +~~~~~~ + UINT64 NumCoders; + for (NumCoders) + { + BYTE + { + 0:3 DecompressionMethod.IDSize + 4: + 0 - IsSimple + 1 - Is not simple + 5: + 0 - No Attributes + 1 - There Are Attributes + 7: + 0 - Last Method in Alternative_Method_List + 1 - There are more alternative methods + } + BYTE DecompressionMethod.ID[DecompressionMethod.IDSize] + if (!IsSimple) + { + UINT64 NumInStreams; + UINT64 NumOutStreams; + } + if (DecompressionMethod[0] != 0) + { + UINT64 PropertiesSize + BYTE Properties[PropertiesSize] + } + } + + NumBindPairs = NumOutStreamsTotal - 1; + + for (NumBindPairs) + { + UINT64 InIndex; + UINT64 OutIndex; + } + + NumPackedStreams = NumInStreamsTotal - NumBindPairs; + if (NumPackedStreams > 1) + for(NumPackedStreams) + { + UINT64 Index; + }; + + + + +Coders Info +~~~~~~~~~~~ + + BYTE NID::kUnPackInfo (0x07) + + + BYTE NID::kFolder (0x0B) + UINT64 NumFolders + BYTE External + switch(External) + { + case 0: + Folders[NumFolders] + case 1: + UINT64 DataStreamIndex + } + + + BYTE ID::kCodersUnPackSize (0x0C) + for(Folders) + for(Folder.NumOutStreams) + UINT64 UnPackSize; + + + [] + BYTE NID::kCRC (0x0A) + UnPackDigests[NumFolders] + [] + + + + BYTE NID::kEnd + + + +SubStreams Info +~~~~~~~~~~~~~~ + BYTE NID::kSubStreamsInfo; (0x08) + + [] + BYTE NID::kNumUnPackStream; (0x0D) + UINT64 NumUnPackStreamsInFolders[NumFolders]; + [] + + + [] + BYTE NID::kSize (0x09) + UINT64 UnPackSizes[] + [] + + + [] + BYTE NID::kCRC (0x0A) + Digests[Number of streams with unknown CRC] + [] + + + BYTE NID::kEnd + + +Streams Info +~~~~~~~~~~~~ + + [] + PackInfo + [] + + + [] + CodersInfo + [] + + + [] + SubStreamsInfo + [] + + BYTE NID::kEnd + + +FilesInfo +~~~~~~~~~ + BYTE NID::kFilesInfo; (0x05) + UINT64 NumFiles + + while(true) + { + BYTE PropertyType; + if (aType == 0) + break; + + UINT64 Size; + + switch(PropertyType) + { + kEmptyStream: (0x0E) + for(NumFiles) + BIT IsEmptyStream + + kEmptyFile: (0x0F) + for(EmptyStreams) + BIT IsEmptyFile + + kAnti: (0x10) + for(EmptyStreams) + BIT IsAntiFile + + case kCreationTime: (0x12) + case kLastAccessTime: (0x13) + case kLastWriteTime: (0x14) + BYTE AllAreDefined + if (AllAreDefined == 0) + { + for(NumFiles) + BIT TimeDefined + } + BYTE External; + if(External != 0) + UINT64 DataIndex + [] + for(Definded Items) + UINT32 Time + [] + + kNames: (0x11) + BYTE External; + if(External != 0) + UINT64 DataIndex + [] + for(Files) + { + wchar_t Names[NameSize]; + wchar_t 0; + } + [] + + kAttributes: (0x15) + BYTE AllAreDefined + if (AllAreDefined == 0) + { + for(NumFiles) + BIT AttributesAreDefined + } + BYTE External; + if(External != 0) + UINT64 DataIndex + [] + for(Definded Attributes) + UINT32 Attributes + [] + } + } + + +Header +~~~~~~ + BYTE NID::kHeader (0x01) + + [] + ArchiveProperties + [] + + [] + BYTE NID::kAdditionalStreamsInfo; (0x03) + StreamsInfo + [] + + [] + BYTE NID::kMainStreamsInfo; (0x04) + StreamsInfo + [] + + [] + FilesInfo + [] + + BYTE NID::kEnd + + +HeaderInfo +~~~~~~~~~~ + [] + BYTE NID::kEncodedHeader; (0x17) + StreamsInfo for Encoded Header + [] + + +--- +End of document diff --git a/lzma443/C/7zip/Archive/7z_C/7zAlloc.c b/lzma443/C/7zip/Archive/7z_C/7zAlloc.c new file mode 100755 index 0000000..21bb30c --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/7zAlloc.c @@ -0,0 +1,70 @@ +/* 7zAlloc.c */ + +#include +#include "7zAlloc.h" + +/* #define _SZ_ALLOC_DEBUG */ +/* use _SZ_ALLOC_DEBUG to debug alloc/free operations */ + +#ifdef _SZ_ALLOC_DEBUG + +#ifdef _WIN32 +#include +#endif +#include +int g_allocCount = 0; +int g_allocCountTemp = 0; +#endif + +void *SzAlloc(size_t size) +{ + if (size == 0) + return 0; + #ifdef _SZ_ALLOC_DEBUG + fprintf(stderr, "\nAlloc %10d bytes; count = %10d", size, g_allocCount); + g_allocCount++; + #endif + return malloc(size); +} + +void SzFree(void *address) +{ + #ifdef _SZ_ALLOC_DEBUG + if (address != 0) + { + g_allocCount--; + fprintf(stderr, "\nFree; count = %10d", g_allocCount); + } + #endif + free(address); +} + +void *SzAllocTemp(size_t size) +{ + if (size == 0) + return 0; + #ifdef _SZ_ALLOC_DEBUG + fprintf(stderr, "\nAlloc_temp %10d bytes; count = %10d", size, g_allocCountTemp); + g_allocCountTemp++; + #ifdef _WIN32 + return HeapAlloc(GetProcessHeap(), 0, size); + #endif + #endif + return malloc(size); +} + +void SzFreeTemp(void *address) +{ + #ifdef _SZ_ALLOC_DEBUG + if (address != 0) + { + g_allocCountTemp--; + fprintf(stderr, "\nFree_temp; count = %10d", g_allocCountTemp); + } + #ifdef _WIN32 + HeapFree(GetProcessHeap(), 0, address); + return; + #endif + #endif + free(address); +} diff --git a/lzma443/C/7zip/Archive/7z_C/7zAlloc.h b/lzma443/C/7zip/Archive/7z_C/7zAlloc.h new file mode 100755 index 0000000..4ca4170 --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/7zAlloc.h @@ -0,0 +1,20 @@ +/* 7zAlloc.h */ + +#ifndef __7Z_ALLOC_H +#define __7Z_ALLOC_H + +#include + +typedef struct _ISzAlloc +{ + void *(*Alloc)(size_t size); + void (*Free)(void *address); /* address can be 0 */ +} ISzAlloc; + +void *SzAlloc(size_t size); +void SzFree(void *address); + +void *SzAllocTemp(size_t size); +void SzFreeTemp(void *address); + +#endif diff --git a/lzma443/C/7zip/Archive/7z_C/7zBuffer.c b/lzma443/C/7zip/Archive/7z_C/7zBuffer.c new file mode 100755 index 0000000..3c4b71e --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/7zBuffer.c @@ -0,0 +1,29 @@ +/* 7zBuffer.c */ + +#include "7zBuffer.h" +#include "7zAlloc.h" + +void SzByteBufferInit(CSzByteBuffer *buffer) +{ + buffer->Capacity = 0; + buffer->Items = 0; +} + +int SzByteBufferCreate(CSzByteBuffer *buffer, size_t newCapacity, void * (*allocFunc)(size_t size)) +{ + buffer->Capacity = newCapacity; + if (newCapacity == 0) + { + buffer->Items = 0; + return 1; + } + buffer->Items = (Byte *)allocFunc(newCapacity); + return (buffer->Items != 0); +} + +void SzByteBufferFree(CSzByteBuffer *buffer, void (*freeFunc)(void *)) +{ + freeFunc(buffer->Items); + buffer->Items = 0; + buffer->Capacity = 0; +} diff --git a/lzma443/C/7zip/Archive/7z_C/7zBuffer.h b/lzma443/C/7zip/Archive/7z_C/7zBuffer.h new file mode 100755 index 0000000..17e5906 --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/7zBuffer.h @@ -0,0 +1,19 @@ +/* 7zBuffer.h */ + +#ifndef __7Z_BUFFER_H +#define __7Z_BUFFER_H + +#include +#include "7zTypes.h" + +typedef struct _CSzByteBuffer +{ + size_t Capacity; + Byte *Items; +}CSzByteBuffer; + +void SzByteBufferInit(CSzByteBuffer *buffer); +int SzByteBufferCreate(CSzByteBuffer *buffer, size_t newCapacity, void * (*allocFunc)(size_t size)); +void SzByteBufferFree(CSzByteBuffer *buffer, void (*freeFunc)(void *)); + +#endif diff --git a/lzma443/C/7zip/Archive/7z_C/7zCrc.c b/lzma443/C/7zip/Archive/7z_C/7zCrc.c new file mode 100755 index 0000000..9773840 --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/7zCrc.c @@ -0,0 +1,76 @@ +/* 7zCrc.c */ + +#include "7zCrc.h" + +#define kCrcPoly 0xEDB88320 + +UInt32 g_CrcTable[256]; + +void InitCrcTable() +{ + UInt32 i; + for (i = 0; i < 256; i++) + { + UInt32 r = i; + int j; + for (j = 0; j < 8; j++) + if (r & 1) + r = (r >> 1) ^ kCrcPoly; + else + r >>= 1; + g_CrcTable[i] = r; + } +} + +void CrcInit(UInt32 *crc) { *crc = 0xFFFFFFFF; } +UInt32 CrcGetDigest(UInt32 *crc) { return *crc ^ 0xFFFFFFFF; } + +void CrcUpdateByte(UInt32 *crc, Byte b) +{ + *crc = g_CrcTable[((Byte)(*crc)) ^ b] ^ (*crc >> 8); +} + +void CrcUpdateUInt16(UInt32 *crc, UInt16 v) +{ + CrcUpdateByte(crc, (Byte)v); + CrcUpdateByte(crc, (Byte)(v >> 8)); +} + +void CrcUpdateUInt32(UInt32 *crc, UInt32 v) +{ + int i; + for (i = 0; i < 4; i++) + CrcUpdateByte(crc, (Byte)(v >> (8 * i))); +} + +void CrcUpdateUInt64(UInt32 *crc, UInt64 v) +{ + int i; + for (i = 0; i < 8; i++) + { + CrcUpdateByte(crc, (Byte)(v)); + v >>= 8; + } +} + +void CrcUpdate(UInt32 *crc, const void *data, size_t size) +{ + UInt32 v = *crc; + const Byte *p = (const Byte *)data; + for (; size > 0 ; size--, p++) + v = g_CrcTable[((Byte)(v)) ^ *p] ^ (v >> 8); + *crc = v; +} + +UInt32 CrcCalculateDigest(const void *data, size_t size) +{ + UInt32 crc; + CrcInit(&crc); + CrcUpdate(&crc, data, size); + return CrcGetDigest(&crc); +} + +int CrcVerifyDigest(UInt32 digest, const void *data, size_t size) +{ + return (CrcCalculateDigest(data, size) == digest); +} diff --git a/lzma443/C/7zip/Archive/7z_C/7zCrc.h b/lzma443/C/7zip/Archive/7z_C/7zCrc.h new file mode 100755 index 0000000..adcc563 --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/7zCrc.h @@ -0,0 +1,24 @@ +/* 7zCrc.h */ + +#ifndef __7Z_CRC_H +#define __7Z_CRC_H + +#include + +#include "7zTypes.h" + +extern UInt32 g_CrcTable[256]; +void InitCrcTable(); + +void CrcInit(UInt32 *crc); +UInt32 CrcGetDigest(UInt32 *crc); +void CrcUpdateByte(UInt32 *crc, Byte v); +void CrcUpdateUInt16(UInt32 *crc, UInt16 v); +void CrcUpdateUInt32(UInt32 *crc, UInt32 v); +void CrcUpdateUInt64(UInt32 *crc, UInt64 v); +void CrcUpdate(UInt32 *crc, const void *data, size_t size); + +UInt32 CrcCalculateDigest(const void *data, size_t size); +int CrcVerifyDigest(UInt32 digest, const void *data, size_t size); + +#endif diff --git a/lzma443/C/7zip/Archive/7z_C/7zDecode.c b/lzma443/C/7zip/Archive/7z_C/7zDecode.c new file mode 100755 index 0000000..b42ff92 --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/7zDecode.c @@ -0,0 +1,150 @@ +/* 7zDecode.c */ + +#include "7zDecode.h" +#ifdef _SZ_ONE_DIRECTORY +#include "LzmaDecode.h" +#else +#include "../../Compress/LZMA_C/LzmaDecode.h" +#endif + +CMethodID k_Copy = { { 0x0 }, 1 }; +CMethodID k_LZMA = { { 0x3, 0x1, 0x1 }, 3 }; + +#ifdef _LZMA_IN_CB + +typedef struct _CLzmaInCallbackImp +{ + ILzmaInCallback InCallback; + ISzInStream *InStream; + size_t Size; +} CLzmaInCallbackImp; + +int LzmaReadImp(void *object, const unsigned char **buffer, SizeT *size) +{ + CLzmaInCallbackImp *cb = (CLzmaInCallbackImp *)object; + size_t processedSize; + SZ_RESULT res; + *size = 0; + res = cb->InStream->Read((void *)cb->InStream, (void **)buffer, cb->Size, &processedSize); + *size = (SizeT)processedSize; + if (processedSize > cb->Size) + return (int)SZE_FAIL; + cb->Size -= processedSize; + if (res == SZ_OK) + return 0; + return (int)res; +} + +#endif + +SZ_RESULT SzDecode(const CFileSize *packSizes, const CFolder *folder, + #ifdef _LZMA_IN_CB + ISzInStream *inStream, + #else + const Byte *inBuffer, + #endif + Byte *outBuffer, size_t outSize, + size_t *outSizeProcessed, ISzAlloc *allocMain) +{ + UInt32 si; + size_t inSize = 0; + CCoderInfo *coder; + if (folder->NumPackStreams != 1) + return SZE_NOTIMPL; + if (folder->NumCoders != 1) + return SZE_NOTIMPL; + coder = folder->Coders; + *outSizeProcessed = 0; + + for (si = 0; si < folder->NumPackStreams; si++) + inSize += (size_t)packSizes[si]; + + if (AreMethodsEqual(&coder->MethodID, &k_Copy)) + { + size_t i; + if (inSize != outSize) + return SZE_DATA_ERROR; + #ifdef _LZMA_IN_CB + for (i = 0; i < inSize;) + { + size_t j; + Byte *inBuffer; + size_t bufferSize; + RINOK(inStream->Read((void *)inStream, (void **)&inBuffer, inSize - i, &bufferSize)); + if (bufferSize == 0) + return SZE_DATA_ERROR; + if (bufferSize > inSize - i) + return SZE_FAIL; + *outSizeProcessed += bufferSize; + for (j = 0; j < bufferSize && i < inSize; j++, i++) + outBuffer[i] = inBuffer[j]; + } + #else + for (i = 0; i < inSize; i++) + outBuffer[i] = inBuffer[i]; + *outSizeProcessed = inSize; + #endif + return SZ_OK; + } + + if (AreMethodsEqual(&coder->MethodID, &k_LZMA)) + { + #ifdef _LZMA_IN_CB + CLzmaInCallbackImp lzmaCallback; + #else + SizeT inProcessed; + #endif + + CLzmaDecoderState state; /* it's about 24-80 bytes structure, if int is 32-bit */ + int result; + SizeT outSizeProcessedLoc; + + #ifdef _LZMA_IN_CB + lzmaCallback.Size = inSize; + lzmaCallback.InStream = inStream; + lzmaCallback.InCallback.Read = LzmaReadImp; + #endif + + if (LzmaDecodeProperties(&state.Properties, coder->Properties.Items, + coder->Properties.Capacity) != LZMA_RESULT_OK) + return SZE_FAIL; + + state.Probs = (CProb *)allocMain->Alloc(LzmaGetNumProbs(&state.Properties) * sizeof(CProb)); + if (state.Probs == 0) + return SZE_OUTOFMEMORY; + + #ifdef _LZMA_OUT_READ + if (state.Properties.DictionarySize == 0) + state.Dictionary = 0; + else + { + state.Dictionary = (unsigned char *)allocMain->Alloc(state.Properties.DictionarySize); + if (state.Dictionary == 0) + { + allocMain->Free(state.Probs); + return SZE_OUTOFMEMORY; + } + } + LzmaDecoderInit(&state); + #endif + + result = LzmaDecode(&state, + #ifdef _LZMA_IN_CB + &lzmaCallback.InCallback, + #else + inBuffer, (SizeT)inSize, &inProcessed, + #endif + outBuffer, (SizeT)outSize, &outSizeProcessedLoc); + *outSizeProcessed = (size_t)outSizeProcessedLoc; + allocMain->Free(state.Probs); + #ifdef _LZMA_OUT_READ + allocMain->Free(state.Dictionary); + #endif + if (result == LZMA_RESULT_DATA_ERROR) + return SZE_DATA_ERROR; + if (result != LZMA_RESULT_OK) + return SZE_FAIL; + return SZ_OK; + } + return SZE_NOTIMPL; +} diff --git a/lzma443/C/7zip/Archive/7z_C/7zDecode.h b/lzma443/C/7zip/Archive/7z_C/7zDecode.h new file mode 100755 index 0000000..74bb180 --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/7zDecode.h @@ -0,0 +1,21 @@ +/* 7zDecode.h */ + +#ifndef __7Z_DECODE_H +#define __7Z_DECODE_H + +#include "7zItem.h" +#include "7zAlloc.h" +#ifdef _LZMA_IN_CB +#include "7zIn.h" +#endif + +SZ_RESULT SzDecode(const CFileSize *packSizes, const CFolder *folder, + #ifdef _LZMA_IN_CB + ISzInStream *stream, + #else + const Byte *inBuffer, + #endif + Byte *outBuffer, size_t outSize, + size_t *outSizeProcessed, ISzAlloc *allocMain); + +#endif diff --git a/lzma443/C/7zip/Archive/7z_C/7zExtract.c b/lzma443/C/7zip/Archive/7z_C/7zExtract.c new file mode 100755 index 0000000..6ef872c --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/7zExtract.c @@ -0,0 +1,116 @@ +/* 7zExtract.c */ + +#include "7zExtract.h" +#include "7zDecode.h" +#include "7zCrc.h" + +SZ_RESULT SzExtract( + ISzInStream *inStream, + CArchiveDatabaseEx *db, + UInt32 fileIndex, + UInt32 *blockIndex, + Byte **outBuffer, + size_t *outBufferSize, + size_t *offset, + size_t *outSizeProcessed, + ISzAlloc *allocMain, + ISzAlloc *allocTemp) +{ + UInt32 folderIndex = db->FileIndexToFolderIndexMap[fileIndex]; + SZ_RESULT res = SZ_OK; + *offset = 0; + *outSizeProcessed = 0; + if (folderIndex == (UInt32)-1) + { + allocMain->Free(*outBuffer); + *blockIndex = folderIndex; + *outBuffer = 0; + *outBufferSize = 0; + return SZ_OK; + } + + if (*outBuffer == 0 || *blockIndex != folderIndex) + { + CFolder *folder = db->Database.Folders + folderIndex; + CFileSize unPackSize = SzFolderGetUnPackSize(folder); + #ifndef _LZMA_IN_CB + CFileSize packSize = SzArDbGetFolderFullPackSize(db, folderIndex); + Byte *inBuffer = 0; + size_t processedSize; + #endif + *blockIndex = folderIndex; + allocMain->Free(*outBuffer); + *outBuffer = 0; + + RINOK(inStream->Seek(inStream, SzArDbGetFolderStreamPos(db, folderIndex, 0))); + + #ifndef _LZMA_IN_CB + if (packSize != 0) + { + inBuffer = (Byte *)allocTemp->Alloc((size_t)packSize); + if (inBuffer == 0) + return SZE_OUTOFMEMORY; + } + res = inStream->Read(inStream, inBuffer, (size_t)packSize, &processedSize); + if (res == SZ_OK && processedSize != (size_t)packSize) + res = SZE_FAIL; + #endif + if (res == SZ_OK) + { + *outBufferSize = (size_t)unPackSize; + if (unPackSize != 0) + { + *outBuffer = (Byte *)allocMain->Alloc((size_t)unPackSize); + if (*outBuffer == 0) + res = SZE_OUTOFMEMORY; + } + if (res == SZ_OK) + { + size_t outRealSize; + res = SzDecode(db->Database.PackSizes + + db->FolderStartPackStreamIndex[folderIndex], folder, + #ifdef _LZMA_IN_CB + inStream, + #else + inBuffer, + #endif + *outBuffer, (size_t)unPackSize, &outRealSize, allocTemp); + if (res == SZ_OK) + { + if (outRealSize == (size_t)unPackSize) + { + if (folder->UnPackCRCDefined) + { + if (!CrcVerifyDigest(folder->UnPackCRC, *outBuffer, (size_t)unPackSize)) + res = SZE_FAIL; + } + } + else + res = SZE_FAIL; + } + } + } + #ifndef _LZMA_IN_CB + allocTemp->Free(inBuffer); + #endif + } + if (res == SZ_OK) + { + UInt32 i; + CFileItem *fileItem = db->Database.Files + fileIndex; + *offset = 0; + for(i = db->FolderStartFileIndex[folderIndex]; i < fileIndex; i++) + *offset += (UInt32)db->Database.Files[i].Size; + *outSizeProcessed = (size_t)fileItem->Size; + if (*offset + *outSizeProcessed > *outBufferSize) + return SZE_FAIL; + { + if (fileItem->IsFileCRCDefined) + { + if (!CrcVerifyDigest(fileItem->FileCRC, *outBuffer + *offset, *outSizeProcessed)) + res = SZE_FAIL; + } + } + } + return res; +} diff --git a/lzma443/C/7zip/Archive/7z_C/7zExtract.h b/lzma443/C/7zip/Archive/7z_C/7zExtract.h new file mode 100755 index 0000000..e9a4fb4 --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/7zExtract.h @@ -0,0 +1,40 @@ +/* 7zExtract.h */ + +#ifndef __7Z_EXTRACT_H +#define __7Z_EXTRACT_H + +#include "7zIn.h" + +/* + SzExtract extracts file from archive + + *outBuffer must be 0 before first call for each new archive. + + Extracting cache: + If you need to decompress more than one file, you can send + these values from previous call: + *blockIndex, + *outBuffer, + *outBufferSize + You can consider "*outBuffer" as cache of solid block. If your archive is solid, + it will increase decompression speed. + + If you use external function, you can declare these 3 cache variables + (blockIndex, outBuffer, outBufferSize) as static in that external function. + + Free *outBuffer and set *outBuffer to 0, if you want to flush cache. +*/ + +SZ_RESULT SzExtract( + ISzInStream *inStream, + CArchiveDatabaseEx *db, + UInt32 fileIndex, /* index of file */ + UInt32 *blockIndex, /* index of solid block */ + Byte **outBuffer, /* pointer to pointer to output buffer (allocated with allocMain) */ + size_t *outBufferSize, /* buffer size for output buffer */ + size_t *offset, /* offset of stream for required file in *outBuffer */ + size_t *outSizeProcessed, /* size of file in *outBuffer */ + ISzAlloc *allocMain, + ISzAlloc *allocTemp); + +#endif diff --git a/lzma443/C/7zip/Archive/7z_C/7zHeader.c b/lzma443/C/7zip/Archive/7z_C/7zHeader.c new file mode 100755 index 0000000..3be4bc2 --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/7zHeader.c @@ -0,0 +1,5 @@ +/* 7zHeader.c */ + +#include "7zHeader.h" + +Byte k7zSignature[k7zSignatureSize] = {'7', 'z', 0xBC, 0xAF, 0x27, 0x1C}; diff --git a/lzma443/C/7zip/Archive/7z_C/7zHeader.h b/lzma443/C/7zip/Archive/7z_C/7zHeader.h new file mode 100755 index 0000000..0356aaa --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/7zHeader.h @@ -0,0 +1,55 @@ +/* 7zHeader.h */ + +#ifndef __7Z_HEADER_H +#define __7Z_HEADER_H + +#include "7zTypes.h" + +#define k7zSignatureSize 6 +extern Byte k7zSignature[k7zSignatureSize]; + +#define k7zMajorVersion 0 + +#define k7zStartHeaderSize 0x20 + +enum EIdEnum +{ + k7zIdEnd, + + k7zIdHeader, + + k7zIdArchiveProperties, + + k7zIdAdditionalStreamsInfo, + k7zIdMainStreamsInfo, + k7zIdFilesInfo, + + k7zIdPackInfo, + k7zIdUnPackInfo, + k7zIdSubStreamsInfo, + + k7zIdSize, + k7zIdCRC, + + k7zIdFolder, + + k7zIdCodersUnPackSize, + k7zIdNumUnPackStream, + + k7zIdEmptyStream, + k7zIdEmptyFile, + k7zIdAnti, + + k7zIdName, + k7zIdCreationTime, + k7zIdLastAccessTime, + k7zIdLastWriteTime, + k7zIdWinAttributes, + k7zIdComment, + + k7zIdEncodedHeader, + + k7zIdStartPos +}; + +#endif diff --git a/lzma443/C/7zip/Archive/7z_C/7zIn.c b/lzma443/C/7zip/Archive/7z_C/7zIn.c new file mode 100755 index 0000000..8ff1e62 --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/7zIn.c @@ -0,0 +1,1281 @@ +/* 7zIn.c */ + +#include "7zIn.h" +#include "7zCrc.h" +#include "7zDecode.h" + +#define RINOM(x) { if((x) == 0) return SZE_OUTOFMEMORY; } + +void SzArDbExInit(CArchiveDatabaseEx *db) +{ + SzArchiveDatabaseInit(&db->Database); + db->FolderStartPackStreamIndex = 0; + db->PackStreamStartPositions = 0; + db->FolderStartFileIndex = 0; + db->FileIndexToFolderIndexMap = 0; +} + +void SzArDbExFree(CArchiveDatabaseEx *db, void (*freeFunc)(void *)) +{ + freeFunc(db->FolderStartPackStreamIndex); + freeFunc(db->PackStreamStartPositions); + freeFunc(db->FolderStartFileIndex); + freeFunc(db->FileIndexToFolderIndexMap); + SzArchiveDatabaseFree(&db->Database, freeFunc); + SzArDbExInit(db); +} + +/* +CFileSize GetFolderPackStreamSize(int folderIndex, int streamIndex) const +{ + return PackSizes[FolderStartPackStreamIndex[folderIndex] + streamIndex]; +} + +CFileSize GetFilePackSize(int fileIndex) const +{ + int folderIndex = FileIndexToFolderIndexMap[fileIndex]; + if (folderIndex >= 0) + { + const CFolder &folderInfo = Folders[folderIndex]; + if (FolderStartFileIndex[folderIndex] == fileIndex) + return GetFolderFullPackSize(folderIndex); + } + return 0; +} +*/ + +#define MY_ALLOC(T, p, size, allocFunc) { if ((size) == 0) p = 0; else \ + if ((p = (T *)allocFunc((size) * sizeof(T))) == 0) return SZE_OUTOFMEMORY; } + +SZ_RESULT SzArDbExFill(CArchiveDatabaseEx *db, void * (*allocFunc)(size_t size)) +{ + UInt32 startPos = 0; + CFileSize startPosSize = 0; + UInt32 i; + UInt32 folderIndex = 0; + UInt32 indexInFolder = 0; + MY_ALLOC(UInt32, db->FolderStartPackStreamIndex, db->Database.NumFolders, allocFunc); + for(i = 0; i < db->Database.NumFolders; i++) + { + db->FolderStartPackStreamIndex[i] = startPos; + startPos += db->Database.Folders[i].NumPackStreams; + } + + MY_ALLOC(CFileSize, db->PackStreamStartPositions, db->Database.NumPackStreams, allocFunc); + + for(i = 0; i < db->Database.NumPackStreams; i++) + { + db->PackStreamStartPositions[i] = startPosSize; + startPosSize += db->Database.PackSizes[i]; + } + + MY_ALLOC(UInt32, db->FolderStartFileIndex, db->Database.NumFolders, allocFunc); + MY_ALLOC(UInt32, db->FileIndexToFolderIndexMap, db->Database.NumFiles, allocFunc); + + for (i = 0; i < db->Database.NumFiles; i++) + { + CFileItem *file = db->Database.Files + i; + int emptyStream = !file->HasStream; + if (emptyStream && indexInFolder == 0) + { + db->FileIndexToFolderIndexMap[i] = (UInt32)-1; + continue; + } + if (indexInFolder == 0) + { + /* + v3.13 incorrectly worked with empty folders + v4.07: Loop for skipping empty folders + */ + while(1) + { + if (folderIndex >= db->Database.NumFolders) + return SZE_ARCHIVE_ERROR; + db->FolderStartFileIndex[folderIndex] = i; + if (db->Database.Folders[folderIndex].NumUnPackStreams != 0) + break; + folderIndex++; + } + } + db->FileIndexToFolderIndexMap[i] = folderIndex; + if (emptyStream) + continue; + indexInFolder++; + if (indexInFolder >= db->Database.Folders[folderIndex].NumUnPackStreams) + { + folderIndex++; + indexInFolder = 0; + } + } + return SZ_OK; +} + + +CFileSize SzArDbGetFolderStreamPos(CArchiveDatabaseEx *db, UInt32 folderIndex, UInt32 indexInFolder) +{ + return db->ArchiveInfo.DataStartPosition + + db->PackStreamStartPositions[db->FolderStartPackStreamIndex[folderIndex] + indexInFolder]; +} + +CFileSize SzArDbGetFolderFullPackSize(CArchiveDatabaseEx *db, UInt32 folderIndex) +{ + UInt32 packStreamIndex = db->FolderStartPackStreamIndex[folderIndex]; + CFolder *folder = db->Database.Folders + folderIndex; + CFileSize size = 0; + UInt32 i; + for (i = 0; i < folder->NumPackStreams; i++) + size += db->Database.PackSizes[packStreamIndex + i]; + return size; +} + + +/* +SZ_RESULT SzReadTime(const CObjectVector &dataVector, + CObjectVector &files, UInt64 type) +{ + CBoolVector boolVector; + RINOK(ReadBoolVector2(files.Size(), boolVector)) + + CStreamSwitch streamSwitch; + RINOK(streamSwitch.Set(this, &dataVector)); + + for(int i = 0; i < files.Size(); i++) + { + CFileItem &file = files[i]; + CArchiveFileTime fileTime; + bool defined = boolVector[i]; + if (defined) + { + UInt32 low, high; + RINOK(SzReadUInt32(low)); + RINOK(SzReadUInt32(high)); + fileTime.dwLowDateTime = low; + fileTime.dwHighDateTime = high; + } + switch(type) + { + case k7zIdCreationTime: + file.IsCreationTimeDefined = defined; + if (defined) + file.CreationTime = fileTime; + break; + case k7zIdLastWriteTime: + file.IsLastWriteTimeDefined = defined; + if (defined) + file.LastWriteTime = fileTime; + break; + case k7zIdLastAccessTime: + file.IsLastAccessTimeDefined = defined; + if (defined) + file.LastAccessTime = fileTime; + break; + } + } + return SZ_OK; +} +*/ + +SZ_RESULT SafeReadDirect(ISzInStream *inStream, Byte *data, size_t size) +{ + #ifdef _LZMA_IN_CB + while (size > 0) + { + Byte *inBuffer; + size_t processedSize; + RINOK(inStream->Read(inStream, (void **)&inBuffer, size, &processedSize)); + if (processedSize == 0 || processedSize > size) + return SZE_FAIL; + size -= processedSize; + do + { + *data++ = *inBuffer++; + } + while (--processedSize != 0); + } + #else + size_t processedSize; + RINOK(inStream->Read(inStream, data, size, &processedSize)); + if (processedSize != size) + return SZE_FAIL; + #endif + return SZ_OK; +} + +SZ_RESULT SafeReadDirectByte(ISzInStream *inStream, Byte *data) +{ + return SafeReadDirect(inStream, data, 1); +} + +SZ_RESULT SafeReadDirectUInt32(ISzInStream *inStream, UInt32 *value) +{ + int i; + *value = 0; + for (i = 0; i < 4; i++) + { + Byte b; + RINOK(SafeReadDirectByte(inStream, &b)); + *value |= ((UInt32)b << (8 * i)); + } + return SZ_OK; +} + +SZ_RESULT SafeReadDirectUInt64(ISzInStream *inStream, UInt64 *value) +{ + int i; + *value = 0; + for (i = 0; i < 8; i++) + { + Byte b; + RINOK(SafeReadDirectByte(inStream, &b)); + *value |= ((UInt32)b << (8 * i)); + } + return SZ_OK; +} + +int TestSignatureCandidate(Byte *testBytes) +{ + size_t i; + for (i = 0; i < k7zSignatureSize; i++) + if (testBytes[i] != k7zSignature[i]) + return 0; + return 1; +} + +typedef struct _CSzState +{ + Byte *Data; + size_t Size; +}CSzData; + +SZ_RESULT SzReadByte(CSzData *sd, Byte *b) +{ + if (sd->Size == 0) + return SZE_ARCHIVE_ERROR; + sd->Size--; + *b = *sd->Data++; + return SZ_OK; +} + +SZ_RESULT SzReadBytes(CSzData *sd, Byte *data, size_t size) +{ + size_t i; + for (i = 0; i < size; i++) + { + RINOK(SzReadByte(sd, data + i)); + } + return SZ_OK; +} + +SZ_RESULT SzReadUInt32(CSzData *sd, UInt32 *value) +{ + int i; + *value = 0; + for (i = 0; i < 4; i++) + { + Byte b; + RINOK(SzReadByte(sd, &b)); + *value |= ((UInt32)(b) << (8 * i)); + } + return SZ_OK; +} + +SZ_RESULT SzReadNumber(CSzData *sd, UInt64 *value) +{ + Byte firstByte; + Byte mask = 0x80; + int i; + RINOK(SzReadByte(sd, &firstByte)); + *value = 0; + for (i = 0; i < 8; i++) + { + Byte b; + if ((firstByte & mask) == 0) + { + UInt64 highPart = firstByte & (mask - 1); + *value += (highPart << (8 * i)); + return SZ_OK; + } + RINOK(SzReadByte(sd, &b)); + *value |= ((UInt64)b << (8 * i)); + mask >>= 1; + } + return SZ_OK; +} + +SZ_RESULT SzReadSize(CSzData *sd, CFileSize *value) +{ + UInt64 value64; + RINOK(SzReadNumber(sd, &value64)); + *value = (CFileSize)value64; + return SZ_OK; +} + +SZ_RESULT SzReadNumber32(CSzData *sd, UInt32 *value) +{ + UInt64 value64; + RINOK(SzReadNumber(sd, &value64)); + if (value64 >= 0x80000000) + return SZE_NOTIMPL; + if (value64 >= ((UInt64)(1) << ((sizeof(size_t) - 1) * 8 + 2))) + return SZE_NOTIMPL; + *value = (UInt32)value64; + return SZ_OK; +} + +SZ_RESULT SzReadID(CSzData *sd, UInt64 *value) +{ + return SzReadNumber(sd, value); +} + +SZ_RESULT SzSkeepDataSize(CSzData *sd, UInt64 size) +{ + if (size > sd->Size) + return SZE_ARCHIVE_ERROR; + sd->Size -= (size_t)size; + sd->Data += (size_t)size; + return SZ_OK; +} + +SZ_RESULT SzSkeepData(CSzData *sd) +{ + UInt64 size; + RINOK(SzReadNumber(sd, &size)); + return SzSkeepDataSize(sd, size); +} + +SZ_RESULT SzReadArchiveProperties(CSzData *sd) +{ + while(1) + { + UInt64 type; + RINOK(SzReadID(sd, &type)); + if (type == k7zIdEnd) + break; + SzSkeepData(sd); + } + return SZ_OK; +} + +SZ_RESULT SzWaitAttribute(CSzData *sd, UInt64 attribute) +{ + while(1) + { + UInt64 type; + RINOK(SzReadID(sd, &type)); + if (type == attribute) + return SZ_OK; + if (type == k7zIdEnd) + return SZE_ARCHIVE_ERROR; + RINOK(SzSkeepData(sd)); + } +} + +SZ_RESULT SzReadBoolVector(CSzData *sd, size_t numItems, Byte **v, void * (*allocFunc)(size_t size)) +{ + Byte b = 0; + Byte mask = 0; + size_t i; + MY_ALLOC(Byte, *v, numItems, allocFunc); + for(i = 0; i < numItems; i++) + { + if (mask == 0) + { + RINOK(SzReadByte(sd, &b)); + mask = 0x80; + } + (*v)[i] = (Byte)(((b & mask) != 0) ? 1 : 0); + mask >>= 1; + } + return SZ_OK; +} + +SZ_RESULT SzReadBoolVector2(CSzData *sd, size_t numItems, Byte **v, void * (*allocFunc)(size_t size)) +{ + Byte allAreDefined; + size_t i; + RINOK(SzReadByte(sd, &allAreDefined)); + if (allAreDefined == 0) + return SzReadBoolVector(sd, numItems, v, allocFunc); + MY_ALLOC(Byte, *v, numItems, allocFunc); + for(i = 0; i < numItems; i++) + (*v)[i] = 1; + return SZ_OK; +} + +SZ_RESULT SzReadHashDigests( + CSzData *sd, + size_t numItems, + Byte **digestsDefined, + UInt32 **digests, + void * (*allocFunc)(size_t size)) +{ + size_t i; + RINOK(SzReadBoolVector2(sd, numItems, digestsDefined, allocFunc)); + MY_ALLOC(UInt32, *digests, numItems, allocFunc); + for(i = 0; i < numItems; i++) + if ((*digestsDefined)[i]) + { + RINOK(SzReadUInt32(sd, (*digests) + i)); + } + return SZ_OK; +} + +SZ_RESULT SzReadPackInfo( + CSzData *sd, + CFileSize *dataOffset, + UInt32 *numPackStreams, + CFileSize **packSizes, + Byte **packCRCsDefined, + UInt32 **packCRCs, + void * (*allocFunc)(size_t size)) +{ + UInt32 i; + RINOK(SzReadSize(sd, dataOffset)); + RINOK(SzReadNumber32(sd, numPackStreams)); + + RINOK(SzWaitAttribute(sd, k7zIdSize)); + + MY_ALLOC(CFileSize, *packSizes, (size_t)*numPackStreams, allocFunc); + + for(i = 0; i < *numPackStreams; i++) + { + RINOK(SzReadSize(sd, (*packSizes) + i)); + } + + while(1) + { + UInt64 type; + RINOK(SzReadID(sd, &type)); + if (type == k7zIdEnd) + break; + if (type == k7zIdCRC) + { + RINOK(SzReadHashDigests(sd, (size_t)*numPackStreams, packCRCsDefined, packCRCs, allocFunc)); + continue; + } + RINOK(SzSkeepData(sd)); + } + if (*packCRCsDefined == 0) + { + MY_ALLOC(Byte, *packCRCsDefined, (size_t)*numPackStreams, allocFunc); + MY_ALLOC(UInt32, *packCRCs, (size_t)*numPackStreams, allocFunc); + for(i = 0; i < *numPackStreams; i++) + { + (*packCRCsDefined)[i] = 0; + (*packCRCs)[i] = 0; + } + } + return SZ_OK; +} + +SZ_RESULT SzReadSwitch(CSzData *sd) +{ + Byte external; + RINOK(SzReadByte(sd, &external)); + return (external == 0) ? SZ_OK: SZE_ARCHIVE_ERROR; +} + +SZ_RESULT SzGetNextFolderItem(CSzData *sd, CFolder *folder, void * (*allocFunc)(size_t size)) +{ + UInt32 numCoders; + UInt32 numBindPairs; + UInt32 numPackedStreams; + UInt32 i; + UInt32 numInStreams = 0; + UInt32 numOutStreams = 0; + RINOK(SzReadNumber32(sd, &numCoders)); + folder->NumCoders = numCoders; + + MY_ALLOC(CCoderInfo, folder->Coders, (size_t)numCoders, allocFunc); + + for (i = 0; i < numCoders; i++) + SzCoderInfoInit(folder->Coders + i); + + for (i = 0; i < numCoders; i++) + { + Byte mainByte; + CCoderInfo *coder = folder->Coders + i; + { + RINOK(SzReadByte(sd, &mainByte)); + coder->MethodID.IDSize = (Byte)(mainByte & 0xF); + RINOK(SzReadBytes(sd, coder->MethodID.ID, coder->MethodID.IDSize)); + if ((mainByte & 0x10) != 0) + { + RINOK(SzReadNumber32(sd, &coder->NumInStreams)); + RINOK(SzReadNumber32(sd, &coder->NumOutStreams)); + } + else + { + coder->NumInStreams = 1; + coder->NumOutStreams = 1; + } + if ((mainByte & 0x20) != 0) + { + UInt64 propertiesSize = 0; + RINOK(SzReadNumber(sd, &propertiesSize)); + if (!SzByteBufferCreate(&coder->Properties, (size_t)propertiesSize, allocFunc)) + return SZE_OUTOFMEMORY; + RINOK(SzReadBytes(sd, coder->Properties.Items, (size_t)propertiesSize)); + } + } + while ((mainByte & 0x80) != 0) + { + RINOK(SzReadByte(sd, &mainByte)); + RINOK(SzSkeepDataSize(sd, (mainByte & 0xF))); + if ((mainByte & 0x10) != 0) + { + UInt32 n; + RINOK(SzReadNumber32(sd, &n)); + RINOK(SzReadNumber32(sd, &n)); + } + if ((mainByte & 0x20) != 0) + { + UInt64 propertiesSize = 0; + RINOK(SzReadNumber(sd, &propertiesSize)); + RINOK(SzSkeepDataSize(sd, propertiesSize)); + } + } + numInStreams += (UInt32)coder->NumInStreams; + numOutStreams += (UInt32)coder->NumOutStreams; + } + + numBindPairs = numOutStreams - 1; + folder->NumBindPairs = numBindPairs; + + + MY_ALLOC(CBindPair, folder->BindPairs, (size_t)numBindPairs, allocFunc); + + for (i = 0; i < numBindPairs; i++) + { + CBindPair *bindPair = folder->BindPairs + i;; + RINOK(SzReadNumber32(sd, &bindPair->InIndex)); + RINOK(SzReadNumber32(sd, &bindPair->OutIndex)); + } + + numPackedStreams = numInStreams - (UInt32)numBindPairs; + + folder->NumPackStreams = numPackedStreams; + MY_ALLOC(UInt32, folder->PackStreams, (size_t)numPackedStreams, allocFunc); + + if (numPackedStreams == 1) + { + UInt32 j; + UInt32 pi = 0; + for (j = 0; j < numInStreams; j++) + if (SzFolderFindBindPairForInStream(folder, j) < 0) + { + folder->PackStreams[pi++] = j; + break; + } + } + else + for(i = 0; i < numPackedStreams; i++) + { + RINOK(SzReadNumber32(sd, folder->PackStreams + i)); + } + return SZ_OK; +} + +SZ_RESULT SzReadUnPackInfo( + CSzData *sd, + UInt32 *numFolders, + CFolder **folders, /* for allocFunc */ + void * (*allocFunc)(size_t size), + ISzAlloc *allocTemp) +{ + UInt32 i; + RINOK(SzWaitAttribute(sd, k7zIdFolder)); + RINOK(SzReadNumber32(sd, numFolders)); + { + RINOK(SzReadSwitch(sd)); + + MY_ALLOC(CFolder, *folders, (size_t)*numFolders, allocFunc); + + for(i = 0; i < *numFolders; i++) + SzFolderInit((*folders) + i); + + for(i = 0; i < *numFolders; i++) + { + RINOK(SzGetNextFolderItem(sd, (*folders) + i, allocFunc)); + } + } + + RINOK(SzWaitAttribute(sd, k7zIdCodersUnPackSize)); + + for(i = 0; i < *numFolders; i++) + { + UInt32 j; + CFolder *folder = (*folders) + i; + UInt32 numOutStreams = SzFolderGetNumOutStreams(folder); + + MY_ALLOC(CFileSize, folder->UnPackSizes, (size_t)numOutStreams, allocFunc); + + for(j = 0; j < numOutStreams; j++) + { + RINOK(SzReadSize(sd, folder->UnPackSizes + j)); + } + } + + while(1) + { + UInt64 type; + RINOK(SzReadID(sd, &type)); + if (type == k7zIdEnd) + return SZ_OK; + if (type == k7zIdCRC) + { + SZ_RESULT res; + Byte *crcsDefined = 0; + UInt32 *crcs = 0; + res = SzReadHashDigests(sd, *numFolders, &crcsDefined, &crcs, allocTemp->Alloc); + if (res == SZ_OK) + { + for(i = 0; i < *numFolders; i++) + { + CFolder *folder = (*folders) + i; + folder->UnPackCRCDefined = crcsDefined[i]; + folder->UnPackCRC = crcs[i]; + } + } + allocTemp->Free(crcs); + allocTemp->Free(crcsDefined); + RINOK(res); + continue; + } + RINOK(SzSkeepData(sd)); + } +} + +SZ_RESULT SzReadSubStreamsInfo( + CSzData *sd, + UInt32 numFolders, + CFolder *folders, + UInt32 *numUnPackStreams, + CFileSize **unPackSizes, + Byte **digestsDefined, + UInt32 **digests, + ISzAlloc *allocTemp) +{ + UInt64 type = 0; + UInt32 i; + UInt32 si = 0; + UInt32 numDigests = 0; + + for(i = 0; i < numFolders; i++) + folders[i].NumUnPackStreams = 1; + *numUnPackStreams = numFolders; + + while(1) + { + RINOK(SzReadID(sd, &type)); + if (type == k7zIdNumUnPackStream) + { + *numUnPackStreams = 0; + for(i = 0; i < numFolders; i++) + { + UInt32 numStreams; + RINOK(SzReadNumber32(sd, &numStreams)); + folders[i].NumUnPackStreams = numStreams; + *numUnPackStreams += numStreams; + } + continue; + } + if (type == k7zIdCRC || type == k7zIdSize) + break; + if (type == k7zIdEnd) + break; + RINOK(SzSkeepData(sd)); + } + + if (*numUnPackStreams == 0) + { + *unPackSizes = 0; + *digestsDefined = 0; + *digests = 0; + } + else + { + *unPackSizes = (CFileSize *)allocTemp->Alloc((size_t)*numUnPackStreams * sizeof(CFileSize)); + RINOM(*unPackSizes); + *digestsDefined = (Byte *)allocTemp->Alloc((size_t)*numUnPackStreams * sizeof(Byte)); + RINOM(*digestsDefined); + *digests = (UInt32 *)allocTemp->Alloc((size_t)*numUnPackStreams * sizeof(UInt32)); + RINOM(*digests); + } + + for(i = 0; i < numFolders; i++) + { + /* + v3.13 incorrectly worked with empty folders + v4.07: we check that folder is empty + */ + CFileSize sum = 0; + UInt32 j; + UInt32 numSubstreams = folders[i].NumUnPackStreams; + if (numSubstreams == 0) + continue; + if (type == k7zIdSize) + for (j = 1; j < numSubstreams; j++) + { + CFileSize size; + RINOK(SzReadSize(sd, &size)); + (*unPackSizes)[si++] = size; + sum += size; + } + (*unPackSizes)[si++] = SzFolderGetUnPackSize(folders + i) - sum; + } + if (type == k7zIdSize) + { + RINOK(SzReadID(sd, &type)); + } + + for(i = 0; i < *numUnPackStreams; i++) + { + (*digestsDefined)[i] = 0; + (*digests)[i] = 0; + } + + + for(i = 0; i < numFolders; i++) + { + UInt32 numSubstreams = folders[i].NumUnPackStreams; + if (numSubstreams != 1 || !folders[i].UnPackCRCDefined) + numDigests += numSubstreams; + } + + + si = 0; + while(1) + { + if (type == k7zIdCRC) + { + int digestIndex = 0; + Byte *digestsDefined2 = 0; + UInt32 *digests2 = 0; + SZ_RESULT res = SzReadHashDigests(sd, numDigests, &digestsDefined2, &digests2, allocTemp->Alloc); + if (res == SZ_OK) + { + for (i = 0; i < numFolders; i++) + { + CFolder *folder = folders + i; + UInt32 numSubstreams = folder->NumUnPackStreams; + if (numSubstreams == 1 && folder->UnPackCRCDefined) + { + (*digestsDefined)[si] = 1; + (*digests)[si] = folder->UnPackCRC; + si++; + } + else + { + UInt32 j; + for (j = 0; j < numSubstreams; j++, digestIndex++) + { + (*digestsDefined)[si] = digestsDefined2[digestIndex]; + (*digests)[si] = digests2[digestIndex]; + si++; + } + } + } + } + allocTemp->Free(digestsDefined2); + allocTemp->Free(digests2); + RINOK(res); + } + else if (type == k7zIdEnd) + return SZ_OK; + else + { + RINOK(SzSkeepData(sd)); + } + RINOK(SzReadID(sd, &type)); + } +} + + +SZ_RESULT SzReadStreamsInfo( + CSzData *sd, + CFileSize *dataOffset, + CArchiveDatabase *db, + UInt32 *numUnPackStreams, + CFileSize **unPackSizes, /* allocTemp */ + Byte **digestsDefined, /* allocTemp */ + UInt32 **digests, /* allocTemp */ + void * (*allocFunc)(size_t size), + ISzAlloc *allocTemp) +{ + while(1) + { + UInt64 type; + RINOK(SzReadID(sd, &type)); + if ((UInt64)(int)type != type) + return SZE_FAIL; + switch((int)type) + { + case k7zIdEnd: + return SZ_OK; + case k7zIdPackInfo: + { + RINOK(SzReadPackInfo(sd, dataOffset, &db->NumPackStreams, + &db->PackSizes, &db->PackCRCsDefined, &db->PackCRCs, allocFunc)); + break; + } + case k7zIdUnPackInfo: + { + RINOK(SzReadUnPackInfo(sd, &db->NumFolders, &db->Folders, allocFunc, allocTemp)); + break; + } + case k7zIdSubStreamsInfo: + { + RINOK(SzReadSubStreamsInfo(sd, db->NumFolders, db->Folders, + numUnPackStreams, unPackSizes, digestsDefined, digests, allocTemp)); + break; + } + default: + return SZE_FAIL; + } + } +} + +Byte kUtf8Limits[5] = { 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; + +SZ_RESULT SzReadFileNames(CSzData *sd, UInt32 numFiles, CFileItem *files, + void * (*allocFunc)(size_t size)) +{ + UInt32 i; + for(i = 0; i < numFiles; i++) + { + UInt32 len = 0; + UInt32 pos = 0; + CFileItem *file = files + i; + while(pos + 2 <= sd->Size) + { + int numAdds; + UInt32 value = (UInt32)(sd->Data[pos] | (((UInt32)sd->Data[pos + 1]) << 8)); + pos += 2; + len++; + if (value == 0) + break; + if (value < 0x80) + continue; + if (value >= 0xD800 && value < 0xE000) + { + UInt32 c2; + if (value >= 0xDC00) + return SZE_ARCHIVE_ERROR; + if (pos + 2 > sd->Size) + return SZE_ARCHIVE_ERROR; + c2 = (UInt32)(sd->Data[pos] | (((UInt32)sd->Data[pos + 1]) << 8)); + pos += 2; + if (c2 < 0xDC00 || c2 >= 0xE000) + return SZE_ARCHIVE_ERROR; + value = ((value - 0xD800) << 10) | (c2 - 0xDC00); + } + for (numAdds = 1; numAdds < 5; numAdds++) + if (value < (((UInt32)1) << (numAdds * 5 + 6))) + break; + len += numAdds; + } + + MY_ALLOC(char, file->Name, (size_t)len, allocFunc); + + len = 0; + while(2 <= sd->Size) + { + int numAdds; + UInt32 value = (UInt32)(sd->Data[0] | (((UInt32)sd->Data[1]) << 8)); + SzSkeepDataSize(sd, 2); + if (value < 0x80) + { + file->Name[len++] = (char)value; + if (value == 0) + break; + continue; + } + if (value >= 0xD800 && value < 0xE000) + { + UInt32 c2 = (UInt32)(sd->Data[0] | (((UInt32)sd->Data[1]) << 8)); + SzSkeepDataSize(sd, 2); + value = ((value - 0xD800) << 10) | (c2 - 0xDC00); + } + for (numAdds = 1; numAdds < 5; numAdds++) + if (value < (((UInt32)1) << (numAdds * 5 + 6))) + break; + file->Name[len++] = (char)(kUtf8Limits[numAdds - 1] + (value >> (6 * numAdds))); + do + { + numAdds--; + file->Name[len++] = (char)(0x80 + ((value >> (6 * numAdds)) & 0x3F)); + } + while(numAdds > 0); + + len += numAdds; + } + } + return SZ_OK; +} + +SZ_RESULT SzReadHeader2( + CSzData *sd, + CArchiveDatabaseEx *db, /* allocMain */ + CFileSize **unPackSizes, /* allocTemp */ + Byte **digestsDefined, /* allocTemp */ + UInt32 **digests, /* allocTemp */ + Byte **emptyStreamVector, /* allocTemp */ + Byte **emptyFileVector, /* allocTemp */ + ISzAlloc *allocMain, + ISzAlloc *allocTemp) +{ + UInt64 type; + UInt32 numUnPackStreams = 0; + UInt32 numFiles = 0; + CFileItem *files = 0; + UInt32 numEmptyStreams = 0; + UInt32 i; + + RINOK(SzReadID(sd, &type)); + + if (type == k7zIdArchiveProperties) + { + RINOK(SzReadArchiveProperties(sd)); + RINOK(SzReadID(sd, &type)); + } + + + if (type == k7zIdMainStreamsInfo) + { + RINOK(SzReadStreamsInfo(sd, + &db->ArchiveInfo.DataStartPosition, + &db->Database, + &numUnPackStreams, + unPackSizes, + digestsDefined, + digests, allocMain->Alloc, allocTemp)); + db->ArchiveInfo.DataStartPosition += db->ArchiveInfo.StartPositionAfterHeader; + RINOK(SzReadID(sd, &type)); + } + + if (type == k7zIdEnd) + return SZ_OK; + if (type != k7zIdFilesInfo) + return SZE_ARCHIVE_ERROR; + + RINOK(SzReadNumber32(sd, &numFiles)); + db->Database.NumFiles = numFiles; + + MY_ALLOC(CFileItem, files, (size_t)numFiles, allocMain->Alloc); + + db->Database.Files = files; + for(i = 0; i < numFiles; i++) + SzFileInit(files + i); + + while(1) + { + UInt64 type; + UInt64 size; + RINOK(SzReadID(sd, &type)); + if (type == k7zIdEnd) + break; + RINOK(SzReadNumber(sd, &size)); + + if ((UInt64)(int)type != type) + { + RINOK(SzSkeepDataSize(sd, size)); + } + else + switch((int)type) + { + case k7zIdName: + { + RINOK(SzReadSwitch(sd)); + RINOK(SzReadFileNames(sd, numFiles, files, allocMain->Alloc)) + break; + } + case k7zIdEmptyStream: + { + RINOK(SzReadBoolVector(sd, numFiles, emptyStreamVector, allocTemp->Alloc)); + numEmptyStreams = 0; + for (i = 0; i < numFiles; i++) + if ((*emptyStreamVector)[i]) + numEmptyStreams++; + break; + } + case k7zIdEmptyFile: + { + RINOK(SzReadBoolVector(sd, numEmptyStreams, emptyFileVector, allocTemp->Alloc)); + break; + } + default: + { + RINOK(SzSkeepDataSize(sd, size)); + } + } + } + + { + UInt32 emptyFileIndex = 0; + UInt32 sizeIndex = 0; + for(i = 0; i < numFiles; i++) + { + CFileItem *file = files + i; + file->IsAnti = 0; + if (*emptyStreamVector == 0) + file->HasStream = 1; + else + file->HasStream = (Byte)((*emptyStreamVector)[i] ? 0 : 1); + if(file->HasStream) + { + file->IsDirectory = 0; + file->Size = (*unPackSizes)[sizeIndex]; + file->FileCRC = (*digests)[sizeIndex]; + file->IsFileCRCDefined = (Byte)(*digestsDefined)[sizeIndex]; + sizeIndex++; + } + else + { + if (*emptyFileVector == 0) + file->IsDirectory = 1; + else + file->IsDirectory = (Byte)((*emptyFileVector)[emptyFileIndex] ? 0 : 1); + emptyFileIndex++; + file->Size = 0; + file->IsFileCRCDefined = 0; + } + } + } + return SzArDbExFill(db, allocMain->Alloc); +} + +SZ_RESULT SzReadHeader( + CSzData *sd, + CArchiveDatabaseEx *db, + ISzAlloc *allocMain, + ISzAlloc *allocTemp) +{ + CFileSize *unPackSizes = 0; + Byte *digestsDefined = 0; + UInt32 *digests = 0; + Byte *emptyStreamVector = 0; + Byte *emptyFileVector = 0; + SZ_RESULT res = SzReadHeader2(sd, db, + &unPackSizes, &digestsDefined, &digests, + &emptyStreamVector, &emptyFileVector, + allocMain, allocTemp); + allocTemp->Free(unPackSizes); + allocTemp->Free(digestsDefined); + allocTemp->Free(digests); + allocTemp->Free(emptyStreamVector); + allocTemp->Free(emptyFileVector); + return res; +} + +SZ_RESULT SzReadAndDecodePackedStreams2( + ISzInStream *inStream, + CSzData *sd, + CSzByteBuffer *outBuffer, + CFileSize baseOffset, + CArchiveDatabase *db, + CFileSize **unPackSizes, + Byte **digestsDefined, + UInt32 **digests, + #ifndef _LZMA_IN_CB + Byte **inBuffer, + #endif + ISzAlloc *allocTemp) +{ + + UInt32 numUnPackStreams = 0; + CFileSize dataStartPos; + CFolder *folder; + #ifndef _LZMA_IN_CB + CFileSize packSize = 0; + UInt32 i = 0; + #endif + CFileSize unPackSize; + size_t outRealSize; + SZ_RESULT res; + + RINOK(SzReadStreamsInfo(sd, &dataStartPos, db, + &numUnPackStreams, unPackSizes, digestsDefined, digests, + allocTemp->Alloc, allocTemp)); + + dataStartPos += baseOffset; + if (db->NumFolders != 1) + return SZE_ARCHIVE_ERROR; + + folder = db->Folders; + unPackSize = SzFolderGetUnPackSize(folder); + + RINOK(inStream->Seek(inStream, dataStartPos)); + + #ifndef _LZMA_IN_CB + for (i = 0; i < db->NumPackStreams; i++) + packSize += db->PackSizes[i]; + + MY_ALLOC(Byte, *inBuffer, (size_t)packSize, allocTemp->Alloc); + + RINOK(SafeReadDirect(inStream, *inBuffer, (size_t)packSize)); + #endif + + if (!SzByteBufferCreate(outBuffer, (size_t)unPackSize, allocTemp->Alloc)) + return SZE_OUTOFMEMORY; + + res = SzDecode(db->PackSizes, folder, + #ifdef _LZMA_IN_CB + inStream, + #else + *inBuffer, + #endif + outBuffer->Items, (size_t)unPackSize, + &outRealSize, allocTemp); + RINOK(res) + if (outRealSize != (UInt32)unPackSize) + return SZE_FAIL; + if (folder->UnPackCRCDefined) + if (!CrcVerifyDigest(folder->UnPackCRC, outBuffer->Items, (size_t)unPackSize)) + return SZE_FAIL; + return SZ_OK; +} + +SZ_RESULT SzReadAndDecodePackedStreams( + ISzInStream *inStream, + CSzData *sd, + CSzByteBuffer *outBuffer, + CFileSize baseOffset, + ISzAlloc *allocTemp) +{ + CArchiveDatabase db; + CFileSize *unPackSizes = 0; + Byte *digestsDefined = 0; + UInt32 *digests = 0; + #ifndef _LZMA_IN_CB + Byte *inBuffer = 0; + #endif + SZ_RESULT res; + SzArchiveDatabaseInit(&db); + res = SzReadAndDecodePackedStreams2(inStream, sd, outBuffer, baseOffset, + &db, &unPackSizes, &digestsDefined, &digests, + #ifndef _LZMA_IN_CB + &inBuffer, + #endif + allocTemp); + SzArchiveDatabaseFree(&db, allocTemp->Free); + allocTemp->Free(unPackSizes); + allocTemp->Free(digestsDefined); + allocTemp->Free(digests); + #ifndef _LZMA_IN_CB + allocTemp->Free(inBuffer); + #endif + return res; +} + +SZ_RESULT SzArchiveOpen2( + ISzInStream *inStream, + CArchiveDatabaseEx *db, + ISzAlloc *allocMain, + ISzAlloc *allocTemp) +{ + Byte signature[k7zSignatureSize]; + Byte version; + UInt32 crcFromArchive; + UInt64 nextHeaderOffset; + UInt64 nextHeaderSize; + UInt32 nextHeaderCRC; + UInt32 crc; + CFileSize pos = 0; + CSzByteBuffer buffer; + CSzData sd; + SZ_RESULT res; + + RINOK(SafeReadDirect(inStream, signature, k7zSignatureSize)); + + if (!TestSignatureCandidate(signature)) + return SZE_ARCHIVE_ERROR; + + /* + db.Clear(); + db.ArchiveInfo.StartPosition = _arhiveBeginStreamPosition; + */ + RINOK(SafeReadDirectByte(inStream, &version)); + if (version != k7zMajorVersion) + return SZE_ARCHIVE_ERROR; + RINOK(SafeReadDirectByte(inStream, &version)); + + RINOK(SafeReadDirectUInt32(inStream, &crcFromArchive)); + + CrcInit(&crc); + RINOK(SafeReadDirectUInt64(inStream, &nextHeaderOffset)); + CrcUpdateUInt64(&crc, nextHeaderOffset); + RINOK(SafeReadDirectUInt64(inStream, &nextHeaderSize)); + CrcUpdateUInt64(&crc, nextHeaderSize); + RINOK(SafeReadDirectUInt32(inStream, &nextHeaderCRC)); + CrcUpdateUInt32(&crc, nextHeaderCRC); + + pos = k7zStartHeaderSize; + db->ArchiveInfo.StartPositionAfterHeader = pos; + + if (CrcGetDigest(&crc) != crcFromArchive) + return SZE_ARCHIVE_ERROR; + + if (nextHeaderSize == 0) + return SZ_OK; + + RINOK(inStream->Seek(inStream, (CFileSize)(pos + nextHeaderOffset))); + + if (!SzByteBufferCreate(&buffer, (size_t)nextHeaderSize, allocTemp->Alloc)) + return SZE_OUTOFMEMORY; + + res = SafeReadDirect(inStream, buffer.Items, (size_t)nextHeaderSize); + if (res == SZ_OK) + { + if (CrcVerifyDigest(nextHeaderCRC, buffer.Items, (UInt32)nextHeaderSize)) + { + while (1) + { + UInt64 type; + sd.Data = buffer.Items; + sd.Size = buffer.Capacity; + res = SzReadID(&sd, &type); + if (res != SZ_OK) + break; + if (type == k7zIdHeader) + { + res = SzReadHeader(&sd, db, allocMain, allocTemp); + break; + } + if (type != k7zIdEncodedHeader) + { + res = SZE_ARCHIVE_ERROR; + break; + } + { + CSzByteBuffer outBuffer; + res = SzReadAndDecodePackedStreams(inStream, &sd, &outBuffer, + db->ArchiveInfo.StartPositionAfterHeader, + allocTemp); + if (res != SZ_OK) + { + SzByteBufferFree(&outBuffer, allocTemp->Free); + break; + } + SzByteBufferFree(&buffer, allocTemp->Free); + buffer.Items = outBuffer.Items; + buffer.Capacity = outBuffer.Capacity; + } + } + } + } + SzByteBufferFree(&buffer, allocTemp->Free); + return res; +} + +SZ_RESULT SzArchiveOpen( + ISzInStream *inStream, + CArchiveDatabaseEx *db, + ISzAlloc *allocMain, + ISzAlloc *allocTemp) +{ + SZ_RESULT res = SzArchiveOpen2(inStream, db, allocMain, allocTemp); + if (res != SZ_OK) + SzArDbExFree(db, allocMain->Free); + return res; +} diff --git a/lzma443/C/7zip/Archive/7z_C/7zIn.h b/lzma443/C/7zip/Archive/7z_C/7zIn.h new file mode 100755 index 0000000..6bfa2a7 --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/7zIn.h @@ -0,0 +1,55 @@ +/* 7zIn.h */ + +#ifndef __7Z_IN_H +#define __7Z_IN_H + +#include "7zHeader.h" +#include "7zItem.h" +#include "7zAlloc.h" + +typedef struct _CInArchiveInfo +{ + CFileSize StartPositionAfterHeader; + CFileSize DataStartPosition; +}CInArchiveInfo; + +typedef struct _CArchiveDatabaseEx +{ + CArchiveDatabase Database; + CInArchiveInfo ArchiveInfo; + UInt32 *FolderStartPackStreamIndex; + CFileSize *PackStreamStartPositions; + UInt32 *FolderStartFileIndex; + UInt32 *FileIndexToFolderIndexMap; +}CArchiveDatabaseEx; + +void SzArDbExInit(CArchiveDatabaseEx *db); +void SzArDbExFree(CArchiveDatabaseEx *db, void (*freeFunc)(void *)); +CFileSize SzArDbGetFolderStreamPos(CArchiveDatabaseEx *db, UInt32 folderIndex, UInt32 indexInFolder); +CFileSize SzArDbGetFolderFullPackSize(CArchiveDatabaseEx *db, UInt32 folderIndex); + +typedef struct _ISzInStream +{ + #ifdef _LZMA_IN_CB + SZ_RESULT (*Read)( + void *object, /* pointer to ISzInStream itself */ + void **buffer, /* out: pointer to buffer with data */ + size_t maxRequiredSize, /* max required size to read */ + size_t *processedSize); /* real processed size. + processedSize can be less than maxRequiredSize. + If processedSize == 0, then there are no more + bytes in stream. */ + #else + SZ_RESULT (*Read)(void *object, void *buffer, size_t size, size_t *processedSize); + #endif + SZ_RESULT (*Seek)(void *object, CFileSize pos); +} ISzInStream; + + +int SzArchiveOpen( + ISzInStream *inStream, + CArchiveDatabaseEx *db, + ISzAlloc *allocMain, + ISzAlloc *allocTemp); + +#endif diff --git a/lzma443/C/7zip/Archive/7z_C/7zItem.c b/lzma443/C/7zip/Archive/7z_C/7zItem.c new file mode 100755 index 0000000..2a40805 --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/7zItem.c @@ -0,0 +1,133 @@ +/* 7zItem.c */ + +#include "7zItem.h" +#include "7zAlloc.h" + +void SzCoderInfoInit(CCoderInfo *coder) +{ + SzByteBufferInit(&coder->Properties); +} + +void SzCoderInfoFree(CCoderInfo *coder, void (*freeFunc)(void *p)) +{ + SzByteBufferFree(&coder->Properties, freeFunc); + SzCoderInfoInit(coder); +} + +void SzFolderInit(CFolder *folder) +{ + folder->NumCoders = 0; + folder->Coders = 0; + folder->NumBindPairs = 0; + folder->BindPairs = 0; + folder->NumPackStreams = 0; + folder->PackStreams = 0; + folder->UnPackSizes = 0; + folder->UnPackCRCDefined = 0; + folder->UnPackCRC = 0; + folder->NumUnPackStreams = 0; +} + +void SzFolderFree(CFolder *folder, void (*freeFunc)(void *p)) +{ + UInt32 i; + for (i = 0; i < folder->NumCoders; i++) + SzCoderInfoFree(&folder->Coders[i], freeFunc); + freeFunc(folder->Coders); + freeFunc(folder->BindPairs); + freeFunc(folder->PackStreams); + freeFunc(folder->UnPackSizes); + SzFolderInit(folder); +} + +UInt32 SzFolderGetNumOutStreams(CFolder *folder) +{ + UInt32 result = 0; + UInt32 i; + for (i = 0; i < folder->NumCoders; i++) + result += folder->Coders[i].NumOutStreams; + return result; +} + +int SzFolderFindBindPairForInStream(CFolder *folder, UInt32 inStreamIndex) +{ + UInt32 i; + for(i = 0; i < folder->NumBindPairs; i++) + if (folder->BindPairs[i].InIndex == inStreamIndex) + return i; + return -1; +} + + +int SzFolderFindBindPairForOutStream(CFolder *folder, UInt32 outStreamIndex) +{ + UInt32 i; + for(i = 0; i < folder->NumBindPairs; i++) + if (folder->BindPairs[i].OutIndex == outStreamIndex) + return i; + return -1; +} + +CFileSize SzFolderGetUnPackSize(CFolder *folder) +{ + int i = (int)SzFolderGetNumOutStreams(folder); + if (i == 0) + return 0; + for (i--; i >= 0; i--) + if (SzFolderFindBindPairForOutStream(folder, i) < 0) + return folder->UnPackSizes[i]; + /* throw 1; */ + return 0; +} + +/* +int FindPackStreamArrayIndex(int inStreamIndex) const +{ + for(int i = 0; i < PackStreams.Size(); i++) + if (PackStreams[i] == inStreamIndex) + return i; + return -1; +} +*/ + +void SzFileInit(CFileItem *fileItem) +{ + fileItem->IsFileCRCDefined = 0; + fileItem->HasStream = 1; + fileItem->IsDirectory = 0; + fileItem->IsAnti = 0; + fileItem->Name = 0; +} + +void SzFileFree(CFileItem *fileItem, void (*freeFunc)(void *p)) +{ + freeFunc(fileItem->Name); + SzFileInit(fileItem); +} + +void SzArchiveDatabaseInit(CArchiveDatabase *db) +{ + db->NumPackStreams = 0; + db->PackSizes = 0; + db->PackCRCsDefined = 0; + db->PackCRCs = 0; + db->NumFolders = 0; + db->Folders = 0; + db->NumFiles = 0; + db->Files = 0; +} + +void SzArchiveDatabaseFree(CArchiveDatabase *db, void (*freeFunc)(void *)) +{ + UInt32 i; + for (i = 0; i < db->NumFolders; i++) + SzFolderFree(&db->Folders[i], freeFunc); + for (i = 0; i < db->NumFiles; i++) + SzFileFree(&db->Files[i], freeFunc); + freeFunc(db->PackSizes); + freeFunc(db->PackCRCsDefined); + freeFunc(db->PackCRCs); + freeFunc(db->Folders); + freeFunc(db->Files); + SzArchiveDatabaseInit(db); +} diff --git a/lzma443/C/7zip/Archive/7z_C/7zItem.h b/lzma443/C/7zip/Archive/7z_C/7zItem.h new file mode 100755 index 0000000..876539a --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/7zItem.h @@ -0,0 +1,90 @@ +/* 7zItem.h */ + +#ifndef __7Z_ITEM_H +#define __7Z_ITEM_H + +#include "7zMethodID.h" +#include "7zHeader.h" +#include "7zBuffer.h" + +typedef struct _CCoderInfo +{ + UInt32 NumInStreams; + UInt32 NumOutStreams; + CMethodID MethodID; + CSzByteBuffer Properties; +}CCoderInfo; + +void SzCoderInfoInit(CCoderInfo *coder); +void SzCoderInfoFree(CCoderInfo *coder, void (*freeFunc)(void *p)); + +typedef struct _CBindPair +{ + UInt32 InIndex; + UInt32 OutIndex; +}CBindPair; + +typedef struct _CFolder +{ + UInt32 NumCoders; + CCoderInfo *Coders; + UInt32 NumBindPairs; + CBindPair *BindPairs; + UInt32 NumPackStreams; + UInt32 *PackStreams; + CFileSize *UnPackSizes; + int UnPackCRCDefined; + UInt32 UnPackCRC; + + UInt32 NumUnPackStreams; +}CFolder; + +void SzFolderInit(CFolder *folder); +CFileSize SzFolderGetUnPackSize(CFolder *folder); +int SzFolderFindBindPairForInStream(CFolder *folder, UInt32 inStreamIndex); +UInt32 SzFolderGetNumOutStreams(CFolder *folder); +CFileSize SzFolderGetUnPackSize(CFolder *folder); + +/* #define CArchiveFileTime UInt64 */ + +typedef struct _CFileItem +{ + /* + CArchiveFileTime LastWriteTime; + CFileSize StartPos; + UInt32 Attributes; + */ + CFileSize Size; + UInt32 FileCRC; + char *Name; + + Byte IsFileCRCDefined; + Byte HasStream; + Byte IsDirectory; + Byte IsAnti; + /* + int AreAttributesDefined; + int IsLastWriteTimeDefined; + int IsStartPosDefined; + */ +}CFileItem; + +void SzFileInit(CFileItem *fileItem); + +typedef struct _CArchiveDatabase +{ + UInt32 NumPackStreams; + CFileSize *PackSizes; + Byte *PackCRCsDefined; + UInt32 *PackCRCs; + UInt32 NumFolders; + CFolder *Folders; + UInt32 NumFiles; + CFileItem *Files; +}CArchiveDatabase; + +void SzArchiveDatabaseInit(CArchiveDatabase *db); +void SzArchiveDatabaseFree(CArchiveDatabase *db, void (*freeFunc)(void *)); + + +#endif diff --git a/lzma443/C/7zip/Archive/7z_C/7zMain.c b/lzma443/C/7zip/Archive/7z_C/7zMain.c new file mode 100755 index 0000000..73bf36b --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/7zMain.c @@ -0,0 +1,225 @@ +/* +7zMain.c +Test application for 7z Decoder +LZMA SDK 4.43 Copyright (c) 1999-2006 Igor Pavlov (2006-06-04) +*/ + +#include +#include +#include + +#include "7zCrc.h" +#include "7zIn.h" +#include "7zExtract.h" + +typedef struct _CFileInStream +{ + ISzInStream InStream; + FILE *File; +} CFileInStream; + +#ifdef _LZMA_IN_CB + +#define kBufferSize (1 << 12) +Byte g_Buffer[kBufferSize]; + +SZ_RESULT SzFileReadImp(void *object, void **buffer, size_t maxRequiredSize, size_t *processedSize) +{ + CFileInStream *s = (CFileInStream *)object; + size_t processedSizeLoc; + if (maxRequiredSize > kBufferSize) + maxRequiredSize = kBufferSize; + processedSizeLoc = fread(g_Buffer, 1, maxRequiredSize, s->File); + *buffer = g_Buffer; + if (processedSize != 0) + *processedSize = processedSizeLoc; + return SZ_OK; +} + +#else + +SZ_RESULT SzFileReadImp(void *object, void *buffer, size_t size, size_t *processedSize) +{ + CFileInStream *s = (CFileInStream *)object; + size_t processedSizeLoc = fread(buffer, 1, size, s->File); + if (processedSize != 0) + *processedSize = processedSizeLoc; + return SZ_OK; +} + +#endif + +SZ_RESULT SzFileSeekImp(void *object, CFileSize pos) +{ + CFileInStream *s = (CFileInStream *)object; + int res = fseek(s->File, (long)pos, SEEK_SET); + if (res == 0) + return SZ_OK; + return SZE_FAIL; +} + +void PrintError(char *sz) +{ + printf("\nERROR: %s\n", sz); +} + +int main(int numargs, char *args[]) +{ + CFileInStream archiveStream; + CArchiveDatabaseEx db; + SZ_RESULT res; + ISzAlloc allocImp; + ISzAlloc allocTempImp; + + printf("\n7z ANSI-C Decoder 4.43 Copyright (c) 1999-2006 Igor Pavlov 2006-06-04\n"); + if (numargs == 1) + { + printf( + "\nUsage: 7zDec \n\n" + "\n" + " e: Extract files from archive\n" + " l: List contents of archive\n" + " t: Test integrity of archive\n"); + return 0; + } + if (numargs < 3) + { + PrintError("incorrect command"); + return 1; + } + + archiveStream.File = fopen(args[2], "rb"); + if (archiveStream.File == 0) + { + PrintError("can not open input file"); + return 1; + } + + archiveStream.InStream.Read = SzFileReadImp; + archiveStream.InStream.Seek = SzFileSeekImp; + + allocImp.Alloc = SzAlloc; + allocImp.Free = SzFree; + + allocTempImp.Alloc = SzAllocTemp; + allocTempImp.Free = SzFreeTemp; + + InitCrcTable(); + SzArDbExInit(&db); + res = SzArchiveOpen(&archiveStream.InStream, &db, &allocImp, &allocTempImp); + if (res == SZ_OK) + { + char *command = args[1]; + int listCommand = 0; + int testCommand = 0; + int extractCommand = 0; + if (strcmp(command, "l") == 0) + listCommand = 1; + if (strcmp(command, "t") == 0) + testCommand = 1; + else if (strcmp(command, "e") == 0) + extractCommand = 1; + + if (listCommand) + { + UInt32 i; + for (i = 0; i < db.Database.NumFiles; i++) + { + CFileItem *f = db.Database.Files + i; + printf("%10d %s\n", (int)f->Size, f->Name); + } + } + else if (testCommand || extractCommand) + { + UInt32 i; + + /* + if you need cache, use these 3 variables. + if you use external function, you can make these variable as static. + */ + UInt32 blockIndex = 0xFFFFFFFF; /* it can have any value before first call (if outBuffer = 0) */ + Byte *outBuffer = 0; /* it must be 0 before first call for each new archive. */ + size_t outBufferSize = 0; /* it can have any value before first call (if outBuffer = 0) */ + + printf("\n"); + for (i = 0; i < db.Database.NumFiles; i++) + { + size_t offset; + size_t outSizeProcessed; + CFileItem *f = db.Database.Files + i; + if (f->IsDirectory) + printf("Directory "); + else + printf(testCommand ? + "Testing ": + "Extracting"); + printf(" %s", f->Name); + if (f->IsDirectory) + { + printf("\n"); + continue; + } + res = SzExtract(&archiveStream.InStream, &db, i, + &blockIndex, &outBuffer, &outBufferSize, + &offset, &outSizeProcessed, + &allocImp, &allocTempImp); + if (res != SZ_OK) + break; + if (!testCommand) + { + FILE *outputHandle; + UInt32 processedSize; + char *fileName = f->Name; + size_t nameLen = strlen(f->Name); + for (; nameLen > 0; nameLen--) + if (f->Name[nameLen - 1] == '/') + { + fileName = f->Name + nameLen; + break; + } + + outputHandle = fopen(fileName, "wb+"); + if (outputHandle == 0) + { + PrintError("can not open output file"); + res = SZE_FAIL; + break; + } + processedSize = fwrite(outBuffer + offset, 1, outSizeProcessed, outputHandle); + if (processedSize != outSizeProcessed) + { + PrintError("can not write output file"); + res = SZE_FAIL; + break; + } + if (fclose(outputHandle)) + { + PrintError("can not close output file"); + res = SZE_FAIL; + break; + } + } + printf("\n"); + } + allocImp.Free(outBuffer); + } + else + { + PrintError("incorrect command"); + res = SZE_FAIL; + } + } + SzArDbExFree(&db, allocImp.Free); + + fclose(archiveStream.File); + if (res == SZ_OK) + { + printf("\nEverything is Ok\n"); + return 0; + } + if (res == SZE_OUTOFMEMORY) + PrintError("can not allocate memory"); + else + printf("\nERROR #%d\n", res); + return 1; +} diff --git a/lzma443/C/7zip/Archive/7z_C/7zMethodID.c b/lzma443/C/7zip/Archive/7z_C/7zMethodID.c new file mode 100755 index 0000000..5047359 --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/7zMethodID.c @@ -0,0 +1,14 @@ +/* 7zMethodID.c */ + +#include "7zMethodID.h" + +int AreMethodsEqual(CMethodID *a1, CMethodID *a2) +{ + int i; + if (a1->IDSize != a2->IDSize) + return 0; + for (i = 0; i < a1->IDSize; i++) + if (a1->ID[i] != a2->ID[i]) + return 0; + return 1; +} diff --git a/lzma443/C/7zip/Archive/7z_C/7zMethodID.h b/lzma443/C/7zip/Archive/7z_C/7zMethodID.h new file mode 100755 index 0000000..162fcd1 --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/7zMethodID.h @@ -0,0 +1,18 @@ +/* 7zMethodID.h */ + +#ifndef __7Z_METHOD_ID_H +#define __7Z_METHOD_ID_H + +#include "7zTypes.h" + +#define kMethodIDSize 15 + +typedef struct _CMethodID +{ + Byte ID[kMethodIDSize]; + Byte IDSize; +} CMethodID; + +int AreMethodsEqual(CMethodID *a1, CMethodID *a2); + +#endif diff --git a/lzma443/C/7zip/Archive/7z_C/7zTypes.h b/lzma443/C/7zip/Archive/7z_C/7zTypes.h new file mode 100755 index 0000000..60dd68c --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/7zTypes.h @@ -0,0 +1,67 @@ +/* 7zTypes.h */ + +#ifndef __COMMON_TYPES_H +#define __COMMON_TYPES_H + +#ifndef _7ZIP_BYTE_DEFINED +#define _7ZIP_BYTE_DEFINED +typedef unsigned char Byte; +#endif + +#ifndef _7ZIP_UINT16_DEFINED +#define _7ZIP_UINT16_DEFINED +typedef unsigned short UInt16; +#endif + +#ifndef _7ZIP_UINT32_DEFINED +#define _7ZIP_UINT32_DEFINED +#ifdef _LZMA_UINT32_IS_ULONG +typedef unsigned long UInt32; +#else +typedef unsigned int UInt32; +#endif +#endif + +/* #define _SZ_NO_INT_64 */ +/* define it your compiler doesn't support long long int */ + +#ifndef _7ZIP_UINT64_DEFINED +#define _7ZIP_UINT64_DEFINED +#ifdef _SZ_NO_INT_64 +typedef unsigned long UInt64; +#else +#ifdef _MSC_VER +typedef unsigned __int64 UInt64; +#else +typedef unsigned long long int UInt64; +#endif +#endif +#endif + + +/* #define _SZ_FILE_SIZE_64 */ +/* Use _SZ_FILE_SIZE_64 if you need support for files larger than 4 GB*/ + +#ifndef CFileSize +#ifdef _SZ_FILE_SIZE_64 +typedef UInt64 CFileSize; +#else +typedef UInt32 CFileSize; +#endif +#endif + +#define SZ_RESULT int + +#define SZ_OK (0) +#define SZE_DATA_ERROR (1) +#define SZE_OUTOFMEMORY (2) +#define SZE_CRC_ERROR (3) + +#define SZE_NOTIMPL (4) +#define SZE_FAIL (5) + +#define SZE_ARCHIVE_ERROR (6) + +#define RINOK(x) { int __result_ = (x); if(__result_ != 0) return __result_; } + +#endif diff --git a/lzma443/C/7zip/Archive/7z_C/7z_C.dsp b/lzma443/C/7zip/Archive/7z_C/7z_C.dsp new file mode 100755 index 0000000..596ccb5 --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/7z_C.dsp @@ -0,0 +1,182 @@ +# Microsoft Developer Studio Project File - Name="7z_C" - Package Owner=<4> +# Microsoft Developer Studio Generated Build File, Format Version 6.00 +# ** DO NOT EDIT ** + +# TARGTYPE "Win32 (x86) Console Application" 0x0103 + +CFG=7z_C - Win32 Debug +!MESSAGE This is not a valid makefile. To build this project using NMAKE, +!MESSAGE use the Export Makefile command and run +!MESSAGE +!MESSAGE NMAKE /f "7z_C.mak". +!MESSAGE +!MESSAGE You can specify a configuration when running NMAKE +!MESSAGE by defining the macro CFG on the command line. For example: +!MESSAGE +!MESSAGE NMAKE /f "7z_C.mak" CFG="7z_C - Win32 Debug" +!MESSAGE +!MESSAGE Possible choices for configuration are: +!MESSAGE +!MESSAGE "7z_C - Win32 Release" (based on "Win32 (x86) Console Application") +!MESSAGE "7z_C - Win32 Debug" (based on "Win32 (x86) Console Application") +!MESSAGE + +# Begin Project +# PROP AllowPerConfigDependencies 0 +# PROP Scc_ProjName "" +# PROP Scc_LocalPath "" +CPP=cl.exe +RSC=rc.exe + +!IF "$(CFG)" == "7z_C - Win32 Release" + +# PROP BASE Use_MFC 0 +# PROP BASE Use_Debug_Libraries 0 +# PROP BASE Output_Dir "Release" +# PROP BASE Intermediate_Dir "Release" +# PROP BASE Target_Dir "" +# PROP Use_MFC 0 +# PROP Use_Debug_Libraries 0 +# PROP Output_Dir "Release" +# PROP Intermediate_Dir "Release" +# PROP Ignore_Export_Lib 0 +# PROP Target_Dir "" +# ADD BASE CPP /nologo /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /c +# ADD CPP /nologo /W4 /GX /O2 /D "NDEBUG" /D "WIN32" /D "_CONSOLE" /D "_MBCS" /D "_LZMA_PROB32" /D "_LZMA_IN_CB" /YX /FD /c +# ADD BASE RSC /l 0x419 /d "NDEBUG" +# ADD RSC /l 0x419 /d "NDEBUG" +BSC32=bscmake.exe +# ADD BASE BSC32 /nologo +# ADD BSC32 /nologo +LINK32=link.exe +# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:console /machine:I386 +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:console /machine:I386 /out:"Release/7zDec.exe" + +!ELSEIF "$(CFG)" == "7z_C - Win32 Debug" + +# PROP BASE Use_MFC 0 +# PROP BASE Use_Debug_Libraries 1 +# PROP BASE Output_Dir "Debug" +# PROP BASE Intermediate_Dir "Debug" +# PROP BASE Target_Dir "" +# PROP Use_MFC 0 +# PROP Use_Debug_Libraries 1 +# PROP Output_Dir "Debug" +# PROP Intermediate_Dir "Debug" +# PROP Ignore_Export_Lib 0 +# PROP Target_Dir "" +# ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /GZ /c +# ADD CPP /nologo /W3 /Gm /GX /ZI /Od /D "_DEBUG" /D "WIN32" /D "_CONSOLE" /D "_MBCS" /D "_LZMA_PROB32" /D "_LZMA_IN_CB" /YX /FD /GZ /c +# ADD BASE RSC /l 0x419 /d "_DEBUG" +# ADD RSC /l 0x419 /d "_DEBUG" +BSC32=bscmake.exe +# ADD BASE BSC32 /nologo +# ADD BSC32 /nologo +LINK32=link.exe +# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:console /debug /machine:I386 /pdbtype:sept +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:console /debug /machine:I386 /out:"Debug/7zDec.exe" /pdbtype:sept + +!ENDIF + +# Begin Target + +# Name "7z_C - Win32 Release" +# Name "7z_C - Win32 Debug" +# Begin Group "LZMA" + +# PROP Default_Filter "" +# Begin Source File + +SOURCE=..\..\Compress\LZMA_C\LzmaDecode.c +# End Source File +# Begin Source File + +SOURCE=..\..\Compress\LZMA_C\LzmaDecode.h +# End Source File +# Begin Source File + +SOURCE=..\..\Compress\LZMA_C\LzmaTypes.h +# End Source File +# End Group +# Begin Source File + +SOURCE=.\7zAlloc.c +# End Source File +# Begin Source File + +SOURCE=.\7zAlloc.h +# End Source File +# Begin Source File + +SOURCE=.\7zBuffer.c +# End Source File +# Begin Source File + +SOURCE=.\7zBuffer.h +# End Source File +# Begin Source File + +SOURCE=.\7zCrc.c +# End Source File +# Begin Source File + +SOURCE=.\7zCrc.h +# End Source File +# Begin Source File + +SOURCE=.\7zDecode.c +# End Source File +# Begin Source File + +SOURCE=.\7zDecode.h +# End Source File +# Begin Source File + +SOURCE=.\7zExtract.c +# End Source File +# Begin Source File + +SOURCE=.\7zExtract.h +# End Source File +# Begin Source File + +SOURCE=.\7zHeader.c +# End Source File +# Begin Source File + +SOURCE=.\7zHeader.h +# End Source File +# Begin Source File + +SOURCE=.\7zIn.c +# End Source File +# Begin Source File + +SOURCE=.\7zIn.h +# End Source File +# Begin Source File + +SOURCE=.\7zItem.c +# End Source File +# Begin Source File + +SOURCE=.\7zItem.h +# End Source File +# Begin Source File + +SOURCE=.\7zMain.c +# End Source File +# Begin Source File + +SOURCE=.\7zMethodID.c +# End Source File +# Begin Source File + +SOURCE=.\7zMethodID.h +# End Source File +# Begin Source File + +SOURCE=.\7zTypes.h +# End Source File +# End Target +# End Project diff --git a/lzma443/C/7zip/Archive/7z_C/7z_C.dsw b/lzma443/C/7zip/Archive/7z_C/7z_C.dsw new file mode 100755 index 0000000..f1ee72b --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/7z_C.dsw @@ -0,0 +1,29 @@ +Microsoft Developer Studio Workspace File, Format Version 6.00 +# WARNING: DO NOT EDIT OR DELETE THIS WORKSPACE FILE! + +############################################################################### + +Project: "7z_C"=.\7z_C.dsp - Package Owner=<4> + +Package=<5> +{{{ +}}} + +Package=<4> +{{{ +}}} + +############################################################################### + +Global: + +Package=<5> +{{{ +}}} + +Package=<3> +{{{ +}}} + +############################################################################### + diff --git a/lzma443/C/7zip/Archive/7z_C/makefile.gcc b/lzma443/C/7zip/Archive/7z_C/makefile.gcc new file mode 100755 index 0000000..cc8bebf --- /dev/null +++ b/lzma443/C/7zip/Archive/7z_C/makefile.gcc @@ -0,0 +1,50 @@ +PROG = 7zDec +CXX = g++ +LIB = +RM = rm -f +CFLAGS = -c -O2 -Wall + +OBJS = 7zAlloc.o 7zBuffer.o 7zCrc.o 7zDecode.o 7zExtract.o 7zHeader.o 7zIn.o 7zItem.o 7zMain.o 7zMethodID.o LzmaDecode.o + +all: $(PROG) + +$(PROG): $(OBJS) + $(CXX) -o $(PROG) $(LDFLAGS) $(OBJS) $(LIB) + +7zAlloc.o: 7zAlloc.c + $(CXX) $(CFLAGS) 7zAlloc.c + +7zBuffer.o: 7zBuffer.c + $(CXX) $(CFLAGS) 7zBuffer.c + +7zCrc.o: 7zCrc.c + $(CXX) $(CFLAGS) 7zCrc.c + +7zDecode.o: 7zDecode.c + $(CXX) $(CFLAGS) 7zDecode.c + +7zExtract.o: 7zExtract.c + $(CXX) $(CFLAGS) 7zExtract.c + +7zHeader.o: 7zHeader.c + $(CXX) $(CFLAGS) 7zHeader.c + +7zIn.o: 7zIn.c + $(CXX) $(CFLAGS) 7zIn.c + +7zItem.o: 7zItem.c + $(CXX) $(CFLAGS) 7zItem.c + +7zMain.o: 7zMain.c + $(CXX) $(CFLAGS) 7zMain.c + +7zMethodID.o: 7zMethodID.c + $(CXX) $(CFLAGS) 7zMethodID.c + +LzmaDecode.o: ../../Compress/LZMA_C/LzmaDecode.c + $(CXX) $(CFLAGS) ../../Compress/LZMA_C/LzmaDecode.c + + +clean: + -$(RM) $(PROG) $(OBJS) + diff --git a/lzma443/C/7zip/Common/FileStreams.cpp b/lzma443/C/7zip/Common/FileStreams.cpp new file mode 100755 index 0000000..8a000e4 --- /dev/null +++ b/lzma443/C/7zip/Common/FileStreams.cpp @@ -0,0 +1,251 @@ +// FileStreams.cpp + +#include "StdAfx.h" + +#ifndef _WIN32 +#include +#include +#include +#endif + +#include "FileStreams.h" + +static inline HRESULT ConvertBoolToHRESULT(bool result) +{ + // return result ? S_OK: E_FAIL; + #ifdef _WIN32 + return result ? S_OK: (::GetLastError()); + #else + return result ? S_OK: E_FAIL; + #endif +} + +bool CInFileStream::Open(LPCTSTR fileName) +{ + return File.Open(fileName); +} + +#ifdef _WIN32 +#ifndef _UNICODE +bool CInFileStream::Open(LPCWSTR fileName) +{ + return File.Open(fileName); +} +#endif +#endif + +STDMETHODIMP CInFileStream::Read(void *data, UInt32 size, UInt32 *processedSize) +{ + #ifdef _WIN32 + + UInt32 realProcessedSize; + bool result = File.ReadPart(data, size, realProcessedSize); + if(processedSize != NULL) + *processedSize = realProcessedSize; + return ConvertBoolToHRESULT(result); + + #else + + if(processedSize != NULL) + *processedSize = 0; + ssize_t res = File.Read(data, (size_t)size); + if (res == -1) + return E_FAIL; + if(processedSize != NULL) + *processedSize = (UInt32)res; + return S_OK; + + #endif +} + +#ifndef _WIN32_WCE +STDMETHODIMP CStdInFileStream::Read(void *data, UInt32 size, UInt32 *processedSize) +{ + #ifdef _WIN32 + UInt32 realProcessedSize; + BOOL res = ::ReadFile(GetStdHandle(STD_INPUT_HANDLE), + data, size, (DWORD *)&realProcessedSize, NULL); + if(processedSize != NULL) + *processedSize = realProcessedSize; + if (res == FALSE && GetLastError() == ERROR_BROKEN_PIPE) + return S_OK; + return ConvertBoolToHRESULT(res != FALSE); + + #else + + if(processedSize != NULL) + *processedSize = 0; + ssize_t res; + do + { + res = read(0, data, (size_t)size); + } + while (res < 0 && (errno == EINTR)); + if (res == -1) + return E_FAIL; + if(processedSize != NULL) + *processedSize = (UInt32)res; + return S_OK; + + #endif +} + +#endif + +STDMETHODIMP CInFileStream::Seek(Int64 offset, UInt32 seekOrigin, + UInt64 *newPosition) +{ + if(seekOrigin >= 3) + return STG_E_INVALIDFUNCTION; + + #ifdef _WIN32 + + UInt64 realNewPosition; + bool result = File.Seek(offset, seekOrigin, realNewPosition); + if(newPosition != NULL) + *newPosition = realNewPosition; + return ConvertBoolToHRESULT(result); + + #else + + off_t res = File.Seek(offset, seekOrigin); + if (res == -1) + return E_FAIL; + if(newPosition != NULL) + *newPosition = (UInt64)res; + return S_OK; + + #endif +} + +STDMETHODIMP CInFileStream::GetSize(UInt64 *size) +{ + return ConvertBoolToHRESULT(File.GetLength(*size)); +} + + +////////////////////////// +// COutFileStream + +bool COutFileStream::Create(LPCTSTR fileName, bool createAlways) +{ + return File.Create(fileName, createAlways); +} + +#ifdef _WIN32 +#ifndef _UNICODE +bool COutFileStream::Create(LPCWSTR fileName, bool createAlways) +{ + return File.Create(fileName, createAlways); +} +#endif +#endif + +STDMETHODIMP COutFileStream::Write(const void *data, UInt32 size, UInt32 *processedSize) +{ + #ifdef _WIN32 + + UInt32 realProcessedSize; + bool result = File.WritePart(data, size, realProcessedSize); + if(processedSize != NULL) + *processedSize = realProcessedSize; + return ConvertBoolToHRESULT(result); + + #else + + if(processedSize != NULL) + *processedSize = 0; + ssize_t res = File.Write(data, (size_t)size); + if (res == -1) + return E_FAIL; + if(processedSize != NULL) + *processedSize = (UInt32)res; + return S_OK; + + #endif +} + +STDMETHODIMP COutFileStream::Seek(Int64 offset, UInt32 seekOrigin, + UInt64 *newPosition) +{ + if(seekOrigin >= 3) + return STG_E_INVALIDFUNCTION; + #ifdef _WIN32 + + UInt64 realNewPosition; + bool result = File.Seek(offset, seekOrigin, realNewPosition); + if(newPosition != NULL) + *newPosition = realNewPosition; + return ConvertBoolToHRESULT(result); + + #else + + off_t res = File.Seek(offset, seekOrigin); + if (res == -1) + return E_FAIL; + if(newPosition != NULL) + *newPosition = (UInt64)res; + return S_OK; + + #endif +} + +STDMETHODIMP COutFileStream::SetSize(Int64 newSize) +{ + #ifdef _WIN32 + UInt64 currentPos; + if(!File.Seek(0, FILE_CURRENT, currentPos)) + return E_FAIL; + bool result = File.SetLength(newSize); + UInt64 currentPos2; + result = result && File.Seek(currentPos, currentPos2); + return result ? S_OK : E_FAIL; + #else + return E_FAIL; + #endif +} + +#ifndef _WIN32_WCE +STDMETHODIMP CStdOutFileStream::Write(const void *data, UInt32 size, UInt32 *processedSize) +{ + if(processedSize != NULL) + *processedSize = 0; + + #ifdef _WIN32 + UInt32 realProcessedSize; + BOOL res = TRUE; + if (size > 0) + { + // Seems that Windows doesn't like big amounts writing to stdout. + // So we limit portions by 32KB. + UInt32 sizeTemp = (1 << 15); + if (sizeTemp > size) + sizeTemp = size; + res = ::WriteFile(GetStdHandle(STD_OUTPUT_HANDLE), + data, sizeTemp, (DWORD *)&realProcessedSize, NULL); + size -= realProcessedSize; + data = (const void *)((const Byte *)data + realProcessedSize); + if(processedSize != NULL) + *processedSize += realProcessedSize; + } + return ConvertBoolToHRESULT(res != FALSE); + + #else + + ssize_t res; + do + { + res = write(1, data, (size_t)size); + } + while (res < 0 && (errno == EINTR)); + if (res == -1) + return E_FAIL; + if(processedSize != NULL) + *processedSize = (UInt32)res; + return S_OK; + + return S_OK; + #endif +} + +#endif diff --git a/lzma443/C/7zip/Common/FileStreams.h b/lzma443/C/7zip/Common/FileStreams.h new file mode 100755 index 0000000..9326372 --- /dev/null +++ b/lzma443/C/7zip/Common/FileStreams.h @@ -0,0 +1,98 @@ +// FileStreams.h + +#ifndef __FILESTREAMS_H +#define __FILESTREAMS_H + +#ifdef _WIN32 +#include "../../Windows/FileIO.h" +#else +#include "../../Common/C_FileIO.h" +#endif + +#include "../IStream.h" +#include "../../Common/MyCom.h" + +class CInFileStream: + public IInStream, + public IStreamGetSize, + public CMyUnknownImp +{ +public: + #ifdef _WIN32 + NWindows::NFile::NIO::CInFile File; + #else + NC::NFile::NIO::CInFile File; + #endif + CInFileStream() {} + virtual ~CInFileStream() {} + + bool Open(LPCTSTR fileName); + #ifdef _WIN32 + #ifndef _UNICODE + bool Open(LPCWSTR fileName); + #endif + #endif + + MY_UNKNOWN_IMP2(IInStream, IStreamGetSize) + + STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); + STDMETHOD(Seek)(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition); + + STDMETHOD(GetSize)(UInt64 *size); +}; + +#ifndef _WIN32_WCE +class CStdInFileStream: + public ISequentialInStream, + public CMyUnknownImp +{ +public: + // HANDLE File; + // CStdInFileStream() File(INVALID_HANDLE_VALUE): {} + // void Open() { File = GetStdHandle(STD_INPUT_HANDLE); }; + MY_UNKNOWN_IMP + + virtual ~CStdInFileStream() {} + STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); +}; +#endif + +class COutFileStream: + public IOutStream, + public CMyUnknownImp +{ +public: + #ifdef _WIN32 + NWindows::NFile::NIO::COutFile File; + #else + NC::NFile::NIO::COutFile File; + #endif + virtual ~COutFileStream() {} + bool Create(LPCTSTR fileName, bool createAlways); + #ifdef _WIN32 + #ifndef _UNICODE + bool Create(LPCWSTR fileName, bool createAlways); + #endif + #endif + + MY_UNKNOWN_IMP1(IOutStream) + + STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize); + STDMETHOD(Seek)(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition); + STDMETHOD(SetSize)(Int64 newSize); +}; + +#ifndef _WIN32_WCE +class CStdOutFileStream: + public ISequentialOutStream, + public CMyUnknownImp +{ +public: + MY_UNKNOWN_IMP + + virtual ~CStdOutFileStream() {} + STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize); +}; +#endif + +#endif diff --git a/lzma443/C/7zip/Common/InBuffer.cpp b/lzma443/C/7zip/Common/InBuffer.cpp new file mode 100755 index 0000000..02f2adf --- /dev/null +++ b/lzma443/C/7zip/Common/InBuffer.cpp @@ -0,0 +1,80 @@ +// InBuffer.cpp + +#include "StdAfx.h" + +#include "InBuffer.h" + +#include "../../Common/Alloc.h" + +CInBuffer::CInBuffer(): + _buffer(0), + _bufferLimit(0), + _bufferBase(0), + _stream(0), + _bufferSize(0) +{} + +bool CInBuffer::Create(UInt32 bufferSize) +{ + const UInt32 kMinBlockSize = 1; + if (bufferSize < kMinBlockSize) + bufferSize = kMinBlockSize; + if (_bufferBase != 0 && _bufferSize == bufferSize) + return true; + Free(); + _bufferSize = bufferSize; + _bufferBase = (Byte *)::MidAlloc(bufferSize); + return (_bufferBase != 0); +} + +void CInBuffer::Free() +{ + ::MidFree(_bufferBase); + _bufferBase = 0; +} + +void CInBuffer::SetStream(ISequentialInStream *stream) +{ + _stream = stream; +} + +void CInBuffer::Init() +{ + _processedSize = 0; + _buffer = _bufferBase; + _bufferLimit = _buffer; + _wasFinished = false; + #ifdef _NO_EXCEPTIONS + ErrorCode = S_OK; + #endif +} + +bool CInBuffer::ReadBlock() +{ + #ifdef _NO_EXCEPTIONS + if (ErrorCode != S_OK) + return false; + #endif + if (_wasFinished) + return false; + _processedSize += (_buffer - _bufferBase); + UInt32 numProcessedBytes; + HRESULT result = _stream->Read(_bufferBase, _bufferSize, &numProcessedBytes); + #ifdef _NO_EXCEPTIONS + ErrorCode = result; + #else + if (result != S_OK) + throw CInBufferException(result); + #endif + _buffer = _bufferBase; + _bufferLimit = _buffer + numProcessedBytes; + _wasFinished = (numProcessedBytes == 0); + return (!_wasFinished); +} + +Byte CInBuffer::ReadBlock2() +{ + if(!ReadBlock()) + return 0xFF; + return *_buffer++; +} diff --git a/lzma443/C/7zip/Common/InBuffer.h b/lzma443/C/7zip/Common/InBuffer.h new file mode 100755 index 0000000..057caa1 --- /dev/null +++ b/lzma443/C/7zip/Common/InBuffer.h @@ -0,0 +1,76 @@ +// InBuffer.h + +#ifndef __INBUFFER_H +#define __INBUFFER_H + +#include "../IStream.h" +#include "../../Common/MyCom.h" + +#ifndef _NO_EXCEPTIONS +class CInBufferException +{ +public: + HRESULT ErrorCode; + CInBufferException(HRESULT errorCode): ErrorCode(errorCode) {} +}; +#endif + +class CInBuffer +{ + Byte *_buffer; + Byte *_bufferLimit; + Byte *_bufferBase; + CMyComPtr _stream; + UInt64 _processedSize; + UInt32 _bufferSize; + bool _wasFinished; + + bool ReadBlock(); + Byte ReadBlock2(); + +public: + #ifdef _NO_EXCEPTIONS + HRESULT ErrorCode; + #endif + + CInBuffer(); + ~CInBuffer() { Free(); } + + bool Create(UInt32 bufferSize); + void Free(); + + void SetStream(ISequentialInStream *stream); + void Init(); + void ReleaseStream() { _stream.Release(); } + + bool ReadByte(Byte &b) + { + if(_buffer >= _bufferLimit) + if(!ReadBlock()) + return false; + b = *_buffer++; + return true; + } + Byte ReadByte() + { + if(_buffer >= _bufferLimit) + return ReadBlock2(); + return *_buffer++; + } + void ReadBytes(void *data, UInt32 size, UInt32 &processedSize) + { + for(processedSize = 0; processedSize < size; processedSize++) + if (!ReadByte(((Byte *)data)[processedSize])) + return; + } + bool ReadBytes(void *data, UInt32 size) + { + UInt32 processedSize; + ReadBytes(data, size, processedSize); + return (processedSize == size); + } + UInt64 GetProcessedSize() const { return _processedSize + (_buffer - _bufferBase); } + bool WasFinished() const { return _wasFinished; } +}; + +#endif diff --git a/lzma443/C/7zip/Common/OutBuffer.cpp b/lzma443/C/7zip/Common/OutBuffer.cpp new file mode 100755 index 0000000..a73fa7c --- /dev/null +++ b/lzma443/C/7zip/Common/OutBuffer.cpp @@ -0,0 +1,116 @@ +// OutByte.cpp + +#include "StdAfx.h" + +#include "OutBuffer.h" + +#include "../../Common/Alloc.h" + +bool COutBuffer::Create(UInt32 bufferSize) +{ + const UInt32 kMinBlockSize = 1; + if (bufferSize < kMinBlockSize) + bufferSize = kMinBlockSize; + if (_buffer != 0 && _bufferSize == bufferSize) + return true; + Free(); + _bufferSize = bufferSize; + _buffer = (Byte *)::MidAlloc(bufferSize); + return (_buffer != 0); +} + +void COutBuffer::Free() +{ + ::MidFree(_buffer); + _buffer = 0; +} + +void COutBuffer::SetStream(ISequentialOutStream *stream) +{ + _stream = stream; +} + +void COutBuffer::Init() +{ + _streamPos = 0; + _limitPos = _bufferSize; + _pos = 0; + _processedSize = 0; + _overDict = false; + #ifdef _NO_EXCEPTIONS + ErrorCode = S_OK; + #endif +} + +UInt64 COutBuffer::GetProcessedSize() const +{ + UInt64 res = _processedSize + _pos - _streamPos; + if (_streamPos > _pos) + res += _bufferSize; + return res; +} + + +HRESULT COutBuffer::FlushPart() +{ + // _streamPos < _bufferSize + UInt32 size = (_streamPos >= _pos) ? (_bufferSize - _streamPos) : (_pos - _streamPos); + HRESULT result = S_OK; + #ifdef _NO_EXCEPTIONS + result = ErrorCode; + #endif + if (_buffer2 != 0) + { + memmove(_buffer2, _buffer + _streamPos, size); + _buffer2 += size; + } + + if (_stream != 0 + #ifdef _NO_EXCEPTIONS + && (ErrorCode == S_OK) + #endif + ) + { + UInt32 processedSize = 0; + result = _stream->Write(_buffer + _streamPos, size, &processedSize); + size = processedSize; + } + _streamPos += size; + if (_streamPos == _bufferSize) + _streamPos = 0; + if (_pos == _bufferSize) + { + _overDict = true; + _pos = 0; + } + _limitPos = (_streamPos > _pos) ? _streamPos : _bufferSize; + _processedSize += size; + return result; +} + +HRESULT COutBuffer::Flush() +{ + #ifdef _NO_EXCEPTIONS + if (ErrorCode != S_OK) + return ErrorCode; + #endif + + while(_streamPos != _pos) + { + HRESULT result = FlushPart(); + if (result != S_OK) + return result; + } + return S_OK; +} + +void COutBuffer::FlushWithCheck() +{ + HRESULT result = FlushPart(); + #ifdef _NO_EXCEPTIONS + ErrorCode = result; + #else + if (result != S_OK) + throw COutBufferException(result); + #endif +} diff --git a/lzma443/C/7zip/Common/OutBuffer.h b/lzma443/C/7zip/Common/OutBuffer.h new file mode 100755 index 0000000..0ce54e2 --- /dev/null +++ b/lzma443/C/7zip/Common/OutBuffer.h @@ -0,0 +1,64 @@ +// OutBuffer.h + +#ifndef __OUTBUFFER_H +#define __OUTBUFFER_H + +#include "../IStream.h" +#include "../../Common/MyCom.h" + +#ifndef _NO_EXCEPTIONS +struct COutBufferException +{ + HRESULT ErrorCode; + COutBufferException(HRESULT errorCode): ErrorCode(errorCode) {} +}; +#endif + +class COutBuffer +{ +protected: + Byte *_buffer; + UInt32 _pos; + UInt32 _limitPos; + UInt32 _streamPos; + UInt32 _bufferSize; + CMyComPtr _stream; + UInt64 _processedSize; + Byte *_buffer2; + bool _overDict; + + HRESULT FlushPart(); + void FlushWithCheck(); +public: + #ifdef _NO_EXCEPTIONS + HRESULT ErrorCode; + #endif + + COutBuffer(): _buffer(0), _pos(0), _stream(0), _buffer2(0) {} + ~COutBuffer() { Free(); } + + bool Create(UInt32 bufferSize); + void Free(); + + void SetMemStream(Byte *buffer) { _buffer2 = buffer; } + void SetStream(ISequentialOutStream *stream); + void Init(); + HRESULT Flush(); + void ReleaseStream() { _stream.Release(); } + + void WriteByte(Byte b) + { + _buffer[_pos++] = b; + if(_pos == _limitPos) + FlushWithCheck(); + } + void WriteBytes(const void *data, size_t size) + { + for (size_t i = 0; i < size; i++) + WriteByte(((const Byte *)data)[i]); + } + + UInt64 GetProcessedSize() const; +}; + +#endif diff --git a/lzma443/C/7zip/Common/StdAfx.h b/lzma443/C/7zip/Common/StdAfx.h new file mode 100755 index 0000000..27a77b1 --- /dev/null +++ b/lzma443/C/7zip/Common/StdAfx.h @@ -0,0 +1,9 @@ +// StdAfx.h + +#ifndef __STDAFX_H +#define __STDAFX_H + +#include "../../Common/MyWindows.h" +#include "../../Common/NewHandler.h" + +#endif diff --git a/lzma443/C/7zip/Common/StreamUtils.cpp b/lzma443/C/7zip/Common/StreamUtils.cpp new file mode 100755 index 0000000..a5d9ac0 --- /dev/null +++ b/lzma443/C/7zip/Common/StreamUtils.cpp @@ -0,0 +1,44 @@ +// StreamUtils.cpp + +#include "StdAfx.h" + +#include "../../Common/MyCom.h" +#include "StreamUtils.h" + +HRESULT ReadStream(ISequentialInStream *stream, void *data, UInt32 size, UInt32 *processedSize) +{ + if (processedSize != 0) + *processedSize = 0; + while(size != 0) + { + UInt32 processedSizeLoc; + HRESULT res = stream->Read(data, size, &processedSizeLoc); + if (processedSize != 0) + *processedSize += processedSizeLoc; + data = (Byte *)((Byte *)data + processedSizeLoc); + size -= processedSizeLoc; + RINOK(res); + if (processedSizeLoc == 0) + return S_OK; + } + return S_OK; +} + +HRESULT WriteStream(ISequentialOutStream *stream, const void *data, UInt32 size, UInt32 *processedSize) +{ + if (processedSize != 0) + *processedSize = 0; + while(size != 0) + { + UInt32 processedSizeLoc; + HRESULT res = stream->Write(data, size, &processedSizeLoc); + if (processedSize != 0) + *processedSize += processedSizeLoc; + data = (const void *)((const Byte *)data + processedSizeLoc); + size -= processedSizeLoc; + RINOK(res); + if (processedSizeLoc == 0) + break; + } + return S_OK; +} diff --git a/lzma443/C/7zip/Common/StreamUtils.h b/lzma443/C/7zip/Common/StreamUtils.h new file mode 100755 index 0000000..59f8873 --- /dev/null +++ b/lzma443/C/7zip/Common/StreamUtils.h @@ -0,0 +1,11 @@ +// StreamUtils.h + +#ifndef __STREAMUTILS_H +#define __STREAMUTILS_H + +#include "../IStream.h" + +HRESULT ReadStream(ISequentialInStream *stream, void *data, UInt32 size, UInt32 *processedSize); +HRESULT WriteStream(ISequentialOutStream *stream, const void *data, UInt32 size, UInt32 *processedSize); + +#endif diff --git a/lzma443/C/7zip/Compress/Branch/ARM.cpp b/lzma443/C/7zip/Compress/Branch/ARM.cpp new file mode 100755 index 0000000..4bd5e18 --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/ARM.cpp @@ -0,0 +1,16 @@ +// ARM.cpp + +#include "StdAfx.h" +#include "ARM.h" + +#include "BranchARM.c" + +UInt32 CBC_ARM_Encoder::SubFilter(Byte *data, UInt32 size) +{ + return ::ARM_Convert(data, size, _bufferPos, 1); +} + +UInt32 CBC_ARM_Decoder::SubFilter(Byte *data, UInt32 size) +{ + return ::ARM_Convert(data, size, _bufferPos, 0); +} diff --git a/lzma443/C/7zip/Compress/Branch/ARM.h b/lzma443/C/7zip/Compress/Branch/ARM.h new file mode 100755 index 0000000..5561299 --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/ARM.h @@ -0,0 +1,10 @@ +// ARM.h + +#ifndef __ARM_H +#define __ARM_H + +#include "BranchCoder.h" + +MyClassA(BC_ARM, 0x05, 1) + +#endif diff --git a/lzma443/C/7zip/Compress/Branch/ARMThumb.cpp b/lzma443/C/7zip/Compress/Branch/ARMThumb.cpp new file mode 100755 index 0000000..fbd2570 --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/ARMThumb.cpp @@ -0,0 +1,16 @@ +// ARMThumb.cpp + +#include "StdAfx.h" +#include "ARMThumb.h" + +#include "BranchARMThumb.c" + +UInt32 CBC_ARMThumb_Encoder::SubFilter(Byte *data, UInt32 size) +{ + return ::ARMThumb_Convert(data, size, _bufferPos, 1); +} + +UInt32 CBC_ARMThumb_Decoder::SubFilter(Byte *data, UInt32 size) +{ + return ::ARMThumb_Convert(data, size, _bufferPos, 0); +} diff --git a/lzma443/C/7zip/Compress/Branch/ARMThumb.h b/lzma443/C/7zip/Compress/Branch/ARMThumb.h new file mode 100755 index 0000000..601e40b --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/ARMThumb.h @@ -0,0 +1,10 @@ +// ARMThumb.h + +#ifndef __ARMTHUMB_H +#define __ARMTHUMB_H + +#include "BranchCoder.h" + +MyClassA(BC_ARMThumb, 0x07, 1) + +#endif diff --git a/lzma443/C/7zip/Compress/Branch/BranchARM.c b/lzma443/C/7zip/Compress/Branch/BranchARM.c new file mode 100755 index 0000000..04bd81e --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/BranchARM.c @@ -0,0 +1,26 @@ +/* BranchARM.c */ + +#include "BranchARM.h" + +UInt32 ARM_Convert(Byte *data, UInt32 size, UInt32 nowPos, int encoding) +{ + UInt32 i; + for (i = 0; i + 4 <= size; i += 4) + { + if (data[i + 3] == 0xEB) + { + UInt32 src = (data[i + 2] << 16) | (data[i + 1] << 8) | (data[i + 0]); + src <<= 2; + UInt32 dest; + if (encoding) + dest = nowPos + i + 8 + src; + else + dest = src - (nowPos + i + 8); + dest >>= 2; + data[i + 2] = (dest >> 16); + data[i + 1] = (dest >> 8); + data[i + 0] = dest; + } + } + return i; +} diff --git a/lzma443/C/7zip/Compress/Branch/BranchARM.h b/lzma443/C/7zip/Compress/Branch/BranchARM.h new file mode 100755 index 0000000..02eb1b4 --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/BranchARM.h @@ -0,0 +1,10 @@ +// BranchARM.h + +#ifndef __BRANCH_ARM_H +#define __BRANCH_ARM_H + +#include "BranchTypes.h" + +UInt32 ARM_Convert(Byte *data, UInt32 size, UInt32 nowPos, int encoding); + +#endif diff --git a/lzma443/C/7zip/Compress/Branch/BranchARMThumb.c b/lzma443/C/7zip/Compress/Branch/BranchARMThumb.c new file mode 100755 index 0000000..4e4b04b --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/BranchARMThumb.c @@ -0,0 +1,35 @@ +/* BranchARMThumb.c */ + +#include "BranchARMThumb.h" + +UInt32 ARMThumb_Convert(Byte *data, UInt32 size, UInt32 nowPos, int encoding) +{ + UInt32 i; + for (i = 0; i + 4 <= size; i += 2) + { + if ((data[i + 1] & 0xF8) == 0xF0 && + (data[i + 3] & 0xF8) == 0xF8) + { + UInt32 src = + ((data[i + 1] & 0x7) << 19) | + (data[i + 0] << 11) | + ((data[i + 3] & 0x7) << 8) | + (data[i + 2]); + + src <<= 1; + UInt32 dest; + if (encoding) + dest = nowPos + i + 4 + src; + else + dest = src - (nowPos + i + 4); + dest >>= 1; + + data[i + 1] = 0xF0 | ((dest >> 19) & 0x7); + data[i + 0] = (dest >> 11); + data[i + 3] = 0xF8 | ((dest >> 8) & 0x7); + data[i + 2] = (dest); + i += 2; + } + } + return i; +} diff --git a/lzma443/C/7zip/Compress/Branch/BranchARMThumb.h b/lzma443/C/7zip/Compress/Branch/BranchARMThumb.h new file mode 100755 index 0000000..d67e6e5 --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/BranchARMThumb.h @@ -0,0 +1,10 @@ +// BranchARMThumb.h + +#ifndef __BRANCH_ARM_THUMB_H +#define __BRANCH_ARM_THUMB_H + +#include "BranchTypes.h" + +UInt32 ARMThumb_Convert(Byte *data, UInt32 size, UInt32 nowPos, int encoding); + +#endif diff --git a/lzma443/C/7zip/Compress/Branch/BranchCoder.cpp b/lzma443/C/7zip/Compress/Branch/BranchCoder.cpp new file mode 100755 index 0000000..8d25f0d --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/BranchCoder.cpp @@ -0,0 +1,18 @@ +// BranchCoder.cpp + +#include "StdAfx.h" +#include "BranchCoder.h" + +STDMETHODIMP CBranchConverter::Init() +{ + _bufferPos = 0; + SubInit(); + return S_OK; +} + +STDMETHODIMP_(UInt32) CBranchConverter::Filter(Byte *data, UInt32 size) +{ + UInt32 processedSize = SubFilter(data, size); + _bufferPos += processedSize; + return processedSize; +} diff --git a/lzma443/C/7zip/Compress/Branch/BranchCoder.h b/lzma443/C/7zip/Compress/Branch/BranchCoder.h new file mode 100755 index 0000000..4b53b6c --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/BranchCoder.h @@ -0,0 +1,54 @@ +// BranchCoder.h + +#ifndef __BRANCH_CODER_H +#define __BRANCH_CODER_H + +#include "Common/MyCom.h" +#include "Common/Types.h" +#include "Common/Alloc.h" + +#include "../../ICoder.h" + +class CBranchConverter: + public ICompressFilter, + public CMyUnknownImp +{ +protected: + UInt32 _bufferPos; + virtual void SubInit() {} + virtual UInt32 SubFilter(Byte *data, UInt32 size) = 0; +public: + MY_UNKNOWN_IMP; + STDMETHOD(Init)(); + STDMETHOD_(UInt32, Filter)(Byte *data, UInt32 size); +}; + +#define MyClassEncoderA(Name) class C ## Name: public CBranchConverter \ + { public: UInt32 SubFilter(Byte *data, UInt32 size); }; + +#define MyClassDecoderA(Name) class C ## Name: public CBranchConverter \ + { public: UInt32 SubFilter(Byte *data, UInt32 size); }; + +#define MyClassEncoderB(Name, ADD_ITEMS, ADD_INIT) class C ## Name: public CBranchConverter, public ADD_ITEMS \ + { public: UInt32 SubFilter(Byte *data, UInt32 size); ADD_INIT}; + +#define MyClassDecoderB(Name, ADD_ITEMS, ADD_INIT) class C ## Name: public CBranchConverter, public ADD_ITEMS \ + { public: UInt32 SubFilter(Byte *data, UInt32 size); ADD_INIT}; + +#define MyClass2b(Name, id, subId, encodingId) \ +DEFINE_GUID(CLSID_CCompressConvert ## Name, \ +0x23170F69, 0x40C1, 0x278B, 0x03, 0x03, id, subId, 0x00, 0x00, encodingId, 0x00); + +#define MyClassA(Name, id, subId) \ +MyClass2b(Name ## _Encoder, id, subId, 0x01) \ +MyClassEncoderA(Name ## _Encoder) \ +MyClass2b(Name ## _Decoder, id, subId, 0x00) \ +MyClassDecoderA(Name ## _Decoder) + +#define MyClassB(Name, id, subId, ADD_ITEMS, ADD_INIT) \ +MyClass2b(Name ## _Encoder, id, subId, 0x01) \ +MyClassEncoderB(Name ## _Encoder, ADD_ITEMS, ADD_INIT) \ +MyClass2b(Name ## _Decoder, id, subId, 0x00) \ +MyClassDecoderB(Name ## _Decoder, ADD_ITEMS, ADD_INIT) + +#endif diff --git a/lzma443/C/7zip/Compress/Branch/BranchIA64.c b/lzma443/C/7zip/Compress/Branch/BranchIA64.c new file mode 100755 index 0000000..e777556 --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/BranchIA64.c @@ -0,0 +1,63 @@ +/* BranchIA64.c */ + +#include "BranchIA64.h" + +const Byte kBranchTable[32] = +{ + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 4, 4, 6, 6, 0, 0, 7, 7, + 4, 4, 0, 0, 4, 4, 0, 0 +}; + +UInt32 IA64_Convert(Byte *data, UInt32 size, UInt32 nowPos, int encoding) +{ + UInt32 i; + for (i = 0; i + 16 <= size; i += 16) + { + UInt32 instrTemplate = data[i] & 0x1F; + UInt32 mask = kBranchTable[instrTemplate]; + UInt32 bitPos = 5; + for (int slot = 0; slot < 3; slot++, bitPos += 41) + { + if (((mask >> slot) & 1) == 0) + continue; + UInt32 bytePos = (bitPos >> 3); + UInt32 bitRes = bitPos & 0x7; + UInt64 instruction = 0; + int j; + for (j = 0; j < 6; j++) + instruction += (UInt64)(data[i + j + bytePos]) << (8 * j); + + UInt64 instNorm = instruction >> bitRes; + if (((instNorm >> 37) & 0xF) == 0x5 + && ((instNorm >> 9) & 0x7) == 0 + /* && (instNorm & 0x3F)== 0 */ + ) + { + UInt32 src = UInt32((instNorm >> 13) & 0xFFFFF); + src |= ((instNorm >> 36) & 1) << 20; + + src <<= 4; + + UInt32 dest; + if (encoding) + dest = nowPos + i + src; + else + dest = src - (nowPos + i); + + dest >>= 4; + + instNorm &= ~(UInt64(0x8FFFFF) << 13); + instNorm |= (UInt64(dest & 0xFFFFF) << 13); + instNorm |= (UInt64(dest & 0x100000) << (36 - 20)); + + instruction &= (1 << bitRes) - 1; + instruction |= (instNorm << bitRes); + for (j = 0; j < 6; j++) + data[i + j + bytePos] = Byte(instruction >> (8 * j)); + } + } + } + return i; +} diff --git a/lzma443/C/7zip/Compress/Branch/BranchIA64.h b/lzma443/C/7zip/Compress/Branch/BranchIA64.h new file mode 100755 index 0000000..b7757fe --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/BranchIA64.h @@ -0,0 +1,10 @@ +// BranchIA64.h + +#ifndef __BRANCH_IA64_H +#define __BRANCH_IA64_H + +#include "BranchTypes.h" + +UInt32 IA64_Convert(Byte *data, UInt32 size, UInt32 nowPos, int encoding); + +#endif diff --git a/lzma443/C/7zip/Compress/Branch/BranchPPC.c b/lzma443/C/7zip/Compress/Branch/BranchPPC.c new file mode 100755 index 0000000..a173eb1 --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/BranchPPC.c @@ -0,0 +1,36 @@ +/* BranchPPC.c */ + +#include "BranchPPC.h" + +UInt32 PPC_B_Convert(Byte *data, UInt32 size, UInt32 nowPos, int encoding) +{ + UInt32 i; + for (i = 0; i + 4 <= size; i += 4) + { + /* PowerPC branch 6(48) 24(Offset) 1(Abs) 1(Link) */ + if ((data[i] >> 2) == 0x12 && + ( + (data[i + 3] & 3) == 1 + /* || (data[i+3] & 3) == 3 */ + ) + ) + { + UInt32 src = ((data[i + 0] & 3) << 24) | + (data[i + 1] << 16) | + (data[i + 2] << 8) | + (data[i + 3] & (~3)); + + UInt32 dest; + if (encoding) + dest = nowPos + i + src; + else + dest = src - (nowPos + i); + data[i + 0] = 0x48 | ((dest >> 24) & 0x3); + data[i + 1] = (dest >> 16); + data[i + 2] = (dest >> 8); + data[i + 3] &= 0x3; + data[i + 3] |= dest; + } + } + return i; +} diff --git a/lzma443/C/7zip/Compress/Branch/BranchPPC.h b/lzma443/C/7zip/Compress/Branch/BranchPPC.h new file mode 100755 index 0000000..c02bba1 --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/BranchPPC.h @@ -0,0 +1,10 @@ +// BranchPPC.h + +#ifndef __BRANCH_PPC_H +#define __BRANCH_PPC_H + +#include "BranchTypes.h" + +UInt32 PPC_B_Convert(Byte *data, UInt32 size, UInt32 nowPos, int encoding); + +#endif diff --git a/lzma443/C/7zip/Compress/Branch/BranchSPARC.c b/lzma443/C/7zip/Compress/Branch/BranchSPARC.c new file mode 100755 index 0000000..c175875 --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/BranchSPARC.c @@ -0,0 +1,36 @@ +/* BranchSPARC.c */ + +#include "BranchSPARC.h" + +UInt32 SPARC_Convert(Byte *data, UInt32 size, UInt32 nowPos, int encoding) +{ + UInt32 i; + for (i = 0; i + 4 <= size; i += 4) + { + if (data[i] == 0x40 && (data[i + 1] & 0xC0) == 0x00 || + data[i] == 0x7F && (data[i + 1] & 0xC0) == 0xC0) + { + UInt32 src = + ((UInt32)data[i + 0] << 24) | + ((UInt32)data[i + 1] << 16) | + ((UInt32)data[i + 2] << 8) | + ((UInt32)data[i + 3]); + + src <<= 2; + UInt32 dest; + if (encoding) + dest = nowPos + i + src; + else + dest = src - (nowPos + i); + dest >>= 2; + + dest = (((0 - ((dest >> 22) & 1)) << 22) & 0x3FFFFFFF) | (dest & 0x3FFFFF) | 0x40000000; + + data[i + 0] = (Byte)(dest >> 24); + data[i + 1] = (Byte)(dest >> 16); + data[i + 2] = (Byte)(dest >> 8); + data[i + 3] = (Byte)dest; + } + } + return i; +} diff --git a/lzma443/C/7zip/Compress/Branch/BranchSPARC.h b/lzma443/C/7zip/Compress/Branch/BranchSPARC.h new file mode 100755 index 0000000..fbe9e67 --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/BranchSPARC.h @@ -0,0 +1,10 @@ +// BranchSPARC.h + +#ifndef __BRANCH_SPARC_H +#define __BRANCH_SPARC_H + +#include "BranchTypes.h" + +UInt32 SPARC_B_Convert(Byte *data, UInt32 size, UInt32 nowPos, int encoding); + +#endif diff --git a/lzma443/C/7zip/Compress/Branch/BranchTypes.h b/lzma443/C/7zip/Compress/Branch/BranchTypes.h new file mode 100755 index 0000000..f7ad3ab --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/BranchTypes.h @@ -0,0 +1,25 @@ +/* BranchTypes.h */ + +#ifndef __BRANCHTYPES_H +#define __BRANCHTYPES_H + +#ifndef _7ZIP_BYTE_DEFINED +#define _7ZIP_BYTE_DEFINED +typedef unsigned char Byte; +#endif + +#ifndef _7ZIP_UINT16_DEFINED +#define _7ZIP_UINT16_DEFINED +typedef unsigned short UInt16; +#endif + +#ifndef _7ZIP_UINT32_DEFINED +#define _7ZIP_UINT32_DEFINED +#ifdef _LZMA_UINT32_IS_ULONG +typedef unsigned long UInt32; +#else +typedef unsigned int UInt32; +#endif +#endif + +#endif diff --git a/lzma443/C/7zip/Compress/Branch/BranchX86.c b/lzma443/C/7zip/Compress/Branch/BranchX86.c new file mode 100755 index 0000000..2c6f69a --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/BranchX86.c @@ -0,0 +1,101 @@ +/* BranchX86.c */ + +#include "BranchX86.h" + +/* +static int inline Test86MSByte(Byte b) +{ + return (b == 0 || b == 0xFF); +} +*/ +#define Test86MSByte(b) ((b) == 0 || (b) == 0xFF) + +const int kMaskToAllowedStatus[8] = {1, 1, 1, 0, 1, 0, 0, 0}; +const Byte kMaskToBitNumber[8] = {0, 1, 2, 2, 3, 3, 3, 3}; + +/* +void x86_Convert_Init(UInt32 *prevMask, UInt32 *prevPos) +{ + *prevMask = 0; + *prevPos = (UInt32)(-5); +} +*/ + +UInt32 x86_Convert(Byte *buffer, UInt32 endPos, UInt32 nowPos, + UInt32 *prevMask, UInt32 *prevPos, int encoding) +{ + UInt32 bufferPos = 0; + UInt32 limit; + + if (endPos < 5) + return 0; + + if (nowPos - *prevPos > 5) + *prevPos = nowPos - 5; + + limit = endPos - 5; + while(bufferPos <= limit) + { + Byte b = buffer[bufferPos]; + UInt32 offset; + if (b != 0xE8 && b != 0xE9) + { + bufferPos++; + continue; + } + offset = (nowPos + bufferPos - *prevPos); + *prevPos = (nowPos + bufferPos); + if (offset > 5) + *prevMask = 0; + else + { + UInt32 i; + for (i = 0; i < offset; i++) + { + *prevMask &= 0x77; + *prevMask <<= 1; + } + } + b = buffer[bufferPos + 4]; + if (Test86MSByte(b) && kMaskToAllowedStatus[(*prevMask >> 1) & 0x7] && + (*prevMask >> 1) < 0x10) + { + UInt32 src = + ((UInt32)(b) << 24) | + ((UInt32)(buffer[bufferPos + 3]) << 16) | + ((UInt32)(buffer[bufferPos + 2]) << 8) | + (buffer[bufferPos + 1]); + + UInt32 dest; + while(1) + { + UInt32 index; + if (encoding) + dest = (nowPos + bufferPos + 5) + src; + else + dest = src - (nowPos + bufferPos + 5); + if (*prevMask == 0) + break; + index = kMaskToBitNumber[*prevMask >> 1]; + b = (Byte)(dest >> (24 - index * 8)); + if (!Test86MSByte(b)) + break; + src = dest ^ ((1 << (32 - index * 8)) - 1); + } + buffer[bufferPos + 4] = (Byte)(~(((dest >> 24) & 1) - 1)); + buffer[bufferPos + 3] = (Byte)(dest >> 16); + buffer[bufferPos + 2] = (Byte)(dest >> 8); + buffer[bufferPos + 1] = (Byte)dest; + bufferPos += 5; + *prevMask = 0; + } + else + { + bufferPos++; + *prevMask |= 1; + if (Test86MSByte(b)) + *prevMask |= 0x10; + } + } + return bufferPos; +} diff --git a/lzma443/C/7zip/Compress/Branch/BranchX86.h b/lzma443/C/7zip/Compress/Branch/BranchX86.h new file mode 100755 index 0000000..25c1ae5 --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/BranchX86.h @@ -0,0 +1,13 @@ +/* BranchX86.h */ + +#ifndef __BRANCHX86_H +#define __BRANCHX86_H + +#include "BranchTypes.h" + +#define x86_Convert_Init(prevMask, prevPos) { prevMask = 0; prevPos = (UInt32)(-5); } + +UInt32 x86_Convert(Byte *buffer, UInt32 endPos, UInt32 nowPos, + UInt32 *prevMask, UInt32 *prevPos, int encoding); + +#endif diff --git a/lzma443/C/7zip/Compress/Branch/IA64.cpp b/lzma443/C/7zip/Compress/Branch/IA64.cpp new file mode 100755 index 0000000..75dfdcb --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/IA64.cpp @@ -0,0 +1,16 @@ +// IA64.cpp + +#include "StdAfx.h" +#include "IA64.h" + +#include "BranchIA64.c" + +UInt32 CBC_IA64_Encoder::SubFilter(Byte *data, UInt32 size) +{ + return ::IA64_Convert(data, size, _bufferPos, 1); +} + +UInt32 CBC_IA64_Decoder::SubFilter(Byte *data, UInt32 size) +{ + return ::IA64_Convert(data, size, _bufferPos, 0); +} diff --git a/lzma443/C/7zip/Compress/Branch/IA64.h b/lzma443/C/7zip/Compress/Branch/IA64.h new file mode 100755 index 0000000..7fe715e --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/IA64.h @@ -0,0 +1,10 @@ +// IA64.h + +#ifndef __IA64_H +#define __IA64_H + +#include "BranchCoder.h" + +MyClassA(BC_IA64, 0x04, 1) + +#endif diff --git a/lzma443/C/7zip/Compress/Branch/PPC.cpp b/lzma443/C/7zip/Compress/Branch/PPC.cpp new file mode 100755 index 0000000..197a8e5 --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/PPC.cpp @@ -0,0 +1,17 @@ +// PPC.cpp + +#include "StdAfx.h" +#include "PPC.h" + +#include "Windows/Defs.h" +#include "BranchPPC.c" + +UInt32 CBC_PPC_B_Encoder::SubFilter(Byte *data, UInt32 size) +{ + return ::PPC_B_Convert(data, size, _bufferPos, 1); +} + +UInt32 CBC_PPC_B_Decoder::SubFilter(Byte *data, UInt32 size) +{ + return ::PPC_B_Convert(data, size, _bufferPos, 0); +} diff --git a/lzma443/C/7zip/Compress/Branch/PPC.h b/lzma443/C/7zip/Compress/Branch/PPC.h new file mode 100755 index 0000000..a0e3344 --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/PPC.h @@ -0,0 +1,10 @@ +// PPC.h + +#ifndef __PPC_H +#define __PPC_H + +#include "BranchCoder.h" + +MyClassA(BC_PPC_B, 0x02, 5) + +#endif diff --git a/lzma443/C/7zip/Compress/Branch/SPARC.cpp b/lzma443/C/7zip/Compress/Branch/SPARC.cpp new file mode 100755 index 0000000..d054eaa --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/SPARC.cpp @@ -0,0 +1,17 @@ +// SPARC.cpp + +#include "StdAfx.h" +#include "SPARC.h" + +#include "Windows/Defs.h" +#include "BranchSPARC.c" + +UInt32 CBC_SPARC_Encoder::SubFilter(Byte *data, UInt32 size) +{ + return ::SPARC_Convert(data, size, _bufferPos, 1); +} + +UInt32 CBC_SPARC_Decoder::SubFilter(Byte *data, UInt32 size) +{ + return ::SPARC_Convert(data, size, _bufferPos, 0); +} diff --git a/lzma443/C/7zip/Compress/Branch/SPARC.h b/lzma443/C/7zip/Compress/Branch/SPARC.h new file mode 100755 index 0000000..e0a682e --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/SPARC.h @@ -0,0 +1,10 @@ +// SPARC.h + +#ifndef __SPARC_H +#define __SPARC_H + +#include "BranchCoder.h" + +MyClassA(BC_SPARC, 0x08, 5) + +#endif diff --git a/lzma443/C/7zip/Compress/Branch/StdAfx.h b/lzma443/C/7zip/Compress/Branch/StdAfx.h new file mode 100755 index 0000000..e7fb698 --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/StdAfx.h @@ -0,0 +1,8 @@ +// StdAfx.h + +#ifndef __STDAFX_H +#define __STDAFX_H + +#include "../../../Common/MyWindows.h" + +#endif diff --git a/lzma443/C/7zip/Compress/Branch/x86.cpp b/lzma443/C/7zip/Compress/Branch/x86.cpp new file mode 100755 index 0000000..013e42b --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/x86.cpp @@ -0,0 +1,18 @@ +// x86.cpp + +#include "StdAfx.h" +#include "x86.h" + +#include "Windows/Defs.h" + +#include "BranchX86.c" + +UInt32 CBCJ_x86_Encoder::SubFilter(Byte *data, UInt32 size) +{ + return ::x86_Convert(data, size, _bufferPos, &_prevMask, &_prevPos, 1); +} + +UInt32 CBCJ_x86_Decoder::SubFilter(Byte *data, UInt32 size) +{ + return ::x86_Convert(data, size, _bufferPos, &_prevMask, &_prevPos, 0); +} diff --git a/lzma443/C/7zip/Compress/Branch/x86.h b/lzma443/C/7zip/Compress/Branch/x86.h new file mode 100755 index 0000000..10d0398 --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/x86.h @@ -0,0 +1,19 @@ +// x86.h + +#ifndef __X86_H +#define __X86_H + +#include "BranchCoder.h" +#include "BranchX86.h" + +struct CBranch86 +{ + UInt32 _prevMask; + UInt32 _prevPos; + void x86Init() { x86_Convert_Init(_prevMask, _prevPos); } +}; + +MyClassB(BCJ_x86, 0x01, 3, CBranch86 , + virtual void SubInit() { x86Init(); }) + +#endif diff --git a/lzma443/C/7zip/Compress/Branch/x86_2.cpp b/lzma443/C/7zip/Compress/Branch/x86_2.cpp new file mode 100755 index 0000000..8f7bff2 --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/x86_2.cpp @@ -0,0 +1,412 @@ +// x86_2.cpp + +#include "StdAfx.h" +#include "x86_2.h" + +#include "../../../Common/Alloc.h" + +static const int kBufferSize = 1 << 17; + +inline bool IsJcc(Byte b0, Byte b1) +{ + return (b0 == 0x0F && (b1 & 0xF0) == 0x80); +} + +#ifndef EXTRACT_ONLY + +static bool inline Test86MSByte(Byte b) +{ + return (b == 0 || b == 0xFF); +} + +bool CBCJ2_x86_Encoder::Create() +{ + if (!_mainStream.Create(1 << 16)) + return false; + if (!_callStream.Create(1 << 20)) + return false; + if (!_jumpStream.Create(1 << 20)) + return false; + if (!_rangeEncoder.Create(1 << 20)) + return false; + if (_buffer == 0) + { + _buffer = (Byte *)MidAlloc(kBufferSize); + if (_buffer == 0) + return false; + } + return true; +} + +CBCJ2_x86_Encoder::~CBCJ2_x86_Encoder() +{ + ::MidFree(_buffer); +} + +HRESULT CBCJ2_x86_Encoder::Flush() +{ + RINOK(_mainStream.Flush()); + RINOK(_callStream.Flush()); + RINOK(_jumpStream.Flush()); + _rangeEncoder.FlushData(); + return _rangeEncoder.FlushStream(); +} + +const UInt32 kDefaultLimit = (1 << 24); + +HRESULT CBCJ2_x86_Encoder::CodeReal(ISequentialInStream **inStreams, + const UInt64 **inSizes, + UInt32 numInStreams, + ISequentialOutStream **outStreams, + const UInt64 **outSizes, + UInt32 numOutStreams, + ICompressProgressInfo *progress) +{ + if (numInStreams != 1 || numOutStreams != 4) + return E_INVALIDARG; + + if (!Create()) + return E_OUTOFMEMORY; + + bool sizeIsDefined = false; + UInt64 inSize; + if (inSizes != NULL) + if (inSizes[0] != NULL) + { + inSize = *inSizes[0]; + if (inSize <= kDefaultLimit) + sizeIsDefined = true; + } + + ISequentialInStream *inStream = inStreams[0]; + + _mainStream.SetStream(outStreams[0]); + _mainStream.Init(); + _callStream.SetStream(outStreams[1]); + _callStream.Init(); + _jumpStream.SetStream(outStreams[2]); + _jumpStream.Init(); + _rangeEncoder.SetStream(outStreams[3]); + _rangeEncoder.Init(); + for (int i = 0; i < 256; i++) + _statusE8Encoder[i].Init(); + _statusE9Encoder.Init(); + _statusJccEncoder.Init(); + CCoderReleaser releaser(this); + + CMyComPtr getSubStreamSize; + { + inStream->QueryInterface(IID_ICompressGetSubStreamSize, (void **)&getSubStreamSize); + } + + UInt32 nowPos = 0; + UInt64 nowPos64 = 0; + UInt32 bufferPos = 0; + + Byte prevByte = 0; + + UInt64 subStreamIndex = 0; + UInt64 subStreamStartPos = 0; + UInt64 subStreamEndPos = 0; + + while(true) + { + UInt32 processedSize = 0; + while(true) + { + UInt32 size = kBufferSize - (bufferPos + processedSize); + UInt32 processedSizeLoc; + if (size == 0) + break; + RINOK(inStream->Read(_buffer + bufferPos + processedSize, size, &processedSizeLoc)); + if (processedSizeLoc == 0) + break; + processedSize += processedSizeLoc; + } + UInt32 endPos = bufferPos + processedSize; + + if (endPos < 5) + { + // change it + for (bufferPos = 0; bufferPos < endPos; bufferPos++) + { + Byte b = _buffer[bufferPos]; + _mainStream.WriteByte(b); + if (b == 0xE8) + _statusE8Encoder[prevByte].Encode(&_rangeEncoder, 0); + else if (b == 0xE9) + _statusE9Encoder.Encode(&_rangeEncoder, 0); + else if (IsJcc(prevByte, b)) + _statusJccEncoder.Encode(&_rangeEncoder, 0); + prevByte = b; + } + return Flush(); + } + + bufferPos = 0; + + UInt32 limit = endPos - 5; + while(bufferPos <= limit) + { + Byte b = _buffer[bufferPos]; + _mainStream.WriteByte(b); + if (b != 0xE8 && b != 0xE9 && !IsJcc(prevByte, b)) + { + bufferPos++; + prevByte = b; + continue; + } + Byte nextByte = _buffer[bufferPos + 4]; + UInt32 src = + (UInt32(nextByte) << 24) | + (UInt32(_buffer[bufferPos + 3]) << 16) | + (UInt32(_buffer[bufferPos + 2]) << 8) | + (_buffer[bufferPos + 1]); + UInt32 dest = (nowPos + bufferPos + 5) + src; + // if (Test86MSByte(nextByte)) + bool convert; + if (getSubStreamSize != NULL) + { + UInt64 currentPos = (nowPos64 + bufferPos); + while (subStreamEndPos < currentPos) + { + UInt64 subStreamSize; + HRESULT result = getSubStreamSize->GetSubStreamSize(subStreamIndex, &subStreamSize); + if (result == S_OK) + { + subStreamStartPos = subStreamEndPos; + subStreamEndPos += subStreamSize; + subStreamIndex++; + } + else if (result == S_FALSE || result == E_NOTIMPL) + { + getSubStreamSize.Release(); + subStreamStartPos = 0; + subStreamEndPos = subStreamStartPos - 1; + } + else + return result; + } + if (getSubStreamSize == NULL) + { + if (sizeIsDefined) + convert = (dest < inSize); + else + convert = Test86MSByte(nextByte); + } + else if (subStreamEndPos - subStreamStartPos > kDefaultLimit) + convert = Test86MSByte(nextByte); + else + { + UInt64 dest64 = (currentPos + 5) + Int64(Int32(src)); + convert = (dest64 >= subStreamStartPos && dest64 < subStreamEndPos); + } + } + else if (sizeIsDefined) + convert = (dest < inSize); + else + convert = Test86MSByte(nextByte); + if (convert) + { + if (b == 0xE8) + _statusE8Encoder[prevByte].Encode(&_rangeEncoder, 1); + else if (b == 0xE9) + _statusE9Encoder.Encode(&_rangeEncoder, 1); + else + _statusJccEncoder.Encode(&_rangeEncoder, 1); + + bufferPos += 5; + if (b == 0xE8) + { + _callStream.WriteByte((Byte)(dest >> 24)); + _callStream.WriteByte((Byte)(dest >> 16)); + _callStream.WriteByte((Byte)(dest >> 8)); + _callStream.WriteByte((Byte)(dest)); + } + else + { + _jumpStream.WriteByte((Byte)(dest >> 24)); + _jumpStream.WriteByte((Byte)(dest >> 16)); + _jumpStream.WriteByte((Byte)(dest >> 8)); + _jumpStream.WriteByte((Byte)(dest)); + } + prevByte = nextByte; + } + else + { + if (b == 0xE8) + _statusE8Encoder[prevByte].Encode(&_rangeEncoder, 0); + else if (b == 0xE9) + _statusE9Encoder.Encode(&_rangeEncoder, 0); + else + _statusJccEncoder.Encode(&_rangeEncoder, 0); + bufferPos++; + prevByte = b; + } + } + nowPos += bufferPos; + nowPos64 += bufferPos; + + if (progress != NULL) + { + RINOK(progress->SetRatioInfo(&nowPos64, NULL)); + } + + UInt32 i = 0; + while(bufferPos < endPos) + _buffer[i++] = _buffer[bufferPos++]; + bufferPos = i; + } +} + +STDMETHODIMP CBCJ2_x86_Encoder::Code(ISequentialInStream **inStreams, + const UInt64 **inSizes, + UInt32 numInStreams, + ISequentialOutStream **outStreams, + const UInt64 **outSizes, + UInt32 numOutStreams, + ICompressProgressInfo *progress) +{ + try + { + return CodeReal(inStreams, inSizes, numInStreams, + outStreams, outSizes,numOutStreams, progress); + } + catch(const COutBufferException &e) { return e.ErrorCode; } + catch(...) { return S_FALSE; } +} + +#endif + +HRESULT CBCJ2_x86_Decoder::CodeReal(ISequentialInStream **inStreams, + const UInt64 **inSizes, + UInt32 numInStreams, + ISequentialOutStream **outStreams, + const UInt64 **outSizes, + UInt32 numOutStreams, + ICompressProgressInfo *progress) +{ + if (numInStreams != 4 || numOutStreams != 1) + return E_INVALIDARG; + + if (!_mainInStream.Create(1 << 16)) + return E_OUTOFMEMORY; + if (!_callStream.Create(1 << 20)) + return E_OUTOFMEMORY; + if (!_jumpStream.Create(1 << 16)) + return E_OUTOFMEMORY; + if (!_rangeDecoder.Create(1 << 20)) + return E_OUTOFMEMORY; + if (!_outStream.Create(1 << 16)) + return E_OUTOFMEMORY; + + _mainInStream.SetStream(inStreams[0]); + _callStream.SetStream(inStreams[1]); + _jumpStream.SetStream(inStreams[2]); + _rangeDecoder.SetStream(inStreams[3]); + _outStream.SetStream(outStreams[0]); + + _mainInStream.Init(); + _callStream.Init(); + _jumpStream.Init(); + _rangeDecoder.Init(); + _outStream.Init(); + + for (int i = 0; i < 256; i++) + _statusE8Decoder[i].Init(); + _statusE9Decoder.Init(); + _statusJccDecoder.Init(); + + CCoderReleaser releaser(this); + + Byte prevByte = 0; + UInt32 processedBytes = 0; + while(true) + { + if (processedBytes > (1 << 20) && progress != NULL) + { + UInt64 nowPos64 = _outStream.GetProcessedSize(); + RINOK(progress->SetRatioInfo(NULL, &nowPos64)); + processedBytes = 0; + } + processedBytes++; + Byte b; + if (!_mainInStream.ReadByte(b)) + return Flush(); + _outStream.WriteByte(b); + if (b != 0xE8 && b != 0xE9 && !IsJcc(prevByte, b)) + { + prevByte = b; + continue; + } + bool status; + if (b == 0xE8) + status = (_statusE8Decoder[prevByte].Decode(&_rangeDecoder) == 1); + else if (b == 0xE9) + status = (_statusE9Decoder.Decode(&_rangeDecoder) == 1); + else + status = (_statusJccDecoder.Decode(&_rangeDecoder) == 1); + if (status) + { + UInt32 src; + if (b == 0xE8) + { + Byte b0; + if(!_callStream.ReadByte(b0)) + return S_FALSE; + src = ((UInt32)b0) << 24; + if(!_callStream.ReadByte(b0)) + return S_FALSE; + src |= ((UInt32)b0) << 16; + if(!_callStream.ReadByte(b0)) + return S_FALSE; + src |= ((UInt32)b0) << 8; + if(!_callStream.ReadByte(b0)) + return S_FALSE; + src |= ((UInt32)b0); + } + else + { + Byte b0; + if(!_jumpStream.ReadByte(b0)) + return S_FALSE; + src = ((UInt32)b0) << 24; + if(!_jumpStream.ReadByte(b0)) + return S_FALSE; + src |= ((UInt32)b0) << 16; + if(!_jumpStream.ReadByte(b0)) + return S_FALSE; + src |= ((UInt32)b0) << 8; + if(!_jumpStream.ReadByte(b0)) + return S_FALSE; + src |= ((UInt32)b0); + } + UInt32 dest = src - (UInt32(_outStream.GetProcessedSize()) + 4) ; + _outStream.WriteByte((Byte)(dest)); + _outStream.WriteByte((Byte)(dest >> 8)); + _outStream.WriteByte((Byte)(dest >> 16)); + _outStream.WriteByte((Byte)(dest >> 24)); + prevByte = (dest >> 24); + processedBytes += 4; + } + else + prevByte = b; + } +} + +STDMETHODIMP CBCJ2_x86_Decoder::Code(ISequentialInStream **inStreams, + const UInt64 **inSizes, + UInt32 numInStreams, + ISequentialOutStream **outStreams, + const UInt64 **outSizes, + UInt32 numOutStreams, + ICompressProgressInfo *progress) +{ + try + { + return CodeReal(inStreams, inSizes, numInStreams, + outStreams, outSizes,numOutStreams, progress); + } + catch(const COutBufferException &e) { return e.ErrorCode; } + catch(...) { return S_FALSE; } +} diff --git a/lzma443/C/7zip/Compress/Branch/x86_2.h b/lzma443/C/7zip/Compress/Branch/x86_2.h new file mode 100755 index 0000000..3d34eb8 --- /dev/null +++ b/lzma443/C/7zip/Compress/Branch/x86_2.h @@ -0,0 +1,133 @@ +// x86_2.h + +#ifndef __BRANCH_X86_2_H +#define __BRANCH_X86_2_H + +#include "../../../Common/MyCom.h" +#include "../RangeCoder/RangeCoderBit.h" +#include "../../ICoder.h" + +// {23170F69-40C1-278B-0303-010100000100} +#define MyClass2_a(Name, id, subId, encodingId) \ +DEFINE_GUID(CLSID_CCompressConvert ## Name, \ +0x23170F69, 0x40C1, 0x278B, 0x03, 0x03, id, subId, 0x00, 0x00, encodingId, 0x00); + +#define MyClass_a(Name, id, subId) \ +MyClass2_a(Name ## _Encoder, id, subId, 0x01) \ +MyClass2_a(Name ## _Decoder, id, subId, 0x00) + +MyClass_a(BCJ2_x86, 0x01, 0x1B) + +const int kNumMoveBits = 5; + +#ifndef EXTRACT_ONLY + +class CBCJ2_x86_Encoder: + public ICompressCoder2, + public CMyUnknownImp +{ + Byte *_buffer; +public: + CBCJ2_x86_Encoder(): _buffer(0) {}; + ~CBCJ2_x86_Encoder(); + bool Create(); + + COutBuffer _mainStream; + COutBuffer _callStream; + COutBuffer _jumpStream; + NCompress::NRangeCoder::CEncoder _rangeEncoder; + NCompress::NRangeCoder::CBitEncoder _statusE8Encoder[256]; + NCompress::NRangeCoder::CBitEncoder _statusE9Encoder; + NCompress::NRangeCoder::CBitEncoder _statusJccEncoder; + + HRESULT Flush(); + void ReleaseStreams() + { + _mainStream.ReleaseStream(); + _callStream.ReleaseStream(); + _jumpStream.ReleaseStream(); + _rangeEncoder.ReleaseStream(); + } + + class CCoderReleaser + { + CBCJ2_x86_Encoder *_coder; + public: + CCoderReleaser(CBCJ2_x86_Encoder *coder): _coder(coder) {} + ~CCoderReleaser() { _coder->ReleaseStreams(); } + }; + +public: + + MY_UNKNOWN_IMP + + HRESULT CodeReal(ISequentialInStream **inStreams, + const UInt64 **inSizes, + UInt32 numInStreams, + ISequentialOutStream **outStreams, + const UInt64 **outSizes, + UInt32 numOutStreams, + ICompressProgressInfo *progress); + STDMETHOD(Code)(ISequentialInStream **inStreams, + const UInt64 **inSizes, + UInt32 numInStreams, + ISequentialOutStream **outStreams, + const UInt64 **outSizes, + UInt32 numOutStreams, + ICompressProgressInfo *progress); +}; + +#endif + +class CBCJ2_x86_Decoder: + public ICompressCoder2, + public CMyUnknownImp +{ +public: + CInBuffer _mainInStream; + CInBuffer _callStream; + CInBuffer _jumpStream; + NCompress::NRangeCoder::CDecoder _rangeDecoder; + NCompress::NRangeCoder::CBitDecoder _statusE8Decoder[256]; + NCompress::NRangeCoder::CBitDecoder _statusE9Decoder; + NCompress::NRangeCoder::CBitDecoder _statusJccDecoder; + + COutBuffer _outStream; + + void ReleaseStreams() + { + _mainInStream.ReleaseStream(); + _callStream.ReleaseStream(); + _jumpStream.ReleaseStream(); + _rangeDecoder.ReleaseStream(); + _outStream.ReleaseStream(); + } + + HRESULT Flush() { return _outStream.Flush(); } + class CCoderReleaser + { + CBCJ2_x86_Decoder *_coder; + public: + CCoderReleaser(CBCJ2_x86_Decoder *coder): _coder(coder) {} + ~CCoderReleaser() { _coder->ReleaseStreams(); } + }; + +public: + MY_UNKNOWN_IMP + HRESULT CodeReal(ISequentialInStream **inStreams, + const UInt64 **inSizes, + UInt32 numInStreams, + ISequentialOutStream **outStreams, + const UInt64 **outSizes, + UInt32 numOutStreams, + ICompressProgressInfo *progress); + STDMETHOD(Code)(ISequentialInStream **inStreams, + const UInt64 **inSizes, + UInt32 numInStreams, + ISequentialOutStream **outStreams, + const UInt64 **outSizes, + UInt32 numOutStreams, + ICompressProgressInfo *progress); +}; + +#endif diff --git a/lzma443/C/7zip/Compress/LZ/BinTree/BinTree.h b/lzma443/C/7zip/Compress/LZ/BinTree/BinTree.h new file mode 100755 index 0000000..b3b3f13 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZ/BinTree/BinTree.h @@ -0,0 +1,54 @@ +// BinTree.h + +#include "../LZInWindow.h" +#include "../IMatchFinder.h" + +namespace BT_NAMESPACE { + +typedef UInt32 CIndex; +const UInt32 kMaxValForNormalize = (UInt32(1) << 31) - 1; + +class CMatchFinder: + public IMatchFinder, + public CLZInWindow, + public CMyUnknownImp, + public IMatchFinderSetNumPasses +{ + UInt32 _cyclicBufferPos; + UInt32 _cyclicBufferSize; // it must be historySize + 1 + UInt32 _matchMaxLen; + CIndex *_hash; + CIndex *_son; + UInt32 _hashMask; + UInt32 _cutValue; + UInt32 _hashSizeSum; + + void Normalize(); + void FreeThisClassMemory(); + void FreeMemory(); + + MY_UNKNOWN_IMP + + STDMETHOD(SetStream)(ISequentialInStream *inStream); + STDMETHOD_(void, ReleaseStream)(); + STDMETHOD(Init)(); + HRESULT MovePos(); + STDMETHOD_(Byte, GetIndexByte)(Int32 index); + STDMETHOD_(UInt32, GetMatchLen)(Int32 index, UInt32 back, UInt32 limit); + STDMETHOD_(UInt32, GetNumAvailableBytes)(); + STDMETHOD_(const Byte *, GetPointerToCurrentPos)(); + STDMETHOD_(Int32, NeedChangeBufferPos)(UInt32 numCheckBytes); + STDMETHOD_(void, ChangeBufferPos)(); + + STDMETHOD(Create)(UInt32 historySize, UInt32 keepAddBufferBefore, + UInt32 matchMaxLen, UInt32 keepAddBufferAfter); + STDMETHOD(GetMatches)(UInt32 *distances); + STDMETHOD(Skip)(UInt32 num); + +public: + CMatchFinder(); + virtual ~CMatchFinder(); + virtual void SetNumPasses(UInt32 numPasses) { _cutValue = numPasses; } +}; + +} diff --git a/lzma443/C/7zip/Compress/LZ/BinTree/BinTree2.h b/lzma443/C/7zip/Compress/LZ/BinTree/BinTree2.h new file mode 100755 index 0000000..74ca8d9 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZ/BinTree/BinTree2.h @@ -0,0 +1,12 @@ +// BinTree2.h + +#ifndef __BINTREE2_H +#define __BINTREE2_H + +#define BT_NAMESPACE NBT2 + +#include "BinTreeMain.h" + +#undef BT_NAMESPACE + +#endif diff --git a/lzma443/C/7zip/Compress/LZ/BinTree/BinTree3.h b/lzma443/C/7zip/Compress/LZ/BinTree/BinTree3.h new file mode 100755 index 0000000..76bd9dd --- /dev/null +++ b/lzma443/C/7zip/Compress/LZ/BinTree/BinTree3.h @@ -0,0 +1,16 @@ +// BinTree3.h + +#ifndef __BINTREE3_H +#define __BINTREE3_H + +#define BT_NAMESPACE NBT3 + +#define HASH_ARRAY_2 + +#include "BinTreeMain.h" + +#undef HASH_ARRAY_2 + +#undef BT_NAMESPACE + +#endif diff --git a/lzma443/C/7zip/Compress/LZ/BinTree/BinTree3Z.h b/lzma443/C/7zip/Compress/LZ/BinTree/BinTree3Z.h new file mode 100755 index 0000000..d2c092b --- /dev/null +++ b/lzma443/C/7zip/Compress/LZ/BinTree/BinTree3Z.h @@ -0,0 +1,16 @@ +// BinTree3Z.h + +#ifndef __BINTREE3Z_H +#define __BINTREE3Z_H + +#define BT_NAMESPACE NBT3Z + +#define HASH_ZIP + +#include "BinTreeMain.h" + +#undef HASH_ZIP + +#undef BT_NAMESPACE + +#endif diff --git a/lzma443/C/7zip/Compress/LZ/BinTree/BinTree4.h b/lzma443/C/7zip/Compress/LZ/BinTree/BinTree4.h new file mode 100755 index 0000000..08e2d1c --- /dev/null +++ b/lzma443/C/7zip/Compress/LZ/BinTree/BinTree4.h @@ -0,0 +1,18 @@ +// BinTree4.h + +#ifndef __BINTREE4_H +#define __BINTREE4_H + +#define BT_NAMESPACE NBT4 + +#define HASH_ARRAY_2 +#define HASH_ARRAY_3 + +#include "BinTreeMain.h" + +#undef HASH_ARRAY_2 +#undef HASH_ARRAY_3 + +#undef BT_NAMESPACE + +#endif diff --git a/lzma443/C/7zip/Compress/LZ/BinTree/BinTreeMain.h b/lzma443/C/7zip/Compress/LZ/BinTree/BinTreeMain.h new file mode 100755 index 0000000..7a6f621 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZ/BinTree/BinTreeMain.h @@ -0,0 +1,531 @@ +// BinTreeMain.h + +#include "../../../../Common/Defs.h" +#include "../../../../Common/CRC.h" +#include "../../../../Common/Alloc.h" + +#include "BinTree.h" + +// #include +// It's for prefetch +// But prefetch doesn't give big gain in K8. + +namespace BT_NAMESPACE { + +#ifdef HASH_ARRAY_2 + static const UInt32 kHash2Size = 1 << 10; + #define kNumHashDirectBytes 0 + #ifdef HASH_ARRAY_3 + static const UInt32 kNumHashBytes = 4; + static const UInt32 kHash3Size = 1 << 16; + #else + static const UInt32 kNumHashBytes = 3; + #endif + static const UInt32 kHashSize = 0; + static const UInt32 kMinMatchCheck = kNumHashBytes; + static const UInt32 kStartMaxLen = 1; +#else + #ifdef HASH_ZIP + #define kNumHashDirectBytes 0 + static const UInt32 kNumHashBytes = 3; + static const UInt32 kHashSize = 1 << 16; + static const UInt32 kMinMatchCheck = kNumHashBytes; + static const UInt32 kStartMaxLen = 1; + #else + #define kNumHashDirectBytes 2 + static const UInt32 kNumHashBytes = 2; + static const UInt32 kHashSize = 1 << (8 * kNumHashBytes); + static const UInt32 kMinMatchCheck = kNumHashBytes + 1; + static const UInt32 kStartMaxLen = 1; + #endif +#endif + +#ifdef HASH_ARRAY_2 +#ifdef HASH_ARRAY_3 +static const UInt32 kHash3Offset = kHash2Size; +#endif +#endif + +static const UInt32 kFixHashSize = 0 + #ifdef HASH_ARRAY_2 + + kHash2Size + #ifdef HASH_ARRAY_3 + + kHash3Size + #endif + #endif + ; + +CMatchFinder::CMatchFinder(): + _hash(0) +{ +} + +void CMatchFinder::FreeThisClassMemory() +{ + BigFree(_hash); + _hash = 0; +} + +void CMatchFinder::FreeMemory() +{ + FreeThisClassMemory(); + CLZInWindow::Free(); +} + +CMatchFinder::~CMatchFinder() +{ + FreeMemory(); +} + +STDMETHODIMP CMatchFinder::Create(UInt32 historySize, UInt32 keepAddBufferBefore, + UInt32 matchMaxLen, UInt32 keepAddBufferAfter) +{ + if (historySize > kMaxValForNormalize - 256) + { + FreeMemory(); + return E_INVALIDARG; + } + _cutValue = + #ifdef _HASH_CHAIN + 8 + (matchMaxLen >> 2); + #else + 16 + (matchMaxLen >> 1); + #endif + UInt32 sizeReserv = (historySize + keepAddBufferBefore + + matchMaxLen + keepAddBufferAfter) / 2 + 256; + if (CLZInWindow::Create(historySize + keepAddBufferBefore, + matchMaxLen + keepAddBufferAfter, sizeReserv)) + { + _matchMaxLen = matchMaxLen; + UInt32 newCyclicBufferSize = historySize + 1; + if (_hash != 0 && newCyclicBufferSize == _cyclicBufferSize) + return S_OK; + FreeThisClassMemory(); + _cyclicBufferSize = newCyclicBufferSize; // don't change it + + UInt32 hs = kHashSize; + + #ifdef HASH_ARRAY_2 + hs = historySize - 1; + hs |= (hs >> 1); + hs |= (hs >> 2); + hs |= (hs >> 4); + hs |= (hs >> 8); + hs >>= 1; + hs |= 0xFFFF; + if (hs > (1 << 24)) + { + #ifdef HASH_ARRAY_3 + hs >>= 1; + #else + hs = (1 << 24) - 1; + #endif + } + _hashMask = hs; + hs++; + #endif + _hashSizeSum = hs + kFixHashSize; + UInt32 numItems = _hashSizeSum + _cyclicBufferSize + #ifndef _HASH_CHAIN + * 2 + #endif + ; + size_t sizeInBytes = (size_t)numItems * sizeof(CIndex); + if (sizeInBytes / sizeof(CIndex) != numItems) + return E_OUTOFMEMORY; + _hash = (CIndex *)BigAlloc(sizeInBytes); + _son = _hash + _hashSizeSum; + if (_hash != 0) + return S_OK; + } + FreeMemory(); + return E_OUTOFMEMORY; +} + +static const UInt32 kEmptyHashValue = 0; + +STDMETHODIMP CMatchFinder::SetStream(ISequentialInStream *stream) +{ + CLZInWindow::SetStream(stream); + return S_OK; +} + +STDMETHODIMP CMatchFinder::Init() +{ + RINOK(CLZInWindow::Init()); + for(UInt32 i = 0; i < _hashSizeSum; i++) + _hash[i] = kEmptyHashValue; + _cyclicBufferPos = 0; + ReduceOffsets(-1); + return S_OK; +} + +STDMETHODIMP_(void) CMatchFinder::ReleaseStream() +{ + // ReleaseStream(); +} + +#ifdef HASH_ARRAY_2 +#ifdef HASH_ARRAY_3 + +#define HASH_CALC { \ + UInt32 temp = CCRC::Table[cur[0]] ^ cur[1]; \ + hash2Value = temp & (kHash2Size - 1); \ + hash3Value = (temp ^ (UInt32(cur[2]) << 8)) & (kHash3Size - 1); \ + hashValue = (temp ^ (UInt32(cur[2]) << 8) ^ (CCRC::Table[cur[3]] << 5)) & _hashMask; } + +#else // no HASH_ARRAY_3 +#define HASH_CALC { \ + UInt32 temp = CCRC::Table[cur[0]] ^ cur[1]; \ + hash2Value = temp & (kHash2Size - 1); \ + hashValue = (temp ^ (UInt32(cur[2]) << 8)) & _hashMask; } +#endif // HASH_ARRAY_3 +#else // no HASH_ARRAY_2 +#ifdef HASH_ZIP +inline UInt32 Hash(const Byte *pointer) +{ + return ((UInt32(pointer[0]) << 8) ^ CCRC::Table[pointer[1]] ^ pointer[2]) & (kHashSize - 1); +} +#else // no HASH_ZIP +inline UInt32 Hash(const Byte *pointer) +{ + return pointer[0] ^ (UInt32(pointer[1]) << 8); +} +#endif // HASH_ZIP +#endif // HASH_ARRAY_2 + +STDMETHODIMP CMatchFinder::GetMatches(UInt32 *distances) +{ + UInt32 lenLimit; + if (_pos + _matchMaxLen <= _streamPos) + lenLimit = _matchMaxLen; + else + { + lenLimit = _streamPos - _pos; + if(lenLimit < kMinMatchCheck) + { + distances[0] = 0; + return MovePos(); + } + } + + int offset = 1; + + UInt32 matchMinPos = (_pos > _cyclicBufferSize) ? (_pos - _cyclicBufferSize) : 0; + const Byte *cur = _buffer + _pos; + + UInt32 maxLen = kStartMaxLen; // to avoid items for len < hashSize; + + #ifdef HASH_ARRAY_2 + UInt32 hash2Value; + #ifdef HASH_ARRAY_3 + UInt32 hash3Value; + #endif + UInt32 hashValue; + HASH_CALC; + #else + UInt32 hashValue = Hash(cur); + #endif + + UInt32 curMatch = _hash[kFixHashSize + hashValue]; + #ifdef HASH_ARRAY_2 + UInt32 curMatch2 = _hash[hash2Value]; + #ifdef HASH_ARRAY_3 + UInt32 curMatch3 = _hash[kHash3Offset + hash3Value]; + #endif + _hash[hash2Value] = _pos; + if(curMatch2 > matchMinPos) + if (_buffer[curMatch2] == cur[0]) + { + distances[offset++] = maxLen = 2; + distances[offset++] = _pos - curMatch2 - 1; + } + + #ifdef HASH_ARRAY_3 + _hash[kHash3Offset + hash3Value] = _pos; + if(curMatch3 > matchMinPos) + if (_buffer[curMatch3] == cur[0]) + { + if (curMatch3 == curMatch2) + offset -= 2; + distances[offset++] = maxLen = 3; + distances[offset++] = _pos - curMatch3 - 1; + curMatch2 = curMatch3; + } + #endif + if (offset != 1 && curMatch2 == curMatch) + { + offset -= 2; + maxLen = kStartMaxLen; + } + #endif + + _hash[kFixHashSize + hashValue] = _pos; + + CIndex *son = _son; + + #ifdef _HASH_CHAIN + son[_cyclicBufferPos] = curMatch; + #else + CIndex *ptr0 = son + (_cyclicBufferPos << 1) + 1; + CIndex *ptr1 = son + (_cyclicBufferPos << 1); + + UInt32 len0, len1; + len0 = len1 = kNumHashDirectBytes; + #endif + + #if kNumHashDirectBytes != 0 + if(curMatch > matchMinPos) + { + if (_buffer[curMatch + kNumHashDirectBytes] != cur[kNumHashDirectBytes]) + { + distances[offset++] = maxLen = kNumHashDirectBytes; + distances[offset++] = _pos - curMatch - 1; + } + } + #endif + UInt32 count = _cutValue; + while(true) + { + if(curMatch <= matchMinPos || count-- == 0) + { + #ifndef _HASH_CHAIN + *ptr0 = *ptr1 = kEmptyHashValue; + #endif + break; + } + UInt32 delta = _pos - curMatch; + UInt32 cyclicPos = (delta <= _cyclicBufferPos) ? + (_cyclicBufferPos - delta): + (_cyclicBufferPos - delta + _cyclicBufferSize); + CIndex *pair = son + + #ifdef _HASH_CHAIN + cyclicPos; + #else + (cyclicPos << 1); + #endif + + // _mm_prefetch((const char *)pair, _MM_HINT_T0); + + const Byte *pb = _buffer + curMatch; + UInt32 len = + #ifdef _HASH_CHAIN + kNumHashDirectBytes; + if (pb[maxLen] == cur[maxLen]) + #else + MyMin(len0, len1); + #endif + if (pb[len] == cur[len]) + { + while(++len != lenLimit) + if (pb[len] != cur[len]) + break; + if (maxLen < len) + { + distances[offset++] = maxLen = len; + distances[offset++] = delta - 1; + if (len == lenLimit) + { + #ifndef _HASH_CHAIN + *ptr1 = pair[0]; + *ptr0 = pair[1]; + #endif + break; + } + } + } + #ifdef _HASH_CHAIN + curMatch = *pair; + #else + if (pb[len] < cur[len]) + { + *ptr1 = curMatch; + ptr1 = pair + 1; + curMatch = *ptr1; + len1 = len; + } + else + { + *ptr0 = curMatch; + ptr0 = pair; + curMatch = *ptr0; + len0 = len; + } + #endif + } + distances[0] = offset - 1; + if (++_cyclicBufferPos == _cyclicBufferSize) + _cyclicBufferPos = 0; + RINOK(CLZInWindow::MovePos()); + if (_pos == kMaxValForNormalize) + Normalize(); + return S_OK; +} + +STDMETHODIMP CMatchFinder::Skip(UInt32 num) +{ + do + { + #ifdef _HASH_CHAIN + if (_streamPos - _pos < kNumHashBytes) + { + RINOK(MovePos()); + continue; + } + #else + UInt32 lenLimit; + if (_pos + _matchMaxLen <= _streamPos) + lenLimit = _matchMaxLen; + else + { + lenLimit = _streamPos - _pos; + if(lenLimit < kMinMatchCheck) + { + RINOK(MovePos()); + continue; + } + } + UInt32 matchMinPos = (_pos > _cyclicBufferSize) ? (_pos - _cyclicBufferSize) : 0; + #endif + const Byte *cur = _buffer + _pos; + + #ifdef HASH_ARRAY_2 + UInt32 hash2Value; + #ifdef HASH_ARRAY_3 + UInt32 hash3Value; + UInt32 hashValue; + HASH_CALC; + _hash[kHash3Offset + hash3Value] = _pos; + #else + UInt32 hashValue; + HASH_CALC; + #endif + _hash[hash2Value] = _pos; + #else + UInt32 hashValue = Hash(cur); + #endif + + UInt32 curMatch = _hash[kFixHashSize + hashValue]; + _hash[kFixHashSize + hashValue] = _pos; + + #ifdef _HASH_CHAIN + _son[_cyclicBufferPos] = curMatch; + #else + CIndex *son = _son; + CIndex *ptr0 = son + (_cyclicBufferPos << 1) + 1; + CIndex *ptr1 = son + (_cyclicBufferPos << 1); + + UInt32 len0, len1; + len0 = len1 = kNumHashDirectBytes; + UInt32 count = _cutValue; + while(true) + { + if(curMatch <= matchMinPos || count-- == 0) + { + *ptr0 = *ptr1 = kEmptyHashValue; + break; + } + + UInt32 delta = _pos - curMatch; + UInt32 cyclicPos = (delta <= _cyclicBufferPos) ? + (_cyclicBufferPos - delta): + (_cyclicBufferPos - delta + _cyclicBufferSize); + CIndex *pair = son + (cyclicPos << 1); + + // _mm_prefetch((const char *)pair, _MM_HINT_T0); + + const Byte *pb = _buffer + curMatch; + UInt32 len = MyMin(len0, len1); + + if (pb[len] == cur[len]) + { + while(++len != lenLimit) + if (pb[len] != cur[len]) + break; + if (len == lenLimit) + { + *ptr1 = pair[0]; + *ptr0 = pair[1]; + break; + } + } + if (pb[len] < cur[len]) + { + *ptr1 = curMatch; + ptr1 = pair + 1; + curMatch = *ptr1; + len1 = len; + } + else + { + *ptr0 = curMatch; + ptr0 = pair; + curMatch = *ptr0; + len0 = len; + } + } + #endif + if (++_cyclicBufferPos == _cyclicBufferSize) + _cyclicBufferPos = 0; + RINOK(CLZInWindow::MovePos()); + if (_pos == kMaxValForNormalize) + Normalize(); + } + while(--num != 0); + return S_OK; +} + +void CMatchFinder::Normalize() +{ + UInt32 subValue = _pos - _cyclicBufferSize; + CIndex *items = _hash; + UInt32 numItems = (_hashSizeSum + _cyclicBufferSize + #ifndef _HASH_CHAIN + * 2 + #endif + ); + for (UInt32 i = 0; i < numItems; i++) + { + UInt32 value = items[i]; + if (value <= subValue) + value = kEmptyHashValue; + else + value -= subValue; + items[i] = value; + } + ReduceOffsets(subValue); +} + +HRESULT CMatchFinder::MovePos() +{ + if (++_cyclicBufferPos == _cyclicBufferSize) + _cyclicBufferPos = 0; + RINOK(CLZInWindow::MovePos()); + if (_pos == kMaxValForNormalize) + Normalize(); + return S_OK; +} + +STDMETHODIMP_(Byte) CMatchFinder::GetIndexByte(Int32 index) + { return CLZInWindow::GetIndexByte(index); } + +STDMETHODIMP_(UInt32) CMatchFinder::GetMatchLen(Int32 index, + UInt32 back, UInt32 limit) + { return CLZInWindow::GetMatchLen(index, back, limit); } + +STDMETHODIMP_(UInt32) CMatchFinder::GetNumAvailableBytes() + { return CLZInWindow::GetNumAvailableBytes(); } + +STDMETHODIMP_(const Byte *) CMatchFinder::GetPointerToCurrentPos() + { return CLZInWindow::GetPointerToCurrentPos(); } + +STDMETHODIMP_(Int32) CMatchFinder::NeedChangeBufferPos(UInt32 numCheckBytes) + { return CLZInWindow::NeedMove(numCheckBytes) ? 1: 0; } + +STDMETHODIMP_(void) CMatchFinder::ChangeBufferPos() + { CLZInWindow::MoveBlock();} + +#undef HASH_CALC +#undef kNumHashDirectBytes + +} diff --git a/lzma443/C/7zip/Compress/LZ/HashChain/HC2.h b/lzma443/C/7zip/Compress/LZ/HashChain/HC2.h new file mode 100755 index 0000000..d8e61a7 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZ/HashChain/HC2.h @@ -0,0 +1,13 @@ +// HC2.h + +#ifndef __HC2_H +#define __HC2_H + +#define BT_NAMESPACE NHC2 + +#include "HCMain.h" + +#undef BT_NAMESPACE + +#endif + diff --git a/lzma443/C/7zip/Compress/LZ/HashChain/HC3.h b/lzma443/C/7zip/Compress/LZ/HashChain/HC3.h new file mode 100755 index 0000000..263690a --- /dev/null +++ b/lzma443/C/7zip/Compress/LZ/HashChain/HC3.h @@ -0,0 +1,16 @@ +// HC3.h + +#ifndef __HC3_H +#define __HC3_H + +#define BT_NAMESPACE NHC3 + +#define HASH_ARRAY_2 + +#include "HCMain.h" + +#undef HASH_ARRAY_2 +#undef BT_NAMESPACE + +#endif + diff --git a/lzma443/C/7zip/Compress/LZ/HashChain/HC4.h b/lzma443/C/7zip/Compress/LZ/HashChain/HC4.h new file mode 100755 index 0000000..1fda4ac --- /dev/null +++ b/lzma443/C/7zip/Compress/LZ/HashChain/HC4.h @@ -0,0 +1,19 @@ +// HC4.h + +#ifndef __HC4_H +#define __HC4_H + +#define BT_NAMESPACE NHC4 + +#define HASH_ARRAY_2 +#define HASH_ARRAY_3 + +#include "HCMain.h" + +#undef HASH_ARRAY_2 +#undef HASH_ARRAY_3 + +#undef BT_NAMESPACE + +#endif + diff --git a/lzma443/C/7zip/Compress/LZ/HashChain/HCMain.h b/lzma443/C/7zip/Compress/LZ/HashChain/HCMain.h new file mode 100755 index 0000000..d509bef --- /dev/null +++ b/lzma443/C/7zip/Compress/LZ/HashChain/HCMain.h @@ -0,0 +1,6 @@ +// HCMain.h + +#define _HASH_CHAIN +#include "../BinTree/BinTreeMain.h" +#undef _HASH_CHAIN + diff --git a/lzma443/C/7zip/Compress/LZ/IMatchFinder.h b/lzma443/C/7zip/Compress/LZ/IMatchFinder.h new file mode 100755 index 0000000..ca07de5 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZ/IMatchFinder.h @@ -0,0 +1,35 @@ +// MatchFinders/IMatchFinder.h + +#ifndef __IMATCHFINDER_H +#define __IMATCHFINDER_H + +struct IInWindowStream: public IUnknown +{ + STDMETHOD(SetStream)(ISequentialInStream *inStream) PURE; + STDMETHOD_(void, ReleaseStream)() PURE; + STDMETHOD(Init)() PURE; + STDMETHOD_(Byte, GetIndexByte)(Int32 index) PURE; + STDMETHOD_(UInt32, GetMatchLen)(Int32 index, UInt32 distance, UInt32 limit) PURE; + STDMETHOD_(UInt32, GetNumAvailableBytes)() PURE; + STDMETHOD_(const Byte *, GetPointerToCurrentPos)() PURE; + STDMETHOD_(Int32, NeedChangeBufferPos)(UInt32 numCheckBytes) PURE; + STDMETHOD_(void, ChangeBufferPos)() PURE; +}; + +struct IMatchFinder: public IInWindowStream +{ + STDMETHOD(Create)(UInt32 historySize, UInt32 keepAddBufferBefore, + UInt32 matchMaxLen, UInt32 keepAddBufferAfter) PURE; + STDMETHOD(GetMatches)(UInt32 *distances) PURE; + STDMETHOD(Skip)(UInt32 num) PURE; +}; + +struct IMatchFinderSetNumPasses +{ + virtual void SetNumPasses(UInt32 numPasses) PURE; + #ifndef _WIN32 + virtual ~IMatchFinderSetNumPasses() {} + #endif +}; + +#endif diff --git a/lzma443/C/7zip/Compress/LZ/LZInWindow.cpp b/lzma443/C/7zip/Compress/LZ/LZInWindow.cpp new file mode 100755 index 0000000..0e65c42 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZ/LZInWindow.cpp @@ -0,0 +1,105 @@ +// LZInWindow.cpp + +#include "StdAfx.h" + +#include "LZInWindow.h" +#include "../../../Common/MyCom.h" +#include "../../../Common/Alloc.h" + +void CLZInWindow::Free() +{ + ::BigFree(_bufferBase); + _bufferBase = 0; +} + +bool CLZInWindow::Create(UInt32 keepSizeBefore, UInt32 keepSizeAfter, UInt32 keepSizeReserv) +{ + _keepSizeBefore = keepSizeBefore; + _keepSizeAfter = keepSizeAfter; + UInt32 blockSize = keepSizeBefore + keepSizeAfter + keepSizeReserv; + if (_bufferBase == 0 || _blockSize != blockSize) + { + Free(); + _blockSize = blockSize; + if (_blockSize != 0) + _bufferBase = (Byte *)::BigAlloc(_blockSize); + } + _pointerToLastSafePosition = _bufferBase + _blockSize - keepSizeAfter; + if (_blockSize == 0) + return true; + return (_bufferBase != 0); +} + +void CLZInWindow::SetStream(ISequentialInStream *stream) +{ + _stream = stream; +} + +HRESULT CLZInWindow::Init() +{ + _buffer = _bufferBase; + _pos = 0; + _streamPos = 0; + _streamEndWasReached = false; + return ReadBlock(); +} + +/* +void CLZInWindow::ReleaseStream() +{ + _stream.Release(); +} +*/ + +/////////////////////////////////////////// +// ReadBlock + +// In State: +// (_buffer + _streamPos) <= (_bufferBase + _blockSize) +// Out State: +// _posLimit <= _blockSize - _keepSizeAfter; +// if(_streamEndWasReached == false): +// _streamPos >= _pos + _keepSizeAfter +// _posLimit = _streamPos - _keepSizeAfter; +// else +// + +HRESULT CLZInWindow::ReadBlock() +{ + if(_streamEndWasReached) + return S_OK; + while(true) + { + UInt32 size = (UInt32)(_bufferBase - _buffer) + _blockSize - _streamPos; + if(size == 0) + return S_OK; + UInt32 numReadBytes; + RINOK(_stream->Read(_buffer + _streamPos, size, &numReadBytes)); + if(numReadBytes == 0) + { + _posLimit = _streamPos; + const Byte *pointerToPostion = _buffer + _posLimit; + if(pointerToPostion > _pointerToLastSafePosition) + _posLimit = (UInt32)(_pointerToLastSafePosition - _buffer); + _streamEndWasReached = true; + return S_OK; + } + _streamPos += numReadBytes; + if(_streamPos >= _pos + _keepSizeAfter) + { + _posLimit = _streamPos - _keepSizeAfter; + return S_OK; + } + } +} + +void CLZInWindow::MoveBlock() +{ + UInt32 offset = (UInt32)(_buffer - _bufferBase) + _pos - _keepSizeBefore; + // we need one additional byte, since MovePos moves on 1 byte. + if (offset > 0) + offset--; + UInt32 numBytes = (UInt32)(_buffer - _bufferBase) + _streamPos - offset; + memmove(_bufferBase, _bufferBase + offset, numBytes); + _buffer -= offset; +} diff --git a/lzma443/C/7zip/Compress/LZ/LZInWindow.h b/lzma443/C/7zip/Compress/LZ/LZInWindow.h new file mode 100755 index 0000000..54f2cb7 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZ/LZInWindow.h @@ -0,0 +1,87 @@ +// LZInWindow.h + +#ifndef __LZ_IN_WINDOW_H +#define __LZ_IN_WINDOW_H + +#include "../../IStream.h" + +class CLZInWindow +{ + Byte *_bufferBase; // pointer to buffer with data + ISequentialInStream *_stream; + UInt32 _posLimit; // offset (from _buffer) when new block reading must be done + bool _streamEndWasReached; // if (true) then _streamPos shows real end of stream + const Byte *_pointerToLastSafePosition; +protected: + Byte *_buffer; // Pointer to virtual Buffer begin + UInt32 _blockSize; // Size of Allocated memory block + UInt32 _pos; // offset (from _buffer) of curent byte + UInt32 _keepSizeBefore; // how many BYTEs must be kept in buffer before _pos + UInt32 _keepSizeAfter; // how many BYTEs must be kept buffer after _pos + UInt32 _streamPos; // offset (from _buffer) of first not read byte from Stream + + void MoveBlock(); + HRESULT ReadBlock(); + void Free(); +public: + CLZInWindow(): _bufferBase(0) {} + virtual ~CLZInWindow() { Free(); } + + // keepSizeBefore + keepSizeAfter + keepSizeReserv < 4G) + bool Create(UInt32 keepSizeBefore, UInt32 keepSizeAfter, UInt32 keepSizeReserv = (1<<17)); + + void SetStream(ISequentialInStream *stream); + HRESULT Init(); + // void ReleaseStream(); + + Byte *GetBuffer() const { return _buffer; } + + const Byte *GetPointerToCurrentPos() const { return _buffer + _pos; } + + HRESULT MovePos() + { + _pos++; + if (_pos > _posLimit) + { + const Byte *pointerToPostion = _buffer + _pos; + if(pointerToPostion > _pointerToLastSafePosition) + MoveBlock(); + return ReadBlock(); + } + else + return S_OK; + } + Byte GetIndexByte(Int32 index) const { return _buffer[(size_t)_pos + index]; } + + // index + limit have not to exceed _keepSizeAfter; + // -2G <= index < 2G + UInt32 GetMatchLen(Int32 index, UInt32 distance, UInt32 limit) const + { + if(_streamEndWasReached) + if ((_pos + index) + limit > _streamPos) + limit = _streamPos - (_pos + index); + distance++; + const Byte *pby = _buffer + (size_t)_pos + index; + UInt32 i; + for(i = 0; i < limit && pby[i] == pby[(size_t)i - distance]; i++); + return i; + } + + UInt32 GetNumAvailableBytes() const { return _streamPos - _pos; } + + void ReduceOffsets(Int32 subValue) + { + _buffer += subValue; + _posLimit -= subValue; + _pos -= subValue; + _streamPos -= subValue; + } + + bool NeedMove(UInt32 numCheckBytes) + { + UInt32 reserv = _pointerToLastSafePosition - (_buffer + _pos); + return (reserv <= numCheckBytes); + } +}; + +#endif diff --git a/lzma443/C/7zip/Compress/LZ/LZOutWindow.cpp b/lzma443/C/7zip/Compress/LZ/LZOutWindow.cpp new file mode 100755 index 0000000..e2d6aba --- /dev/null +++ b/lzma443/C/7zip/Compress/LZ/LZOutWindow.cpp @@ -0,0 +1,17 @@ +// LZOutWindow.cpp + +#include "StdAfx.h" + +#include "../../../Common/Alloc.h" +#include "LZOutWindow.h" + +void CLZOutWindow::Init(bool solid) +{ + if(!solid) + COutBuffer::Init(); + #ifdef _NO_EXCEPTIONS + ErrorCode = S_OK; + #endif +} + + diff --git a/lzma443/C/7zip/Compress/LZ/LZOutWindow.h b/lzma443/C/7zip/Compress/LZ/LZOutWindow.h new file mode 100755 index 0000000..3c50c6e --- /dev/null +++ b/lzma443/C/7zip/Compress/LZ/LZOutWindow.h @@ -0,0 +1,56 @@ +// LZOutWindow.h + +#ifndef __LZ_OUT_WINDOW_H +#define __LZ_OUT_WINDOW_H + +#include "../../IStream.h" +#include "../../Common/OutBuffer.h" + +#ifndef _NO_EXCEPTIONS +typedef COutBufferException CLZOutWindowException; +#endif + +class CLZOutWindow: public COutBuffer +{ +public: + void Init(bool solid = false); + + // distance >= 0, len > 0, + bool CopyBlock(UInt32 distance, UInt32 len) + { + UInt32 pos = _pos - distance - 1; + if (distance >= _pos) + { + if (!_overDict || distance >= _bufferSize) + return false; + pos += _bufferSize; + } + do + { + if (pos == _bufferSize) + pos = 0; + _buffer[_pos++] = _buffer[pos++]; + if (_pos == _limitPos) + FlushWithCheck(); + } + while(--len != 0); + return true; + } + + void PutByte(Byte b) + { + _buffer[_pos++] = b; + if (_pos == _limitPos) + FlushWithCheck(); + } + + Byte GetByte(UInt32 distance) const + { + UInt32 pos = _pos - distance - 1; + if (pos >= _bufferSize) + pos += _bufferSize; + return _buffer[pos]; + } +}; + +#endif diff --git a/lzma443/C/7zip/Compress/LZ/StdAfx.h b/lzma443/C/7zip/Compress/LZ/StdAfx.h new file mode 100755 index 0000000..3ff6d8a --- /dev/null +++ b/lzma443/C/7zip/Compress/LZ/StdAfx.h @@ -0,0 +1,6 @@ +// StdAfx.h + +#ifndef __STDAFX_H +#define __STDAFX_H + +#endif diff --git a/lzma443/C/7zip/Compress/LZMA/LZMA.h b/lzma443/C/7zip/Compress/LZMA/LZMA.h new file mode 100755 index 0000000..7bc4c43 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA/LZMA.h @@ -0,0 +1,82 @@ +// LZMA.h + +#ifndef __LZMA_H +#define __LZMA_H + +namespace NCompress { +namespace NLZMA { + +const UInt32 kNumRepDistances = 4; + +const int kNumStates = 12; + +const Byte kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5}; +const Byte kMatchNextStates[kNumStates] = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10}; +const Byte kRepNextStates[kNumStates] = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11}; +const Byte kShortRepNextStates[kNumStates]= {9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11}; + +class CState +{ +public: + Byte Index; + void Init() { Index = 0; } + void UpdateChar() { Index = kLiteralNextStates[Index]; } + void UpdateMatch() { Index = kMatchNextStates[Index]; } + void UpdateRep() { Index = kRepNextStates[Index]; } + void UpdateShortRep() { Index = kShortRepNextStates[Index]; } + bool IsCharState() const { return Index < 7; } +}; + +const int kNumPosSlotBits = 6; +const int kDicLogSizeMin = 0; +const int kDicLogSizeMax = 32; +const int kDistTableSizeMax = kDicLogSizeMax * 2; + +const UInt32 kNumLenToPosStates = 4; + +inline UInt32 GetLenToPosState(UInt32 len) +{ + len -= 2; + if (len < kNumLenToPosStates) + return len; + return kNumLenToPosStates - 1; +} + +namespace NLength { + +const int kNumPosStatesBitsMax = 4; +const UInt32 kNumPosStatesMax = (1 << kNumPosStatesBitsMax); + +const int kNumPosStatesBitsEncodingMax = 4; +const UInt32 kNumPosStatesEncodingMax = (1 << kNumPosStatesBitsEncodingMax); + +const int kNumLowBits = 3; +const int kNumMidBits = 3; +const int kNumHighBits = 8; +const UInt32 kNumLowSymbols = 1 << kNumLowBits; +const UInt32 kNumMidSymbols = 1 << kNumMidBits; +const UInt32 kNumSymbolsTotal = kNumLowSymbols + kNumMidSymbols + (1 << kNumHighBits); + +} + +const UInt32 kMatchMinLen = 2; +const UInt32 kMatchMaxLen = kMatchMinLen + NLength::kNumSymbolsTotal - 1; + +const int kNumAlignBits = 4; +const UInt32 kAlignTableSize = 1 << kNumAlignBits; +const UInt32 kAlignMask = (kAlignTableSize - 1); + +const UInt32 kStartPosModelIndex = 4; +const UInt32 kEndPosModelIndex = 14; +const UInt32 kNumPosModels = kEndPosModelIndex - kStartPosModelIndex; + +const UInt32 kNumFullDistances = 1 << (kEndPosModelIndex / 2); + +const int kNumLitPosStatesBitsEncodingMax = 4; +const int kNumLitContextBitsMax = 8; + +const int kNumMoveBits = 5; + +}} + +#endif diff --git a/lzma443/C/7zip/Compress/LZMA/LZMADecoder.cpp b/lzma443/C/7zip/Compress/LZMA/LZMADecoder.cpp new file mode 100755 index 0000000..1deae6a --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA/LZMADecoder.cpp @@ -0,0 +1,337 @@ +// LZMADecoder.cpp + +#include "StdAfx.h" + +#include "LZMADecoder.h" +#include "../../../Common/Defs.h" + +namespace NCompress { +namespace NLZMA { + +const int kLenIdFinished = -1; +const int kLenIdNeedInit = -2; + +void CDecoder::Init() +{ + { + for(int i = 0; i < kNumStates; i++) + { + for (UInt32 j = 0; j <= _posStateMask; j++) + { + _isMatch[i][j].Init(); + _isRep0Long[i][j].Init(); + } + _isRep[i].Init(); + _isRepG0[i].Init(); + _isRepG1[i].Init(); + _isRepG2[i].Init(); + } + } + { + for (UInt32 i = 0; i < kNumLenToPosStates; i++) + _posSlotDecoder[i].Init(); + } + { + for(UInt32 i = 0; i < kNumFullDistances - kEndPosModelIndex; i++) + _posDecoders[i].Init(); + } + _posAlignDecoder.Init(); + _lenDecoder.Init(_posStateMask + 1); + _repMatchLenDecoder.Init(_posStateMask + 1); + _literalDecoder.Init(); + + _state.Init(); + _reps[0] = _reps[1] = _reps[2] = _reps[3] = 0; +} + +HRESULT CDecoder::CodeSpec(UInt32 curSize) +{ + if (_outSizeDefined) + { + const UInt64 rem = _outSize - _outWindowStream.GetProcessedSize(); + if (curSize > rem) + curSize = (UInt32)rem; + } + + if (_remainLen == kLenIdFinished) + return S_OK; + if (_remainLen == kLenIdNeedInit) + { + _rangeDecoder.Init(); + Init(); + _remainLen = 0; + } + if (curSize == 0) + return S_OK; + + UInt32 rep0 = _reps[0]; + UInt32 rep1 = _reps[1]; + UInt32 rep2 = _reps[2]; + UInt32 rep3 = _reps[3]; + CState state = _state; + Byte previousByte; + + while(_remainLen > 0 && curSize > 0) + { + previousByte = _outWindowStream.GetByte(rep0); + _outWindowStream.PutByte(previousByte); + _remainLen--; + curSize--; + } + UInt64 nowPos64 = _outWindowStream.GetProcessedSize(); + if (nowPos64 == 0) + previousByte = 0; + else + previousByte = _outWindowStream.GetByte(0); + + while(curSize > 0) + { + { + #ifdef _NO_EXCEPTIONS + if (_rangeDecoder.Stream.ErrorCode != S_OK) + return _rangeDecoder.Stream.ErrorCode; + #endif + if (_rangeDecoder.Stream.WasFinished()) + return S_FALSE; + UInt32 posState = UInt32(nowPos64) & _posStateMask; + if (_isMatch[state.Index][posState].Decode(&_rangeDecoder) == 0) + { + if(!state.IsCharState()) + previousByte = _literalDecoder.DecodeWithMatchByte(&_rangeDecoder, + (UInt32)nowPos64, previousByte, _outWindowStream.GetByte(rep0)); + else + previousByte = _literalDecoder.DecodeNormal(&_rangeDecoder, + (UInt32)nowPos64, previousByte); + _outWindowStream.PutByte(previousByte); + state.UpdateChar(); + curSize--; + nowPos64++; + } + else + { + UInt32 len; + if(_isRep[state.Index].Decode(&_rangeDecoder) == 1) + { + len = 0; + if(_isRepG0[state.Index].Decode(&_rangeDecoder) == 0) + { + if(_isRep0Long[state.Index][posState].Decode(&_rangeDecoder) == 0) + { + state.UpdateShortRep(); + len = 1; + } + } + else + { + UInt32 distance; + if(_isRepG1[state.Index].Decode(&_rangeDecoder) == 0) + distance = rep1; + else + { + if (_isRepG2[state.Index].Decode(&_rangeDecoder) == 0) + distance = rep2; + else + { + distance = rep3; + rep3 = rep2; + } + rep2 = rep1; + } + rep1 = rep0; + rep0 = distance; + } + if (len == 0) + { + len = _repMatchLenDecoder.Decode(&_rangeDecoder, posState) + kMatchMinLen; + state.UpdateRep(); + } + } + else + { + rep3 = rep2; + rep2 = rep1; + rep1 = rep0; + len = kMatchMinLen + _lenDecoder.Decode(&_rangeDecoder, posState); + state.UpdateMatch(); + UInt32 posSlot = _posSlotDecoder[GetLenToPosState(len)].Decode(&_rangeDecoder); + if (posSlot >= kStartPosModelIndex) + { + UInt32 numDirectBits = (posSlot >> 1) - 1; + rep0 = ((2 | (posSlot & 1)) << numDirectBits); + + if (posSlot < kEndPosModelIndex) + rep0 += NRangeCoder::ReverseBitTreeDecode(_posDecoders + + rep0 - posSlot - 1, &_rangeDecoder, numDirectBits); + else + { + rep0 += (_rangeDecoder.DecodeDirectBits( + numDirectBits - kNumAlignBits) << kNumAlignBits); + rep0 += _posAlignDecoder.ReverseDecode(&_rangeDecoder); + if (rep0 == 0xFFFFFFFF) + { + _remainLen = kLenIdFinished; + return S_OK; + } + } + } + else + rep0 = posSlot; + } + UInt32 locLen = len; + if (len > curSize) + locLen = (UInt32)curSize; + if (!_outWindowStream.CopyBlock(rep0, locLen)) + return S_FALSE; + previousByte = _outWindowStream.GetByte(0); + curSize -= locLen; + nowPos64 += locLen; + len -= locLen; + if (len != 0) + { + _remainLen = (Int32)len; + break; + } + + #ifdef _NO_EXCEPTIONS + if (_outWindowStream.ErrorCode != S_OK) + return _outWindowStream.ErrorCode; + #endif + } + } + } + if (_rangeDecoder.Stream.WasFinished()) + return S_FALSE; + _reps[0] = rep0; + _reps[1] = rep1; + _reps[2] = rep2; + _reps[3] = rep3; + _state = state; + + return S_OK; +} + +STDMETHODIMP CDecoder::CodeReal(ISequentialInStream *inStream, + ISequentialOutStream *outStream, + const UInt64 *, const UInt64 *outSize, + ICompressProgressInfo *progress) +{ + SetInStream(inStream); + _outWindowStream.SetStream(outStream); + SetOutStreamSize(outSize); + CDecoderFlusher flusher(this); + + while (true) + { + UInt32 curSize = 1 << 18; + RINOK(CodeSpec(curSize)); + if (_remainLen == kLenIdFinished) + break; + if (progress != NULL) + { + UInt64 inSize = _rangeDecoder.GetProcessedSize(); + UInt64 nowPos64 = _outWindowStream.GetProcessedSize(); + RINOK(progress->SetRatioInfo(&inSize, &nowPos64)); + } + if (_outSizeDefined) + if (_outWindowStream.GetProcessedSize() >= _outSize) + break; + } + flusher.NeedFlush = false; + return Flush(); +} + + +#ifdef _NO_EXCEPTIONS + +#define LZMA_TRY_BEGIN +#define LZMA_TRY_END + +#else + +#define LZMA_TRY_BEGIN try { +#define LZMA_TRY_END } \ + catch(const CInBufferException &e) { return e.ErrorCode; } \ + catch(const CLZOutWindowException &e) { return e.ErrorCode; } \ + catch(...) { return S_FALSE; } + +#endif + + +STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, + ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, + ICompressProgressInfo *progress) +{ + LZMA_TRY_BEGIN + return CodeReal(inStream, outStream, inSize, outSize, progress); + LZMA_TRY_END +} + +STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte *properties, UInt32 size) +{ + if (size < 5) + return E_INVALIDARG; + int lc = properties[0] % 9; + Byte remainder = (Byte)(properties[0] / 9); + int lp = remainder % 5; + int pb = remainder / 5; + if (pb > NLength::kNumPosStatesBitsMax) + return E_INVALIDARG; + _posStateMask = (1 << pb) - 1; + UInt32 dictionarySize = 0; + for (int i = 0; i < 4; i++) + dictionarySize += ((UInt32)(properties[1 + i])) << (i * 8); + if (!_outWindowStream.Create(dictionarySize)) + return E_OUTOFMEMORY; + if (!_literalDecoder.Create(lp, lc)) + return E_OUTOFMEMORY; + if (!_rangeDecoder.Create(1 << 20)) + return E_OUTOFMEMORY; + return S_OK; +} + +STDMETHODIMP CDecoder::GetInStreamProcessedSize(UInt64 *value) +{ + *value = _rangeDecoder.GetProcessedSize(); + return S_OK; +} + +STDMETHODIMP CDecoder::SetInStream(ISequentialInStream *inStream) +{ + _rangeDecoder.SetStream(inStream); + return S_OK; +} + +STDMETHODIMP CDecoder::ReleaseInStream() +{ + _rangeDecoder.ReleaseStream(); + return S_OK; +} + +STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 *outSize) +{ + if ((_outSizeDefined = (outSize != NULL))) + _outSize = *outSize; + _remainLen = kLenIdNeedInit; + _outWindowStream.Init(); + return S_OK; +} + +#ifdef _ST_MODE + +STDMETHODIMP CDecoder::Read(void *data, UInt32 size, UInt32 *processedSize) +{ + LZMA_TRY_BEGIN + if (processedSize) + *processedSize = 0; + const UInt64 startPos = _outWindowStream.GetProcessedSize(); + _outWindowStream.SetMemStream((Byte *)data); + RINOK(CodeSpec(size)); + if (processedSize) + *processedSize = (UInt32)(_outWindowStream.GetProcessedSize() - startPos); + return Flush(); + LZMA_TRY_END +} + +#endif + +}} diff --git a/lzma443/C/7zip/Compress/LZMA/LZMADecoder.h b/lzma443/C/7zip/Compress/LZMA/LZMADecoder.h new file mode 100755 index 0000000..1c10409 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA/LZMADecoder.h @@ -0,0 +1,251 @@ +// LZMA/Decoder.h + +#ifndef __LZMA_DECODER_H +#define __LZMA_DECODER_H + +#include "../../../Common/MyCom.h" +#include "../../../Common/Alloc.h" +#include "../../ICoder.h" +#include "../LZ/LZOutWindow.h" +#include "../RangeCoder/RangeCoderBitTree.h" + +#include "LZMA.h" + +namespace NCompress { +namespace NLZMA { + +typedef NRangeCoder::CBitDecoder CMyBitDecoder; + +class CLiteralDecoder2 +{ + CMyBitDecoder _decoders[0x300]; +public: + void Init() + { + for (int i = 0; i < 0x300; i++) + _decoders[i].Init(); + } + Byte DecodeNormal(NRangeCoder::CDecoder *rangeDecoder) + { + UInt32 symbol = 1; + RC_INIT_VAR + do + { + // symbol = (symbol << 1) | _decoders[0][symbol].Decode(rangeDecoder); + RC_GETBIT(kNumMoveBits, _decoders[symbol].Prob, symbol) + } + while (symbol < 0x100); + RC_FLUSH_VAR + return (Byte)symbol; + } + Byte DecodeWithMatchByte(NRangeCoder::CDecoder *rangeDecoder, Byte matchByte) + { + UInt32 symbol = 1; + RC_INIT_VAR + do + { + UInt32 matchBit = (matchByte >> 7) & 1; + matchByte <<= 1; + // UInt32 bit = _decoders[1 + matchBit][symbol].Decode(rangeDecoder); + // symbol = (symbol << 1) | bit; + UInt32 bit; + RC_GETBIT2(kNumMoveBits, _decoders[0x100 + (matchBit << 8) + symbol].Prob, symbol, + bit = 0, bit = 1) + if (matchBit != bit) + { + while (symbol < 0x100) + { + // symbol = (symbol << 1) | _decoders[0][symbol].Decode(rangeDecoder); + RC_GETBIT(kNumMoveBits, _decoders[symbol].Prob, symbol) + } + break; + } + } + while (symbol < 0x100); + RC_FLUSH_VAR + return (Byte)symbol; + } +}; + +class CLiteralDecoder +{ + CLiteralDecoder2 *_coders; + int _numPrevBits; + int _numPosBits; + UInt32 _posMask; +public: + CLiteralDecoder(): _coders(0) {} + ~CLiteralDecoder() { Free(); } + void Free() + { + MyFree(_coders); + _coders = 0; + } + bool Create(int numPosBits, int numPrevBits) + { + if (_coders == 0 || (numPosBits + numPrevBits) != + (_numPrevBits + _numPosBits) ) + { + Free(); + UInt32 numStates = 1 << (numPosBits + numPrevBits); + _coders = (CLiteralDecoder2 *)MyAlloc(numStates * sizeof(CLiteralDecoder2)); + } + _numPosBits = numPosBits; + _posMask = (1 << numPosBits) - 1; + _numPrevBits = numPrevBits; + return (_coders != 0); + } + void Init() + { + UInt32 numStates = 1 << (_numPrevBits + _numPosBits); + for (UInt32 i = 0; i < numStates; i++) + _coders[i].Init(); + } + UInt32 GetState(UInt32 pos, Byte prevByte) const + { return ((pos & _posMask) << _numPrevBits) + (prevByte >> (8 - _numPrevBits)); } + Byte DecodeNormal(NRangeCoder::CDecoder *rangeDecoder, UInt32 pos, Byte prevByte) + { return _coders[GetState(pos, prevByte)].DecodeNormal(rangeDecoder); } + Byte DecodeWithMatchByte(NRangeCoder::CDecoder *rangeDecoder, UInt32 pos, Byte prevByte, Byte matchByte) + { return _coders[GetState(pos, prevByte)].DecodeWithMatchByte(rangeDecoder, matchByte); } +}; + +namespace NLength { + +class CDecoder +{ + CMyBitDecoder _choice; + CMyBitDecoder _choice2; + NRangeCoder::CBitTreeDecoder _lowCoder[kNumPosStatesMax]; + NRangeCoder::CBitTreeDecoder _midCoder[kNumPosStatesMax]; + NRangeCoder::CBitTreeDecoder _highCoder; +public: + void Init(UInt32 numPosStates) + { + _choice.Init(); + _choice2.Init(); + for (UInt32 posState = 0; posState < numPosStates; posState++) + { + _lowCoder[posState].Init(); + _midCoder[posState].Init(); + } + _highCoder.Init(); + } + UInt32 Decode(NRangeCoder::CDecoder *rangeDecoder, UInt32 posState) + { + if(_choice.Decode(rangeDecoder) == 0) + return _lowCoder[posState].Decode(rangeDecoder); + if(_choice2.Decode(rangeDecoder) == 0) + return kNumLowSymbols + _midCoder[posState].Decode(rangeDecoder); + return kNumLowSymbols + kNumMidSymbols + _highCoder.Decode(rangeDecoder); + } +}; + +} + +class CDecoder: + public ICompressCoder, + public ICompressSetDecoderProperties2, + public ICompressGetInStreamProcessedSize, + #ifdef _ST_MODE + public ICompressSetInStream, + public ICompressSetOutStreamSize, + public ISequentialInStream, + #endif + public CMyUnknownImp +{ + CLZOutWindow _outWindowStream; + NRangeCoder::CDecoder _rangeDecoder; + + CMyBitDecoder _isMatch[kNumStates][NLength::kNumPosStatesMax]; + CMyBitDecoder _isRep[kNumStates]; + CMyBitDecoder _isRepG0[kNumStates]; + CMyBitDecoder _isRepG1[kNumStates]; + CMyBitDecoder _isRepG2[kNumStates]; + CMyBitDecoder _isRep0Long[kNumStates][NLength::kNumPosStatesMax]; + + NRangeCoder::CBitTreeDecoder _posSlotDecoder[kNumLenToPosStates]; + + CMyBitDecoder _posDecoders[kNumFullDistances - kEndPosModelIndex]; + NRangeCoder::CBitTreeDecoder _posAlignDecoder; + + NLength::CDecoder _lenDecoder; + NLength::CDecoder _repMatchLenDecoder; + + CLiteralDecoder _literalDecoder; + + UInt32 _posStateMask; + + /////////////////// + // State + UInt32 _reps[4]; + CState _state; + Int32 _remainLen; // -1 means end of stream. // -2 means need Init + UInt64 _outSize; + bool _outSizeDefined; + + void Init(); + HRESULT CodeSpec(UInt32 size); +public: + + #ifdef _ST_MODE + MY_UNKNOWN_IMP5( + ICompressSetDecoderProperties2, + ICompressGetInStreamProcessedSize, + ICompressSetInStream, + ICompressSetOutStreamSize, + ISequentialInStream) + #else + MY_UNKNOWN_IMP2( + ICompressSetDecoderProperties2, + ICompressGetInStreamProcessedSize) + #endif + + void ReleaseStreams() + { + _outWindowStream.ReleaseStream(); + ReleaseInStream(); + } + + class CDecoderFlusher + { + CDecoder *_decoder; + public: + bool NeedFlush; + CDecoderFlusher(CDecoder *decoder): _decoder(decoder), NeedFlush(true) {} + ~CDecoderFlusher() + { + if (NeedFlush) + _decoder->Flush(); + _decoder->ReleaseStreams(); + } + }; + + HRESULT Flush() { return _outWindowStream.Flush(); } + + STDMETHOD(CodeReal)(ISequentialInStream *inStream, + ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, + ICompressProgressInfo *progress); + + STDMETHOD(Code)(ISequentialInStream *inStream, + ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, + ICompressProgressInfo *progress); + + STDMETHOD(SetDecoderProperties2)(const Byte *data, UInt32 size); + + STDMETHOD(GetInStreamProcessedSize)(UInt64 *value); + + STDMETHOD(SetInStream)(ISequentialInStream *inStream); + STDMETHOD(ReleaseInStream)(); + STDMETHOD(SetOutStreamSize)(const UInt64 *outSize); + + #ifdef _ST_MODE + STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); + #endif + + CDecoder(): _outSizeDefined(false) {} + virtual ~CDecoder() {} +}; + +}} + +#endif diff --git a/lzma443/C/7zip/Compress/LZMA/LZMAEncoder.cpp b/lzma443/C/7zip/Compress/LZMA/LZMAEncoder.cpp new file mode 100755 index 0000000..2ac676f --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA/LZMAEncoder.cpp @@ -0,0 +1,1564 @@ +// LZMA/Encoder.cpp + +#include "StdAfx.h" + +#include "../../../Common/Defs.h" +#include "../../Common/StreamUtils.h" + +#include "LZMAEncoder.h" + +// for minimal compressing code size define these: +// #define COMPRESS_MF_BT +// #define COMPRESS_MF_BT4 + +#if !defined(COMPRESS_MF_BT) && !defined(COMPRESS_MF_HC) +#define COMPRESS_MF_BT +#define COMPRESS_MF_HC +#endif + +#ifdef COMPRESS_MF_BT +#if !defined(COMPRESS_MF_BT2) && !defined(COMPRESS_MF_BT3) && !defined(COMPRESS_MF_BT4) +#define COMPRESS_MF_BT2 +#define COMPRESS_MF_BT3 +#define COMPRESS_MF_BT4 +#endif +#ifdef COMPRESS_MF_BT2 +#include "../LZ/BinTree/BinTree2.h" +#endif +#ifdef COMPRESS_MF_BT3 +#include "../LZ/BinTree/BinTree3.h" +#endif +#ifdef COMPRESS_MF_BT4 +#include "../LZ/BinTree/BinTree4.h" +#endif +#endif + +#ifdef COMPRESS_MF_HC +#include "../LZ/HashChain/HC4.h" +#endif + +#ifdef COMPRESS_MF_MT +#include "../LZ/MT/MT.h" +#endif + +namespace NCompress { +namespace NLZMA { + +const int kDefaultDictionaryLogSize = 22; +const UInt32 kNumFastBytesDefault = 0x20; + +enum +{ + kBT2, + kBT3, + kBT4, + kHC4 +}; + +static const wchar_t *kMatchFinderIDs[] = +{ + L"BT2", + L"BT3", + L"BT4", + L"HC4" +}; + +Byte g_FastPos[1 << 11]; + +class CFastPosInit +{ +public: + CFastPosInit() { Init(); } + void Init() + { + const Byte kFastSlots = 22; + int c = 2; + g_FastPos[0] = 0; + g_FastPos[1] = 1; + + for (Byte slotFast = 2; slotFast < kFastSlots; slotFast++) + { + UInt32 k = (1 << ((slotFast >> 1) - 1)); + for (UInt32 j = 0; j < k; j++, c++) + g_FastPos[c] = slotFast; + } + } +} g_FastPosInit; + + +void CLiteralEncoder2::Encode(NRangeCoder::CEncoder *rangeEncoder, Byte symbol) +{ + UInt32 context = 1; + int i = 8; + do + { + i--; + UInt32 bit = (symbol >> i) & 1; + _encoders[context].Encode(rangeEncoder, bit); + context = (context << 1) | bit; + } + while(i != 0); +} + +void CLiteralEncoder2::EncodeMatched(NRangeCoder::CEncoder *rangeEncoder, + Byte matchByte, Byte symbol) +{ + UInt32 context = 1; + int i = 8; + do + { + i--; + UInt32 bit = (symbol >> i) & 1; + UInt32 matchBit = (matchByte >> i) & 1; + _encoders[0x100 + (matchBit << 8) + context].Encode(rangeEncoder, bit); + context = (context << 1) | bit; + if (matchBit != bit) + { + while(i != 0) + { + i--; + UInt32 bit = (symbol >> i) & 1; + _encoders[context].Encode(rangeEncoder, bit); + context = (context << 1) | bit; + } + break; + } + } + while(i != 0); +} + +UInt32 CLiteralEncoder2::GetPrice(bool matchMode, Byte matchByte, Byte symbol) const +{ + UInt32 price = 0; + UInt32 context = 1; + int i = 8; + if (matchMode) + { + do + { + i--; + UInt32 matchBit = (matchByte >> i) & 1; + UInt32 bit = (symbol >> i) & 1; + price += _encoders[0x100 + (matchBit << 8) + context].GetPrice(bit); + context = (context << 1) | bit; + if (matchBit != bit) + break; + } + while (i != 0); + } + while(i != 0) + { + i--; + UInt32 bit = (symbol >> i) & 1; + price += _encoders[context].GetPrice(bit); + context = (context << 1) | bit; + } + return price; +}; + + +namespace NLength { + +void CEncoder::Init(UInt32 numPosStates) +{ + _choice.Init(); + _choice2.Init(); + for (UInt32 posState = 0; posState < numPosStates; posState++) + { + _lowCoder[posState].Init(); + _midCoder[posState].Init(); + } + _highCoder.Init(); +} + +void CEncoder::Encode(NRangeCoder::CEncoder *rangeEncoder, UInt32 symbol, UInt32 posState) +{ + if(symbol < kNumLowSymbols) + { + _choice.Encode(rangeEncoder, 0); + _lowCoder[posState].Encode(rangeEncoder, symbol); + } + else + { + _choice.Encode(rangeEncoder, 1); + if(symbol < kNumLowSymbols + kNumMidSymbols) + { + _choice2.Encode(rangeEncoder, 0); + _midCoder[posState].Encode(rangeEncoder, symbol - kNumLowSymbols); + } + else + { + _choice2.Encode(rangeEncoder, 1); + _highCoder.Encode(rangeEncoder, symbol - kNumLowSymbols - kNumMidSymbols); + } + } +} + +void CEncoder::SetPrices(UInt32 posState, UInt32 numSymbols, UInt32 *prices) const +{ + UInt32 a0 = _choice.GetPrice0(); + UInt32 a1 = _choice.GetPrice1(); + UInt32 b0 = a1 + _choice2.GetPrice0(); + UInt32 b1 = a1 + _choice2.GetPrice1(); + UInt32 i = 0; + for (i = 0; i < kNumLowSymbols; i++) + { + if (i >= numSymbols) + return; + prices[i] = a0 + _lowCoder[posState].GetPrice(i); + } + for (; i < kNumLowSymbols + kNumMidSymbols; i++) + { + if (i >= numSymbols) + return; + prices[i] = b0 + _midCoder[posState].GetPrice(i - kNumLowSymbols); + } + for (; i < numSymbols; i++) + prices[i] = b1 + _highCoder.GetPrice(i - kNumLowSymbols - kNumMidSymbols); +} + +} +CEncoder::CEncoder(): + _numFastBytes(kNumFastBytesDefault), + _distTableSize(kDefaultDictionaryLogSize * 2), + _posStateBits(2), + _posStateMask(4 - 1), + _numLiteralPosStateBits(0), + _numLiteralContextBits(3), + _dictionarySize(1 << kDefaultDictionaryLogSize), + _dictionarySizePrev(UInt32(-1)), + _numFastBytesPrev(UInt32(-1)), + _matchFinderCycles(0), + _matchFinderIndex(kBT4), + #ifdef COMPRESS_MF_MT + _multiThread(false), + #endif + _writeEndMark(false), + setMfPasses(0) +{ + // _maxMode = false; + _fastMode = false; +} + +HRESULT CEncoder::Create() +{ + if (!_rangeEncoder.Create(1 << 20)) + return E_OUTOFMEMORY; + if (!_matchFinder) + { + switch(_matchFinderIndex) + { + #ifdef COMPRESS_MF_BT + #ifdef COMPRESS_MF_BT2 + case kBT2: + { + NBT2::CMatchFinder *mfSpec = new NBT2::CMatchFinder; + setMfPasses = mfSpec; + _matchFinder = mfSpec; + break; + } + #endif + #ifdef COMPRESS_MF_BT3 + case kBT3: + { + NBT3::CMatchFinder *mfSpec = new NBT3::CMatchFinder; + setMfPasses = mfSpec; + _matchFinder = mfSpec; + break; + } + #endif + #ifdef COMPRESS_MF_BT4 + case kBT4: + { + NBT4::CMatchFinder *mfSpec = new NBT4::CMatchFinder; + setMfPasses = mfSpec; + _matchFinder = mfSpec; + break; + } + #endif + #endif + + #ifdef COMPRESS_MF_HC + case kHC4: + { + NHC4::CMatchFinder *mfSpec = new NHC4::CMatchFinder; + setMfPasses = mfSpec; + _matchFinder = mfSpec; + break; + } + #endif + } + if (_matchFinder == 0) + return E_OUTOFMEMORY; + + #ifdef COMPRESS_MF_MT + if (_multiThread && !(_fastMode && (_matchFinderIndex == kHC4))) + { + CMatchFinderMT *mfSpec = new CMatchFinderMT; + if (mfSpec == 0) + return E_OUTOFMEMORY; + CMyComPtr mf = mfSpec; + RINOK(mfSpec->SetMatchFinder(_matchFinder)); + _matchFinder.Release(); + _matchFinder = mf; + } + #endif + } + + if (!_literalEncoder.Create(_numLiteralPosStateBits, _numLiteralContextBits)) + return E_OUTOFMEMORY; + + if (_dictionarySize == _dictionarySizePrev && _numFastBytesPrev == _numFastBytes) + return S_OK; + RINOK(_matchFinder->Create(_dictionarySize, kNumOpts, _numFastBytes, kMatchMaxLen + 1)); // actually it's + _numFastBytes - _numFastBytes + if (_matchFinderCycles != 0 && setMfPasses != 0) + setMfPasses->SetNumPasses(_matchFinderCycles); + _dictionarySizePrev = _dictionarySize; + _numFastBytesPrev = _numFastBytes; + return S_OK; +} + +static bool AreStringsEqual(const wchar_t *base, const wchar_t *testString) +{ + while (true) + { + wchar_t c = *testString; + if (c >= 'a' && c <= 'z') + c -= 0x20; + if (*base != c) + return false; + if (c == 0) + return true; + base++; + testString++; + } +} + +static int FindMatchFinder(const wchar_t *s) +{ + for (int m = 0; m < (int)(sizeof(kMatchFinderIDs) / sizeof(kMatchFinderIDs[0])); m++) + if (AreStringsEqual(kMatchFinderIDs[m], s)) + return m; + return -1; +} + +STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs, + const PROPVARIANT *properties, UInt32 numProperties) +{ + for (UInt32 i = 0; i < numProperties; i++) + { + const PROPVARIANT &prop = properties[i]; + switch(propIDs[i]) + { + case NCoderPropID::kNumFastBytes: + { + if (prop.vt != VT_UI4) + return E_INVALIDARG; + UInt32 numFastBytes = prop.ulVal; + if(numFastBytes < 5 || numFastBytes > kMatchMaxLen) + return E_INVALIDARG; + _numFastBytes = numFastBytes; + break; + } + case NCoderPropID::kMatchFinderCycles: + { + if (prop.vt != VT_UI4) + return E_INVALIDARG; + _matchFinderCycles = prop.ulVal; + break; + } + case NCoderPropID::kAlgorithm: + { + if (prop.vt != VT_UI4) + return E_INVALIDARG; + UInt32 maximize = prop.ulVal; + _fastMode = (maximize == 0); + // _maxMode = (maximize >= 2); + break; + } + case NCoderPropID::kMatchFinder: + { + if (prop.vt != VT_BSTR) + return E_INVALIDARG; + int matchFinderIndexPrev = _matchFinderIndex; + int m = FindMatchFinder(prop.bstrVal); + if (m < 0) + return E_INVALIDARG; + _matchFinderIndex = m; + if (_matchFinder && matchFinderIndexPrev != _matchFinderIndex) + { + _dictionarySizePrev = (UInt32)-1; + ReleaseMatchFinder(); + } + break; + } + #ifdef COMPRESS_MF_MT + case NCoderPropID::kMultiThread: + { + if (prop.vt != VT_BOOL) + return E_INVALIDARG; + bool newMultiThread = (prop.boolVal == VARIANT_TRUE); + if (newMultiThread != _multiThread) + { + _dictionarySizePrev = (UInt32)-1; + ReleaseMatchFinder(); + _multiThread = newMultiThread; + } + break; + } + case NCoderPropID::kNumThreads: + { + if (prop.vt != VT_UI4) + return E_INVALIDARG; + bool newMultiThread = (prop.ulVal > 1); + if (newMultiThread != _multiThread) + { + _dictionarySizePrev = (UInt32)-1; + ReleaseMatchFinder(); + _multiThread = newMultiThread; + } + break; + } + #endif + case NCoderPropID::kDictionarySize: + { + const int kDicLogSizeMaxCompress = 30; + if (prop.vt != VT_UI4) + return E_INVALIDARG; + UInt32 dictionarySize = prop.ulVal; + if (dictionarySize < UInt32(1 << kDicLogSizeMin) || + dictionarySize > UInt32(1 << kDicLogSizeMaxCompress)) + return E_INVALIDARG; + _dictionarySize = dictionarySize; + UInt32 dicLogSize; + for(dicLogSize = 0; dicLogSize < (UInt32)kDicLogSizeMaxCompress; dicLogSize++) + if (dictionarySize <= (UInt32(1) << dicLogSize)) + break; + _distTableSize = dicLogSize * 2; + break; + } + case NCoderPropID::kPosStateBits: + { + if (prop.vt != VT_UI4) + return E_INVALIDARG; + UInt32 value = prop.ulVal; + if (value > (UInt32)NLength::kNumPosStatesBitsEncodingMax) + return E_INVALIDARG; + _posStateBits = value; + _posStateMask = (1 << _posStateBits) - 1; + break; + } + case NCoderPropID::kLitPosBits: + { + if (prop.vt != VT_UI4) + return E_INVALIDARG; + UInt32 value = prop.ulVal; + if (value > (UInt32)kNumLitPosStatesBitsEncodingMax) + return E_INVALIDARG; + _numLiteralPosStateBits = value; + break; + } + case NCoderPropID::kLitContextBits: + { + if (prop.vt != VT_UI4) + return E_INVALIDARG; + UInt32 value = prop.ulVal; + if (value > (UInt32)kNumLitContextBitsMax) + return E_INVALIDARG; + _numLiteralContextBits = value; + break; + } + case NCoderPropID::kEndMarker: + { + if (prop.vt != VT_BOOL) + return E_INVALIDARG; + SetWriteEndMarkerMode(prop.boolVal == VARIANT_TRUE); + break; + } + default: + return E_INVALIDARG; + } + } + return S_OK; +} + +STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream *outStream) +{ + const UInt32 kPropSize = 5; + Byte properties[kPropSize]; + properties[0] = (_posStateBits * 5 + _numLiteralPosStateBits) * 9 + _numLiteralContextBits; + for (int i = 0; i < 4; i++) + properties[1 + i] = Byte(_dictionarySize >> (8 * i)); + return WriteStream(outStream, properties, kPropSize, NULL); +} + +STDMETHODIMP CEncoder::SetOutStream(ISequentialOutStream *outStream) +{ + _rangeEncoder.SetStream(outStream); + return S_OK; +} + +STDMETHODIMP CEncoder::ReleaseOutStream() +{ + _rangeEncoder.ReleaseStream(); + return S_OK; +} + +HRESULT CEncoder::Init() +{ + CBaseState::Init(); + + // RINOK(_matchFinder->Init(inStream)); + _rangeEncoder.Init(); + + for(int i = 0; i < kNumStates; i++) + { + for (UInt32 j = 0; j <= _posStateMask; j++) + { + _isMatch[i][j].Init(); + _isRep0Long[i][j].Init(); + } + _isRep[i].Init(); + _isRepG0[i].Init(); + _isRepG1[i].Init(); + _isRepG2[i].Init(); + } + + _literalEncoder.Init(); + + { + for(UInt32 i = 0; i < kNumLenToPosStates; i++) + _posSlotEncoder[i].Init(); + } + { + for(UInt32 i = 0; i < kNumFullDistances - kEndPosModelIndex; i++) + _posEncoders[i].Init(); + } + + _lenEncoder.Init(1 << _posStateBits); + _repMatchLenEncoder.Init(1 << _posStateBits); + + _posAlignEncoder.Init(); + + _longestMatchWasFound = false; + _optimumEndIndex = 0; + _optimumCurrentIndex = 0; + _additionalOffset = 0; + + return S_OK; +} + +HRESULT CEncoder::MovePos(UInt32 num) +{ + if (num == 0) + return S_OK; + _additionalOffset += num; + return _matchFinder->Skip(num); +} + +UInt32 CEncoder::Backward(UInt32 &backRes, UInt32 cur) +{ + _optimumEndIndex = cur; + UInt32 posMem = _optimum[cur].PosPrev; + UInt32 backMem = _optimum[cur].BackPrev; + do + { + if (_optimum[cur].Prev1IsChar) + { + _optimum[posMem].MakeAsChar(); + _optimum[posMem].PosPrev = posMem - 1; + if (_optimum[cur].Prev2) + { + _optimum[posMem - 1].Prev1IsChar = false; + _optimum[posMem - 1].PosPrev = _optimum[cur].PosPrev2; + _optimum[posMem - 1].BackPrev = _optimum[cur].BackPrev2; + } + } + UInt32 posPrev = posMem; + UInt32 backCur = backMem; + + backMem = _optimum[posPrev].BackPrev; + posMem = _optimum[posPrev].PosPrev; + + _optimum[posPrev].BackPrev = backCur; + _optimum[posPrev].PosPrev = cur; + cur = posPrev; + } + while(cur != 0); + backRes = _optimum[0].BackPrev; + _optimumCurrentIndex = _optimum[0].PosPrev; + return _optimumCurrentIndex; +} + +/* +Out: + (lenRes == 1) && (backRes == 0xFFFFFFFF) means Literal +*/ + +HRESULT CEncoder::GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes) +{ + if(_optimumEndIndex != _optimumCurrentIndex) + { + const COptimal &optimum = _optimum[_optimumCurrentIndex]; + lenRes = optimum.PosPrev - _optimumCurrentIndex; + backRes = optimum.BackPrev; + _optimumCurrentIndex = optimum.PosPrev; + return S_OK; + } + _optimumCurrentIndex = _optimumEndIndex = 0; + + UInt32 lenMain, numDistancePairs; + if (!_longestMatchWasFound) + { + RINOK(ReadMatchDistances(lenMain, numDistancePairs)); + } + else + { + lenMain = _longestMatchLength; + numDistancePairs = _numDistancePairs; + _longestMatchWasFound = false; + } + + const Byte *data = _matchFinder->GetPointerToCurrentPos() - 1; + UInt32 numAvailableBytes = _matchFinder->GetNumAvailableBytes() + 1; + if (numAvailableBytes < 2) + { + backRes = (UInt32)(-1); + lenRes = 1; + return S_OK; + } + if (numAvailableBytes > kMatchMaxLen) + numAvailableBytes = kMatchMaxLen; + + UInt32 reps[kNumRepDistances]; + UInt32 repLens[kNumRepDistances]; + UInt32 repMaxIndex = 0; + UInt32 i; + for(i = 0; i < kNumRepDistances; i++) + { + reps[i] = _repDistances[i]; + UInt32 backOffset = reps[i] + 1; + if (data[0] != data[(size_t)0 - backOffset] || data[1] != data[(size_t)1 - backOffset]) + { + repLens[i] = 0; + continue; + } + UInt32 lenTest; + for (lenTest = 2; lenTest < numAvailableBytes && + data[lenTest] == data[(size_t)lenTest - backOffset]; lenTest++); + repLens[i] = lenTest; + if (lenTest > repLens[repMaxIndex]) + repMaxIndex = i; + } + if(repLens[repMaxIndex] >= _numFastBytes) + { + backRes = repMaxIndex; + lenRes = repLens[repMaxIndex]; + return MovePos(lenRes - 1); + } + + UInt32 *matchDistances = _matchDistances + 1; + if(lenMain >= _numFastBytes) + { + backRes = matchDistances[numDistancePairs - 1] + kNumRepDistances; + lenRes = lenMain; + return MovePos(lenMain - 1); + } + Byte currentByte = *data; + Byte matchByte = data[(size_t)0 - reps[0] - 1]; + + if(lenMain < 2 && currentByte != matchByte && repLens[repMaxIndex] < 2) + { + backRes = (UInt32)-1; + lenRes = 1; + return S_OK; + } + + _optimum[0].State = _state; + + UInt32 posState = (position & _posStateMask); + + _optimum[1].Price = _isMatch[_state.Index][posState].GetPrice0() + + _literalEncoder.GetSubCoder(position, _previousByte)->GetPrice(!_state.IsCharState(), matchByte, currentByte); + _optimum[1].MakeAsChar(); + + UInt32 matchPrice = _isMatch[_state.Index][posState].GetPrice1(); + UInt32 repMatchPrice = matchPrice + _isRep[_state.Index].GetPrice1(); + + if(matchByte == currentByte) + { + UInt32 shortRepPrice = repMatchPrice + GetRepLen1Price(_state, posState); + if(shortRepPrice < _optimum[1].Price) + { + _optimum[1].Price = shortRepPrice; + _optimum[1].MakeAsShortRep(); + } + } + UInt32 lenEnd = ((lenMain >= repLens[repMaxIndex]) ? lenMain : repLens[repMaxIndex]); + + if(lenEnd < 2) + { + backRes = _optimum[1].BackPrev; + lenRes = 1; + return S_OK; + } + + _optimum[1].PosPrev = 0; + for (i = 0; i < kNumRepDistances; i++) + _optimum[0].Backs[i] = reps[i]; + + UInt32 len = lenEnd; + do + _optimum[len--].Price = kIfinityPrice; + while (len >= 2); + + for(i = 0; i < kNumRepDistances; i++) + { + UInt32 repLen = repLens[i]; + if (repLen < 2) + continue; + UInt32 price = repMatchPrice + GetPureRepPrice(i, _state, posState); + do + { + UInt32 curAndLenPrice = price + _repMatchLenEncoder.GetPrice(repLen - 2, posState); + COptimal &optimum = _optimum[repLen]; + if (curAndLenPrice < optimum.Price) + { + optimum.Price = curAndLenPrice; + optimum.PosPrev = 0; + optimum.BackPrev = i; + optimum.Prev1IsChar = false; + } + } + while(--repLen >= 2); + } + + UInt32 normalMatchPrice = matchPrice + _isRep[_state.Index].GetPrice0(); + + len = ((repLens[0] >= 2) ? repLens[0] + 1 : 2); + if (len <= lenMain) + { + UInt32 offs = 0; + while (len > matchDistances[offs]) + offs += 2; + for(; ; len++) + { + UInt32 distance = matchDistances[offs + 1]; + UInt32 curAndLenPrice = normalMatchPrice + GetPosLenPrice(distance, len, posState); + COptimal &optimum = _optimum[len]; + if (curAndLenPrice < optimum.Price) + { + optimum.Price = curAndLenPrice; + optimum.PosPrev = 0; + optimum.BackPrev = distance + kNumRepDistances; + optimum.Prev1IsChar = false; + } + if (len == matchDistances[offs]) + { + offs += 2; + if (offs == numDistancePairs) + break; + } + } + } + + UInt32 cur = 0; + + while(true) + { + cur++; + if(cur == lenEnd) + { + lenRes = Backward(backRes, cur); + return S_OK; + } + UInt32 newLen, numDistancePairs; + RINOK(ReadMatchDistances(newLen, numDistancePairs)); + if(newLen >= _numFastBytes) + { + _numDistancePairs = numDistancePairs; + _longestMatchLength = newLen; + _longestMatchWasFound = true; + lenRes = Backward(backRes, cur); + return S_OK; + } + position++; + COptimal &curOptimum = _optimum[cur]; + UInt32 posPrev = curOptimum.PosPrev; + CState state; + if (curOptimum.Prev1IsChar) + { + posPrev--; + if (curOptimum.Prev2) + { + state = _optimum[curOptimum.PosPrev2].State; + if (curOptimum.BackPrev2 < kNumRepDistances) + state.UpdateRep(); + else + state.UpdateMatch(); + } + else + state = _optimum[posPrev].State; + state.UpdateChar(); + } + else + state = _optimum[posPrev].State; + if (posPrev == cur - 1) + { + if (curOptimum.IsShortRep()) + state.UpdateShortRep(); + else + state.UpdateChar(); + } + else + { + UInt32 pos; + if (curOptimum.Prev1IsChar && curOptimum.Prev2) + { + posPrev = curOptimum.PosPrev2; + pos = curOptimum.BackPrev2; + state.UpdateRep(); + } + else + { + pos = curOptimum.BackPrev; + if (pos < kNumRepDistances) + state.UpdateRep(); + else + state.UpdateMatch(); + } + const COptimal &prevOptimum = _optimum[posPrev]; + if (pos < kNumRepDistances) + { + reps[0] = prevOptimum.Backs[pos]; + UInt32 i; + for(i = 1; i <= pos; i++) + reps[i] = prevOptimum.Backs[i - 1]; + for(; i < kNumRepDistances; i++) + reps[i] = prevOptimum.Backs[i]; + } + else + { + reps[0] = (pos - kNumRepDistances); + for(UInt32 i = 1; i < kNumRepDistances; i++) + reps[i] = prevOptimum.Backs[i - 1]; + } + } + curOptimum.State = state; + for(UInt32 i = 0; i < kNumRepDistances; i++) + curOptimum.Backs[i] = reps[i]; + UInt32 curPrice = curOptimum.Price; + const Byte *data = _matchFinder->GetPointerToCurrentPos() - 1; + const Byte currentByte = *data; + const Byte matchByte = data[(size_t)0 - reps[0] - 1]; + + UInt32 posState = (position & _posStateMask); + + UInt32 curAnd1Price = curPrice + + _isMatch[state.Index][posState].GetPrice0() + + _literalEncoder.GetSubCoder(position, data[(size_t)0 - 1])->GetPrice(!state.IsCharState(), matchByte, currentByte); + + COptimal &nextOptimum = _optimum[cur + 1]; + + bool nextIsChar = false; + if (curAnd1Price < nextOptimum.Price) + { + nextOptimum.Price = curAnd1Price; + nextOptimum.PosPrev = cur; + nextOptimum.MakeAsChar(); + nextIsChar = true; + } + + UInt32 matchPrice = curPrice + _isMatch[state.Index][posState].GetPrice1(); + UInt32 repMatchPrice = matchPrice + _isRep[state.Index].GetPrice1(); + + if(matchByte == currentByte && + !(nextOptimum.PosPrev < cur && nextOptimum.BackPrev == 0)) + { + UInt32 shortRepPrice = repMatchPrice + GetRepLen1Price(state, posState); + if(shortRepPrice <= nextOptimum.Price) + { + nextOptimum.Price = shortRepPrice; + nextOptimum.PosPrev = cur; + nextOptimum.MakeAsShortRep(); + nextIsChar = true; + } + } + /* + if(newLen == 2 && matchDistances[2] >= kDistLimit2) // test it maybe set 2000 ? + continue; + */ + + UInt32 numAvailableBytesFull = _matchFinder->GetNumAvailableBytes() + 1; + numAvailableBytesFull = MyMin(kNumOpts - 1 - cur, numAvailableBytesFull); + UInt32 numAvailableBytes = numAvailableBytesFull; + + if (numAvailableBytes < 2) + continue; + if (numAvailableBytes > _numFastBytes) + numAvailableBytes = _numFastBytes; + if (!nextIsChar && matchByte != currentByte) // speed optimization + { + // try Literal + rep0 + UInt32 backOffset = reps[0] + 1; + UInt32 limit = MyMin(numAvailableBytesFull, _numFastBytes + 1); + UInt32 temp; + for (temp = 1; temp < limit && + data[temp] == data[(size_t)temp - backOffset]; temp++); + UInt32 lenTest2 = temp - 1; + if (lenTest2 >= 2) + { + CState state2 = state; + state2.UpdateChar(); + UInt32 posStateNext = (position + 1) & _posStateMask; + UInt32 nextRepMatchPrice = curAnd1Price + + _isMatch[state2.Index][posStateNext].GetPrice1() + + _isRep[state2.Index].GetPrice1(); + // for (; lenTest2 >= 2; lenTest2--) + { + UInt32 offset = cur + 1 + lenTest2; + while(lenEnd < offset) + _optimum[++lenEnd].Price = kIfinityPrice; + UInt32 curAndLenPrice = nextRepMatchPrice + GetRepPrice( + 0, lenTest2, state2, posStateNext); + COptimal &optimum = _optimum[offset]; + if (curAndLenPrice < optimum.Price) + { + optimum.Price = curAndLenPrice; + optimum.PosPrev = cur + 1; + optimum.BackPrev = 0; + optimum.Prev1IsChar = true; + optimum.Prev2 = false; + } + } + } + } + + UInt32 startLen = 2; // speed optimization + for(UInt32 repIndex = 0; repIndex < kNumRepDistances; repIndex++) + { + // UInt32 repLen = _matchFinder->GetMatchLen(0 - 1, reps[repIndex], newLen); // test it; + UInt32 backOffset = reps[repIndex] + 1; + if (data[0] != data[(size_t)0 - backOffset] || + data[1] != data[(size_t)1 - backOffset]) + continue; + UInt32 lenTest; + for (lenTest = 2; lenTest < numAvailableBytes && + data[lenTest] == data[(size_t)lenTest - backOffset]; lenTest++); + while(lenEnd < cur + lenTest) + _optimum[++lenEnd].Price = kIfinityPrice; + UInt32 lenTestTemp = lenTest; + UInt32 price = repMatchPrice + GetPureRepPrice(repIndex, state, posState); + do + { + UInt32 curAndLenPrice = price + _repMatchLenEncoder.GetPrice(lenTest - 2, posState); + COptimal &optimum = _optimum[cur + lenTest]; + if (curAndLenPrice < optimum.Price) + { + optimum.Price = curAndLenPrice; + optimum.PosPrev = cur; + optimum.BackPrev = repIndex; + optimum.Prev1IsChar = false; + } + } + while(--lenTest >= 2); + lenTest = lenTestTemp; + + if (repIndex == 0) + startLen = lenTest + 1; + + // if (_maxMode) + { + UInt32 lenTest2 = lenTest + 1; + UInt32 limit = MyMin(numAvailableBytesFull, lenTest2 + _numFastBytes); + for (; lenTest2 < limit && + data[lenTest2] == data[(size_t)lenTest2 - backOffset]; lenTest2++); + lenTest2 -= lenTest + 1; + if (lenTest2 >= 2) + { + CState state2 = state; + state2.UpdateRep(); + UInt32 posStateNext = (position + lenTest) & _posStateMask; + UInt32 curAndLenCharPrice = + price + _repMatchLenEncoder.GetPrice(lenTest - 2, posState) + + _isMatch[state2.Index][posStateNext].GetPrice0() + + _literalEncoder.GetSubCoder(position + lenTest, data[(size_t)lenTest - 1])->GetPrice( + true, data[(size_t)lenTest - backOffset], data[lenTest]); + state2.UpdateChar(); + posStateNext = (position + lenTest + 1) & _posStateMask; + UInt32 nextRepMatchPrice = curAndLenCharPrice + + _isMatch[state2.Index][posStateNext].GetPrice1() + + _isRep[state2.Index].GetPrice1(); + + // for(; lenTest2 >= 2; lenTest2--) + { + UInt32 offset = cur + lenTest + 1 + lenTest2; + while(lenEnd < offset) + _optimum[++lenEnd].Price = kIfinityPrice; + UInt32 curAndLenPrice = nextRepMatchPrice + GetRepPrice( + 0, lenTest2, state2, posStateNext); + COptimal &optimum = _optimum[offset]; + if (curAndLenPrice < optimum.Price) + { + optimum.Price = curAndLenPrice; + optimum.PosPrev = cur + lenTest + 1; + optimum.BackPrev = 0; + optimum.Prev1IsChar = true; + optimum.Prev2 = true; + optimum.PosPrev2 = cur; + optimum.BackPrev2 = repIndex; + } + } + } + } + } + + // for(UInt32 lenTest = 2; lenTest <= newLen; lenTest++) + if (newLen > numAvailableBytes) + { + newLen = numAvailableBytes; + for (numDistancePairs = 0; newLen > matchDistances[numDistancePairs]; numDistancePairs += 2); + matchDistances[numDistancePairs] = newLen; + numDistancePairs += 2; + } + if (newLen >= startLen) + { + UInt32 normalMatchPrice = matchPrice + _isRep[state.Index].GetPrice0(); + while(lenEnd < cur + newLen) + _optimum[++lenEnd].Price = kIfinityPrice; + + UInt32 offs = 0; + while(startLen > matchDistances[offs]) + offs += 2; + UInt32 curBack = matchDistances[offs + 1]; + UInt32 posSlot = GetPosSlot2(curBack); + for(UInt32 lenTest = /*2*/ startLen; ; lenTest++) + { + UInt32 curAndLenPrice = normalMatchPrice; + UInt32 lenToPosState = GetLenToPosState(lenTest); + if (curBack < kNumFullDistances) + curAndLenPrice += _distancesPrices[lenToPosState][curBack]; + else + curAndLenPrice += _posSlotPrices[lenToPosState][posSlot] + _alignPrices[curBack & kAlignMask]; + + curAndLenPrice += _lenEncoder.GetPrice(lenTest - kMatchMinLen, posState); + + COptimal &optimum = _optimum[cur + lenTest]; + if (curAndLenPrice < optimum.Price) + { + optimum.Price = curAndLenPrice; + optimum.PosPrev = cur; + optimum.BackPrev = curBack + kNumRepDistances; + optimum.Prev1IsChar = false; + } + + if (/*_maxMode && */lenTest == matchDistances[offs]) + { + // Try Match + Literal + Rep0 + UInt32 backOffset = curBack + 1; + UInt32 lenTest2 = lenTest + 1; + UInt32 limit = MyMin(numAvailableBytesFull, lenTest2 + _numFastBytes); + for (; lenTest2 < limit && + data[lenTest2] == data[(size_t)lenTest2 - backOffset]; lenTest2++); + lenTest2 -= lenTest + 1; + if (lenTest2 >= 2) + { + CState state2 = state; + state2.UpdateMatch(); + UInt32 posStateNext = (position + lenTest) & _posStateMask; + UInt32 curAndLenCharPrice = curAndLenPrice + + _isMatch[state2.Index][posStateNext].GetPrice0() + + _literalEncoder.GetSubCoder(position + lenTest, data[(size_t)lenTest - 1])->GetPrice( + true, data[(size_t)lenTest - backOffset], data[lenTest]); + state2.UpdateChar(); + posStateNext = (posStateNext + 1) & _posStateMask; + UInt32 nextRepMatchPrice = curAndLenCharPrice + + _isMatch[state2.Index][posStateNext].GetPrice1() + + _isRep[state2.Index].GetPrice1(); + + // for(; lenTest2 >= 2; lenTest2--) + { + UInt32 offset = cur + lenTest + 1 + lenTest2; + while(lenEnd < offset) + _optimum[++lenEnd].Price = kIfinityPrice; + UInt32 curAndLenPrice = nextRepMatchPrice + GetRepPrice(0, lenTest2, state2, posStateNext); + COptimal &optimum = _optimum[offset]; + if (curAndLenPrice < optimum.Price) + { + optimum.Price = curAndLenPrice; + optimum.PosPrev = cur + lenTest + 1; + optimum.BackPrev = 0; + optimum.Prev1IsChar = true; + optimum.Prev2 = true; + optimum.PosPrev2 = cur; + optimum.BackPrev2 = curBack + kNumRepDistances; + } + } + } + offs += 2; + if (offs == numDistancePairs) + break; + curBack = matchDistances[offs + 1]; + if (curBack >= kNumFullDistances) + posSlot = GetPosSlot2(curBack); + } + } + } + } +} + +static inline bool ChangePair(UInt32 smallDist, UInt32 bigDist) +{ + return ((bigDist >> 7) > smallDist); +} + + +HRESULT CEncoder::ReadMatchDistances(UInt32 &lenRes, UInt32 &numDistancePairs) +{ + lenRes = 0; + RINOK(_matchFinder->GetMatches(_matchDistances)); + numDistancePairs = _matchDistances[0]; + if (numDistancePairs > 0) + { + lenRes = _matchDistances[1 + numDistancePairs - 2]; + if (lenRes == _numFastBytes) + lenRes += _matchFinder->GetMatchLen(lenRes - 1, _matchDistances[1 + numDistancePairs - 1], + kMatchMaxLen - lenRes); + } + _additionalOffset++; + return S_OK; +} + +HRESULT CEncoder::GetOptimumFast(UInt32 position, UInt32 &backRes, UInt32 &lenRes) +{ + UInt32 lenMain, numDistancePairs; + if (!_longestMatchWasFound) + { + RINOK(ReadMatchDistances(lenMain, numDistancePairs)); + } + else + { + lenMain = _longestMatchLength; + numDistancePairs = _numDistancePairs; + _longestMatchWasFound = false; + } + + const Byte *data = _matchFinder->GetPointerToCurrentPos() - 1; + UInt32 numAvailableBytes = _matchFinder->GetNumAvailableBytes() + 1; + if (numAvailableBytes > kMatchMaxLen) + numAvailableBytes = kMatchMaxLen; + if (numAvailableBytes < 2) + { + backRes = (UInt32)(-1); + lenRes = 1; + return S_OK; + } + + UInt32 repLens[kNumRepDistances]; + UInt32 repMaxIndex = 0; + + for(UInt32 i = 0; i < kNumRepDistances; i++) + { + UInt32 backOffset = _repDistances[i] + 1; + if (data[0] != data[(size_t)0 - backOffset] || data[1] != data[(size_t)1 - backOffset]) + { + repLens[i] = 0; + continue; + } + UInt32 len; + for (len = 2; len < numAvailableBytes && data[len] == data[(size_t)len - backOffset]; len++); + if(len >= _numFastBytes) + { + backRes = i; + lenRes = len; + return MovePos(lenRes - 1); + } + repLens[i] = len; + if (len > repLens[repMaxIndex]) + repMaxIndex = i; + } + UInt32 *matchDistances = _matchDistances + 1; + if(lenMain >= _numFastBytes) + { + backRes = matchDistances[numDistancePairs - 1] + kNumRepDistances; + lenRes = lenMain; + return MovePos(lenMain - 1); + } + + UInt32 backMain = 0; // for GCC + if (lenMain >= 2) + { + backMain = matchDistances[numDistancePairs - 1]; + while (numDistancePairs > 2 && lenMain == matchDistances[numDistancePairs - 4] + 1) + { + if (!ChangePair(matchDistances[numDistancePairs - 3], backMain)) + break; + numDistancePairs -= 2; + lenMain = matchDistances[numDistancePairs - 2]; + backMain = matchDistances[numDistancePairs - 1]; + } + if (lenMain == 2 && backMain >= 0x80) + lenMain = 1; + } + + if (repLens[repMaxIndex] >= 2) + { + if (repLens[repMaxIndex] + 1 >= lenMain || + (repLens[repMaxIndex] + 2 >= lenMain && (backMain > (1 << 9))) || + (repLens[repMaxIndex] + 3 >= lenMain && (backMain > (1 << 15)))) + { + backRes = repMaxIndex; + lenRes = repLens[repMaxIndex]; + return MovePos(lenRes - 1); + } + } + + if (lenMain >= 2 && numAvailableBytes > 2) + { + RINOK(ReadMatchDistances(_longestMatchLength, _numDistancePairs)); + if (_longestMatchLength >= 2) + { + UInt32 newDistance = matchDistances[_numDistancePairs - 1]; + if ((_longestMatchLength >= lenMain && newDistance < backMain) || + (_longestMatchLength == lenMain + 1 && !ChangePair(backMain, newDistance)) || + _longestMatchLength > lenMain + 1 || + (_longestMatchLength + 1 >= lenMain && lenMain >= 3 && ChangePair(newDistance, backMain))) + { + _longestMatchWasFound = true; + backRes = UInt32(-1); + lenRes = 1; + return S_OK; + } + } + data++; + numAvailableBytes--; + for(UInt32 i = 0; i < kNumRepDistances; i++) + { + UInt32 backOffset = _repDistances[i] + 1; + if (data[1] != data[(size_t)1 - backOffset] || data[2] != data[(size_t)2 - backOffset]) + { + repLens[i] = 0; + continue; + } + UInt32 len; + for (len = 2; len < numAvailableBytes && data[len] == data[(size_t)len - backOffset]; len++); + if (len + 1 >= lenMain) + { + _longestMatchWasFound = true; + backRes = UInt32(-1); + lenRes = 1; + return S_OK; + } + } + backRes = backMain + kNumRepDistances; + lenRes = lenMain; + return MovePos(lenMain - 2); + } + backRes = UInt32(-1); + lenRes = 1; + return S_OK; +} + +HRESULT CEncoder::Flush(UInt32 nowPos) +{ + ReleaseMFStream(); + WriteEndMarker(nowPos & _posStateMask); + _rangeEncoder.FlushData(); + return _rangeEncoder.FlushStream(); +} + +void CEncoder::WriteEndMarker(UInt32 posState) +{ + // This function for writing End Mark for stream version of LZMA. + // In current version this feature is not used. + if (!_writeEndMark) + return; + + _isMatch[_state.Index][posState].Encode(&_rangeEncoder, 1); + _isRep[_state.Index].Encode(&_rangeEncoder, 0); + _state.UpdateMatch(); + UInt32 len = kMatchMinLen; // kMatchMaxLen; + _lenEncoder.Encode(&_rangeEncoder, len - kMatchMinLen, posState, !_fastMode); + UInt32 posSlot = (1 << kNumPosSlotBits) - 1; + UInt32 lenToPosState = GetLenToPosState(len); + _posSlotEncoder[lenToPosState].Encode(&_rangeEncoder, posSlot); + UInt32 footerBits = 30; + UInt32 posReduced = (UInt32(1) << footerBits) - 1; + _rangeEncoder.EncodeDirectBits(posReduced >> kNumAlignBits, footerBits - kNumAlignBits); + _posAlignEncoder.ReverseEncode(&_rangeEncoder, posReduced & kAlignMask); +} + +HRESULT CEncoder::CodeReal(ISequentialInStream *inStream, + ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, + ICompressProgressInfo *progress) +{ + _needReleaseMFStream = false; + CCoderReleaser coderReleaser(this); + RINOK(SetStreams(inStream, outStream, inSize, outSize)); + while(true) + { + UInt64 processedInSize; + UInt64 processedOutSize; + Int32 finished; + RINOK(CodeOneBlock(&processedInSize, &processedOutSize, &finished)); + if (finished != 0) + return S_OK; + if (progress != 0) + { + RINOK(progress->SetRatioInfo(&processedInSize, &processedOutSize)); + } + } +} + +HRESULT CEncoder::SetStreams(ISequentialInStream *inStream, + ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize) +{ + _inStream = inStream; + _finished = false; + RINOK(Create()); + RINOK(SetOutStream(outStream)); + RINOK(Init()); + + // CCoderReleaser releaser(this); + + /* + if (_matchFinder->GetNumAvailableBytes() == 0) + return Flush(); + */ + + if (!_fastMode) + { + FillDistancesPrices(); + FillAlignPrices(); + } + + _lenEncoder.SetTableSize(_numFastBytes + 1 - kMatchMinLen); + _lenEncoder.UpdateTables(1 << _posStateBits); + _repMatchLenEncoder.SetTableSize(_numFastBytes + 1 - kMatchMinLen); + _repMatchLenEncoder.UpdateTables(1 << _posStateBits); + + nowPos64 = 0; + return S_OK; +} + +HRESULT CEncoder::CodeOneBlock(UInt64 *inSize, UInt64 *outSize, Int32 *finished) +{ + if (_inStream != 0) + { + RINOK(_matchFinder->SetStream(_inStream)); + RINOK(_matchFinder->Init()); + _needReleaseMFStream = true; + _inStream = 0; + } + + + *finished = 1; + if (_finished) + return S_OK; + _finished = true; + + if (nowPos64 == 0) + { + if (_matchFinder->GetNumAvailableBytes() == 0) + return Flush(UInt32(nowPos64)); + UInt32 len, numDistancePairs; + RINOK(ReadMatchDistances(len, numDistancePairs)); + UInt32 posState = UInt32(nowPos64) & _posStateMask; + _isMatch[_state.Index][posState].Encode(&_rangeEncoder, 0); + _state.UpdateChar(); + Byte curByte = _matchFinder->GetIndexByte(0 - _additionalOffset); + _literalEncoder.GetSubCoder(UInt32(nowPos64), _previousByte)->Encode(&_rangeEncoder, curByte); + _previousByte = curByte; + _additionalOffset--; + nowPos64++; + } + + UInt32 nowPos32 = (UInt32)nowPos64; + UInt32 progressPosValuePrev = nowPos32; + + if (_matchFinder->GetNumAvailableBytes() == 0) + return Flush(nowPos32); + + while(true) + { + #ifdef _NO_EXCEPTIONS + if (_rangeEncoder.Stream.ErrorCode != S_OK) + return _rangeEncoder.Stream.ErrorCode; + #endif + UInt32 pos, len; + HRESULT result; + if (_fastMode) + result = GetOptimumFast(nowPos32, pos, len); + else + result = GetOptimum(nowPos32, pos, len); + RINOK(result); + + UInt32 posState = nowPos32 & _posStateMask; + if(len == 1 && pos == 0xFFFFFFFF) + { + _isMatch[_state.Index][posState].Encode(&_rangeEncoder, 0); + Byte curByte = _matchFinder->GetIndexByte(0 - _additionalOffset); + CLiteralEncoder2 *subCoder = _literalEncoder.GetSubCoder(nowPos32, _previousByte); + if(_state.IsCharState()) + subCoder->Encode(&_rangeEncoder, curByte); + else + { + Byte matchByte = _matchFinder->GetIndexByte(0 - _repDistances[0] - 1 - _additionalOffset); + subCoder->EncodeMatched(&_rangeEncoder, matchByte, curByte); + } + _state.UpdateChar(); + _previousByte = curByte; + } + else + { + _isMatch[_state.Index][posState].Encode(&_rangeEncoder, 1); + if(pos < kNumRepDistances) + { + _isRep[_state.Index].Encode(&_rangeEncoder, 1); + if(pos == 0) + { + _isRepG0[_state.Index].Encode(&_rangeEncoder, 0); + _isRep0Long[_state.Index][posState].Encode(&_rangeEncoder, ((len == 1) ? 0 : 1)); + } + else + { + UInt32 distance = _repDistances[pos]; + _isRepG0[_state.Index].Encode(&_rangeEncoder, 1); + if (pos == 1) + _isRepG1[_state.Index].Encode(&_rangeEncoder, 0); + else + { + _isRepG1[_state.Index].Encode(&_rangeEncoder, 1); + _isRepG2[_state.Index].Encode(&_rangeEncoder, pos - 2); + if (pos == 3) + _repDistances[3] = _repDistances[2]; + _repDistances[2] = _repDistances[1]; + } + _repDistances[1] = _repDistances[0]; + _repDistances[0] = distance; + } + if (len == 1) + _state.UpdateShortRep(); + else + { + _repMatchLenEncoder.Encode(&_rangeEncoder, len - kMatchMinLen, posState, !_fastMode); + _state.UpdateRep(); + } + } + else + { + _isRep[_state.Index].Encode(&_rangeEncoder, 0); + _state.UpdateMatch(); + _lenEncoder.Encode(&_rangeEncoder, len - kMatchMinLen, posState, !_fastMode); + pos -= kNumRepDistances; + UInt32 posSlot = GetPosSlot(pos); + _posSlotEncoder[GetLenToPosState(len)].Encode(&_rangeEncoder, posSlot); + + if (posSlot >= kStartPosModelIndex) + { + UInt32 footerBits = ((posSlot >> 1) - 1); + UInt32 base = ((2 | (posSlot & 1)) << footerBits); + UInt32 posReduced = pos - base; + + if (posSlot < kEndPosModelIndex) + NRangeCoder::ReverseBitTreeEncode(_posEncoders + base - posSlot - 1, + &_rangeEncoder, footerBits, posReduced); + else + { + _rangeEncoder.EncodeDirectBits(posReduced >> kNumAlignBits, footerBits - kNumAlignBits); + _posAlignEncoder.ReverseEncode(&_rangeEncoder, posReduced & kAlignMask); + _alignPriceCount++; + } + } + _repDistances[3] = _repDistances[2]; + _repDistances[2] = _repDistances[1]; + _repDistances[1] = _repDistances[0]; + _repDistances[0] = pos; + _matchPriceCount++; + } + _previousByte = _matchFinder->GetIndexByte(len - 1 - _additionalOffset); + } + _additionalOffset -= len; + nowPos32 += len; + if (_additionalOffset == 0) + { + if (!_fastMode) + { + if (_matchPriceCount >= (1 << 7)) + FillDistancesPrices(); + if (_alignPriceCount >= kAlignTableSize) + FillAlignPrices(); + } + if (_matchFinder->GetNumAvailableBytes() == 0) + return Flush(nowPos32); + if (nowPos32 - progressPosValuePrev >= (1 << 14)) + { + nowPos64 += nowPos32 - progressPosValuePrev; + *inSize = nowPos64; + *outSize = _rangeEncoder.GetProcessedSize(); + _finished = false; + *finished = 0; + return S_OK; + } + } + } +} + +STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, + ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, + ICompressProgressInfo *progress) +{ + #ifndef _NO_EXCEPTIONS + try + { + #endif + return CodeReal(inStream, outStream, inSize, outSize, progress); + #ifndef _NO_EXCEPTIONS + } + catch(const COutBufferException &e) { return e.ErrorCode; } + catch(...) { return E_FAIL; } + #endif +} + +void CEncoder::FillDistancesPrices() +{ + UInt32 tempPrices[kNumFullDistances]; + for (UInt32 i = kStartPosModelIndex; i < kNumFullDistances; i++) + { + UInt32 posSlot = GetPosSlot(i); + UInt32 footerBits = ((posSlot >> 1) - 1); + UInt32 base = ((2 | (posSlot & 1)) << footerBits); + tempPrices[i] = NRangeCoder::ReverseBitTreeGetPrice(_posEncoders + + base - posSlot - 1, footerBits, i - base); + } + + for (UInt32 lenToPosState = 0; lenToPosState < kNumLenToPosStates; lenToPosState++) + { + UInt32 posSlot; + NRangeCoder::CBitTreeEncoder &encoder = _posSlotEncoder[lenToPosState]; + UInt32 *posSlotPrices = _posSlotPrices[lenToPosState]; + for (posSlot = 0; posSlot < _distTableSize; posSlot++) + posSlotPrices[posSlot] = encoder.GetPrice(posSlot); + for (posSlot = kEndPosModelIndex; posSlot < _distTableSize; posSlot++) + posSlotPrices[posSlot] += ((((posSlot >> 1) - 1) - kNumAlignBits) << NRangeCoder::kNumBitPriceShiftBits); + + UInt32 *distancesPrices = _distancesPrices[lenToPosState]; + UInt32 i; + for (i = 0; i < kStartPosModelIndex; i++) + distancesPrices[i] = posSlotPrices[i]; + for (; i < kNumFullDistances; i++) + distancesPrices[i] = posSlotPrices[GetPosSlot(i)] + tempPrices[i]; + } + _matchPriceCount = 0; +} + +void CEncoder::FillAlignPrices() +{ + for (UInt32 i = 0; i < kAlignTableSize; i++) + _alignPrices[i] = _posAlignEncoder.ReverseGetPrice(i); + _alignPriceCount = 0; +} + +}} diff --git a/lzma443/C/7zip/Compress/LZMA/LZMAEncoder.h b/lzma443/C/7zip/Compress/LZMA/LZMAEncoder.h new file mode 100755 index 0000000..f4c2c15 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA/LZMAEncoder.h @@ -0,0 +1,411 @@ +// LZMA/Encoder.h + +#ifndef __LZMA_ENCODER_H +#define __LZMA_ENCODER_H + +#include "../../../Common/MyCom.h" +#include "../../../Common/Alloc.h" +#include "../../ICoder.h" +#include "../LZ/IMatchFinder.h" +#include "../RangeCoder/RangeCoderBitTree.h" + +#include "LZMA.h" + +namespace NCompress { +namespace NLZMA { + +typedef NRangeCoder::CBitEncoder CMyBitEncoder; + +class CBaseState +{ +protected: + CState _state; + Byte _previousByte; + UInt32 _repDistances[kNumRepDistances]; + void Init() + { + _state.Init(); + _previousByte = 0; + for(UInt32 i = 0 ; i < kNumRepDistances; i++) + _repDistances[i] = 0; + } +}; + +struct COptimal +{ + CState State; + + bool Prev1IsChar; + bool Prev2; + + UInt32 PosPrev2; + UInt32 BackPrev2; + + UInt32 Price; + UInt32 PosPrev; // posNext; + UInt32 BackPrev; + UInt32 Backs[kNumRepDistances]; + void MakeAsChar() { BackPrev = UInt32(-1); Prev1IsChar = false; } + void MakeAsShortRep() { BackPrev = 0; ; Prev1IsChar = false; } + bool IsShortRep() { return (BackPrev == 0); } +}; + + +extern Byte g_FastPos[1 << 11]; +inline UInt32 GetPosSlot(UInt32 pos) +{ + if (pos < (1 << 11)) + return g_FastPos[pos]; + if (pos < (1 << 21)) + return g_FastPos[pos >> 10] + 20; + return g_FastPos[pos >> 20] + 40; +} + +inline UInt32 GetPosSlot2(UInt32 pos) +{ + if (pos < (1 << 17)) + return g_FastPos[pos >> 6] + 12; + if (pos < (1 << 27)) + return g_FastPos[pos >> 16] + 32; + return g_FastPos[pos >> 26] + 52; +} + +const UInt32 kIfinityPrice = 0xFFFFFFF; + +const UInt32 kNumOpts = 1 << 12; + + +class CLiteralEncoder2 +{ + CMyBitEncoder _encoders[0x300]; +public: + void Init() + { + for (int i = 0; i < 0x300; i++) + _encoders[i].Init(); + } + void Encode(NRangeCoder::CEncoder *rangeEncoder, Byte symbol); + void EncodeMatched(NRangeCoder::CEncoder *rangeEncoder, Byte matchByte, Byte symbol); + UInt32 GetPrice(bool matchMode, Byte matchByte, Byte symbol) const; +}; + +class CLiteralEncoder +{ + CLiteralEncoder2 *_coders; + int _numPrevBits; + int _numPosBits; + UInt32 _posMask; +public: + CLiteralEncoder(): _coders(0) {} + ~CLiteralEncoder() { Free(); } + void Free() + { + MyFree(_coders); + _coders = 0; + } + bool Create(int numPosBits, int numPrevBits) + { + if (_coders == 0 || (numPosBits + numPrevBits) != (_numPrevBits + _numPosBits)) + { + Free(); + UInt32 numStates = 1 << (numPosBits + numPrevBits); + _coders = (CLiteralEncoder2 *)MyAlloc(numStates * sizeof(CLiteralEncoder2)); + } + _numPosBits = numPosBits; + _posMask = (1 << numPosBits) - 1; + _numPrevBits = numPrevBits; + return (_coders != 0); + } + void Init() + { + UInt32 numStates = 1 << (_numPrevBits + _numPosBits); + for (UInt32 i = 0; i < numStates; i++) + _coders[i].Init(); + } + CLiteralEncoder2 *GetSubCoder(UInt32 pos, Byte prevByte) + { return &_coders[((pos & _posMask) << _numPrevBits) + (prevByte >> (8 - _numPrevBits))]; } +}; + +namespace NLength { + +class CEncoder +{ + CMyBitEncoder _choice; + CMyBitEncoder _choice2; + NRangeCoder::CBitTreeEncoder _lowCoder[kNumPosStatesEncodingMax]; + NRangeCoder::CBitTreeEncoder _midCoder[kNumPosStatesEncodingMax]; + NRangeCoder::CBitTreeEncoder _highCoder; +public: + void Init(UInt32 numPosStates); + void Encode(NRangeCoder::CEncoder *rangeEncoder, UInt32 symbol, UInt32 posState); + void SetPrices(UInt32 posState, UInt32 numSymbols, UInt32 *prices) const; +}; + +const UInt32 kNumSpecSymbols = kNumLowSymbols + kNumMidSymbols; + +class CPriceTableEncoder: public CEncoder +{ + UInt32 _prices[kNumPosStatesEncodingMax][kNumSymbolsTotal]; + UInt32 _tableSize; + UInt32 _counters[kNumPosStatesEncodingMax]; +public: + void SetTableSize(UInt32 tableSize) { _tableSize = tableSize; } + UInt32 GetPrice(UInt32 symbol, UInt32 posState) const { return _prices[posState][symbol]; } + void UpdateTable(UInt32 posState) + { + SetPrices(posState, _tableSize, _prices[posState]); + _counters[posState] = _tableSize; + } + void UpdateTables(UInt32 numPosStates) + { + for (UInt32 posState = 0; posState < numPosStates; posState++) + UpdateTable(posState); + } + void Encode(NRangeCoder::CEncoder *rangeEncoder, UInt32 symbol, UInt32 posState, bool updatePrice) + { + CEncoder::Encode(rangeEncoder, symbol, posState); + if (updatePrice) + if (--_counters[posState] == 0) + UpdateTable(posState); + } +}; + +} + +class CEncoder : + public ICompressCoder, + public ICompressSetOutStream, + public ICompressSetCoderProperties, + public ICompressWriteCoderProperties, + public CBaseState, + public CMyUnknownImp +{ + COptimal _optimum[kNumOpts]; + CMyComPtr _matchFinder; // test it + NRangeCoder::CEncoder _rangeEncoder; + + CMyBitEncoder _isMatch[kNumStates][NLength::kNumPosStatesEncodingMax]; + CMyBitEncoder _isRep[kNumStates]; + CMyBitEncoder _isRepG0[kNumStates]; + CMyBitEncoder _isRepG1[kNumStates]; + CMyBitEncoder _isRepG2[kNumStates]; + CMyBitEncoder _isRep0Long[kNumStates][NLength::kNumPosStatesEncodingMax]; + + NRangeCoder::CBitTreeEncoder _posSlotEncoder[kNumLenToPosStates]; + + CMyBitEncoder _posEncoders[kNumFullDistances - kEndPosModelIndex]; + NRangeCoder::CBitTreeEncoder _posAlignEncoder; + + NLength::CPriceTableEncoder _lenEncoder; + NLength::CPriceTableEncoder _repMatchLenEncoder; + + CLiteralEncoder _literalEncoder; + + UInt32 _matchDistances[kMatchMaxLen * 2 + 2 + 1]; + + bool _fastMode; + // bool _maxMode; + UInt32 _numFastBytes; + UInt32 _longestMatchLength; + UInt32 _numDistancePairs; + + UInt32 _additionalOffset; + + UInt32 _optimumEndIndex; + UInt32 _optimumCurrentIndex; + + bool _longestMatchWasFound; + + UInt32 _posSlotPrices[kNumLenToPosStates][kDistTableSizeMax]; + + UInt32 _distancesPrices[kNumLenToPosStates][kNumFullDistances]; + + UInt32 _alignPrices[kAlignTableSize]; + UInt32 _alignPriceCount; + + UInt32 _distTableSize; + + UInt32 _posStateBits; + UInt32 _posStateMask; + UInt32 _numLiteralPosStateBits; + UInt32 _numLiteralContextBits; + + UInt32 _dictionarySize; + + UInt32 _dictionarySizePrev; + UInt32 _numFastBytesPrev; + + UInt32 _matchPriceCount; + UInt64 nowPos64; + bool _finished; + ISequentialInStream *_inStream; + + UInt32 _matchFinderCycles; + int _matchFinderIndex; + #ifdef COMPRESS_MF_MT + bool _multiThread; + #endif + + bool _writeEndMark; + + bool _needReleaseMFStream; + + IMatchFinderSetNumPasses *setMfPasses; + + void ReleaseMatchFinder() + { + setMfPasses = 0; + _matchFinder.Release(); + } + + HRESULT ReadMatchDistances(UInt32 &len, UInt32 &numDistancePairs); + + HRESULT MovePos(UInt32 num); + UInt32 GetRepLen1Price(CState state, UInt32 posState) const + { + return _isRepG0[state.Index].GetPrice0() + + _isRep0Long[state.Index][posState].GetPrice0(); + } + + UInt32 GetPureRepPrice(UInt32 repIndex, CState state, UInt32 posState) const + { + UInt32 price; + if(repIndex == 0) + { + price = _isRepG0[state.Index].GetPrice0(); + price += _isRep0Long[state.Index][posState].GetPrice1(); + } + else + { + price = _isRepG0[state.Index].GetPrice1(); + if (repIndex == 1) + price += _isRepG1[state.Index].GetPrice0(); + else + { + price += _isRepG1[state.Index].GetPrice1(); + price += _isRepG2[state.Index].GetPrice(repIndex - 2); + } + } + return price; + } + UInt32 GetRepPrice(UInt32 repIndex, UInt32 len, CState state, UInt32 posState) const + { + return _repMatchLenEncoder.GetPrice(len - kMatchMinLen, posState) + + GetPureRepPrice(repIndex, state, posState); + } + /* + UInt32 GetPosLen2Price(UInt32 pos, UInt32 posState) const + { + if (pos >= kNumFullDistances) + return kIfinityPrice; + return _distancesPrices[0][pos] + _lenEncoder.GetPrice(0, posState); + } + UInt32 GetPosLen3Price(UInt32 pos, UInt32 len, UInt32 posState) const + { + UInt32 price; + UInt32 lenToPosState = GetLenToPosState(len); + if (pos < kNumFullDistances) + price = _distancesPrices[lenToPosState][pos]; + else + price = _posSlotPrices[lenToPosState][GetPosSlot2(pos)] + + _alignPrices[pos & kAlignMask]; + return price + _lenEncoder.GetPrice(len - kMatchMinLen, posState); + } + */ + UInt32 GetPosLenPrice(UInt32 pos, UInt32 len, UInt32 posState) const + { + UInt32 price; + UInt32 lenToPosState = GetLenToPosState(len); + if (pos < kNumFullDistances) + price = _distancesPrices[lenToPosState][pos]; + else + price = _posSlotPrices[lenToPosState][GetPosSlot2(pos)] + + _alignPrices[pos & kAlignMask]; + return price + _lenEncoder.GetPrice(len - kMatchMinLen, posState); + } + + UInt32 Backward(UInt32 &backRes, UInt32 cur); + HRESULT GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes); + HRESULT GetOptimumFast(UInt32 position, UInt32 &backRes, UInt32 &lenRes); + + void FillDistancesPrices(); + void FillAlignPrices(); + + void ReleaseMFStream() + { + if (_matchFinder && _needReleaseMFStream) + { + _matchFinder->ReleaseStream(); + _needReleaseMFStream = false; + } + } + + void ReleaseStreams() + { + ReleaseMFStream(); + ReleaseOutStream(); + } + + HRESULT Flush(UInt32 nowPos); + class CCoderReleaser + { + CEncoder *_coder; + public: + CCoderReleaser(CEncoder *coder): _coder(coder) {} + ~CCoderReleaser() + { + _coder->ReleaseStreams(); + } + }; + friend class CCoderReleaser; + + void WriteEndMarker(UInt32 posState); + +public: + CEncoder(); + void SetWriteEndMarkerMode(bool writeEndMarker) + { _writeEndMark= writeEndMarker; } + + HRESULT Create(); + + MY_UNKNOWN_IMP3( + ICompressSetOutStream, + ICompressSetCoderProperties, + ICompressWriteCoderProperties + ) + + HRESULT Init(); + + // ICompressCoder interface + HRESULT SetStreams(ISequentialInStream *inStream, + ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize); + HRESULT CodeOneBlock(UInt64 *inSize, UInt64 *outSize, Int32 *finished); + + HRESULT CodeReal(ISequentialInStream *inStream, + ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, + ICompressProgressInfo *progress); + + // ICompressCoder interface + STDMETHOD(Code)(ISequentialInStream *inStream, + ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, + ICompressProgressInfo *progress); + + // ICompressSetCoderProperties2 + STDMETHOD(SetCoderProperties)(const PROPID *propIDs, + const PROPVARIANT *properties, UInt32 numProperties); + + // ICompressWriteCoderProperties + STDMETHOD(WriteCoderProperties)(ISequentialOutStream *outStream); + + STDMETHOD(SetOutStream)(ISequentialOutStream *outStream); + STDMETHOD(ReleaseOutStream)(); + + virtual ~CEncoder() {} +}; + +}} + +#endif diff --git a/lzma443/C/7zip/Compress/LZMA/StdAfx.h b/lzma443/C/7zip/Compress/LZMA/StdAfx.h new file mode 100755 index 0000000..e7fb698 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA/StdAfx.h @@ -0,0 +1,8 @@ +// StdAfx.h + +#ifndef __STDAFX_H +#define __STDAFX_H + +#include "../../../Common/MyWindows.h" + +#endif diff --git a/lzma443/C/7zip/Compress/LZMA_Alone/AloneLZMA.dsp b/lzma443/C/7zip/Compress/LZMA_Alone/AloneLZMA.dsp new file mode 100755 index 0000000..9e6b667 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA_Alone/AloneLZMA.dsp @@ -0,0 +1,475 @@ +# Microsoft Developer Studio Project File - Name="AloneLZMA" - Package Owner=<4> +# Microsoft Developer Studio Generated Build File, Format Version 6.00 +# ** DO NOT EDIT ** + +# TARGTYPE "Win32 (x86) Console Application" 0x0103 + +CFG=AloneLZMA - Win32 DebugU +!MESSAGE This is not a valid makefile. To build this project using NMAKE, +!MESSAGE use the Export Makefile command and run +!MESSAGE +!MESSAGE NMAKE /f "AloneLZMA.mak". +!MESSAGE +!MESSAGE You can specify a configuration when running NMAKE +!MESSAGE by defining the macro CFG on the command line. For example: +!MESSAGE +!MESSAGE NMAKE /f "AloneLZMA.mak" CFG="AloneLZMA - Win32 DebugU" +!MESSAGE +!MESSAGE Possible choices for configuration are: +!MESSAGE +!MESSAGE "AloneLZMA - Win32 Release" (based on "Win32 (x86) Console Application") +!MESSAGE "AloneLZMA - Win32 Debug" (based on "Win32 (x86) Console Application") +!MESSAGE "AloneLZMA - Win32 ReleaseU" (based on "Win32 (x86) Console Application") +!MESSAGE "AloneLZMA - Win32 DebugU" (based on "Win32 (x86) Console Application") +!MESSAGE + +# Begin Project +# PROP AllowPerConfigDependencies 0 +# PROP Scc_ProjName "" +# PROP Scc_LocalPath "" +CPP=cl.exe +RSC=rc.exe + +!IF "$(CFG)" == "AloneLZMA - Win32 Release" + +# PROP BASE Use_MFC 0 +# PROP BASE Use_Debug_Libraries 0 +# PROP BASE Output_Dir "Release" +# PROP BASE Intermediate_Dir "Release" +# PROP BASE Target_Dir "" +# PROP Use_MFC 0 +# PROP Use_Debug_Libraries 0 +# PROP Output_Dir "Release" +# PROP Intermediate_Dir "Release" +# PROP Ignore_Export_Lib 0 +# PROP Target_Dir "" +# ADD BASE CPP /nologo /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /c +# ADD CPP /nologo /MT /W3 /GX /O2 /I "..\..\..\\" /D "NDEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /Yu"StdAfx.h" /FD /c +# ADD BASE RSC /l 0x419 /d "NDEBUG" +# ADD RSC /l 0x419 /d "NDEBUG" +BSC32=bscmake.exe +# ADD BASE BSC32 /nologo +# ADD BSC32 /nologo +LINK32=link.exe +# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:console /machine:I386 +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:console /machine:I386 /out:"c:\UTIL\lzma.exe" /opt:NOWIN98 +# SUBTRACT LINK32 /pdb:none + +!ELSEIF "$(CFG)" == "AloneLZMA - Win32 Debug" + +# PROP BASE Use_MFC 0 +# PROP BASE Use_Debug_Libraries 1 +# PROP BASE Output_Dir "Debug" +# PROP BASE Intermediate_Dir "Debug" +# PROP BASE Target_Dir "" +# PROP Use_MFC 0 +# PROP Use_Debug_Libraries 1 +# PROP Output_Dir "Debug" +# PROP Intermediate_Dir "Debug" +# PROP Ignore_Export_Lib 0 +# PROP Target_Dir "" +# ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /GZ /c +# ADD CPP /nologo /W3 /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /Yu"StdAfx.h" /FD /GZ /c +# ADD BASE RSC /l 0x419 /d "_DEBUG" +# ADD RSC /l 0x419 /d "_DEBUG" +BSC32=bscmake.exe +# ADD BASE BSC32 /nologo +# ADD BSC32 /nologo +LINK32=link.exe +# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:console /debug /machine:I386 /pdbtype:sept +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:console /debug /machine:I386 /out:"c:\UTIL\lzma.exe" /pdbtype:sept + +!ELSEIF "$(CFG)" == "AloneLZMA - Win32 ReleaseU" + +# PROP BASE Use_MFC 0 +# PROP BASE Use_Debug_Libraries 0 +# PROP BASE Output_Dir "ReleaseU" +# PROP BASE Intermediate_Dir "ReleaseU" +# PROP BASE Ignore_Export_Lib 0 +# PROP BASE Target_Dir "" +# PROP Use_MFC 0 +# PROP Use_Debug_Libraries 0 +# PROP Output_Dir "ReleaseU" +# PROP Intermediate_Dir "ReleaseU" +# PROP Ignore_Export_Lib 0 +# PROP Target_Dir "" +# ADD BASE CPP /nologo /MD /W3 /GX /O2 /D "NDEBUG" /D "WIN32" /D "_CONSOLE" /D "EXCLUDE_COM" /D "NO_REGISTRY" /D "FORMAT_7Z" /D "FORMAT_BZIP2" /D "FORMAT_ZIP" /D "FORMAT_TAR" /D "FORMAT_GZIP" /D "COMPRESS_LZMA" /D "COMPRESS_BCJ_X86" /D "COMPRESS_BCJ2" /D "COMPRESS_COPY" /D "COMPRESS_MF_PAT" /D "COMPRESS_MF_BT" /D "COMPRESS_PPMD" /D "COMPRESS_DEFLATE" /D "COMPRESS_IMPLODE" /D "COMPRESS_BZIP2" /D "CRYPTO_ZIP" /Yu"StdAfx.h" /FD /c +# ADD CPP /nologo /MD /W3 /GX /O2 /I "..\..\..\\" /D "NDEBUG" /D "UNICODE" /D "_UNICODE" /D "WIN32" /D "_CONSOLE" /Yu"StdAfx.h" /FD /c +# ADD BASE RSC /l 0x419 /d "NDEBUG" +# ADD RSC /l 0x419 /d "NDEBUG" +BSC32=bscmake.exe +# ADD BASE BSC32 /nologo +# ADD BSC32 /nologo +LINK32=link.exe +# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:console /machine:I386 /out:"c:\UTIL\7za2.exe" /opt:NOWIN98 +# SUBTRACT BASE LINK32 /pdb:none +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:console /machine:I386 /out:"c:\UTIL\lzma.exe" /opt:NOWIN98 +# SUBTRACT LINK32 /pdb:none + +!ELSEIF "$(CFG)" == "AloneLZMA - Win32 DebugU" + +# PROP BASE Use_MFC 0 +# PROP BASE Use_Debug_Libraries 1 +# PROP BASE Output_Dir "DebugU" +# PROP BASE Intermediate_Dir "DebugU" +# PROP BASE Ignore_Export_Lib 0 +# PROP BASE Target_Dir "" +# PROP Use_MFC 0 +# PROP Use_Debug_Libraries 1 +# PROP Output_Dir "DebugU" +# PROP Intermediate_Dir "DebugU" +# PROP Ignore_Export_Lib 0 +# PROP Target_Dir "" +# ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "_DEBUG" /D "WIN32" /D "_CONSOLE" /D "EXCLUDE_COM" /D "NO_REGISTRY" /D "FORMAT_7Z" /D "FORMAT_BZIP2" /D "FORMAT_ZIP" /D "FORMAT_TAR" /D "FORMAT_GZIP" /D "COMPRESS_LZMA" /D "COMPRESS_BCJ_X86" /D "COMPRESS_BCJ2" /D "COMPRESS_COPY" /D "COMPRESS_MF_PAT" /D "COMPRESS_MF_BT" /D "COMPRESS_PPMD" /D "COMPRESS_DEFLATE" /D "COMPRESS_IMPLODE" /D "COMPRESS_BZIP2" /D "CRYPTO_ZIP" /D "_MBCS" /Yu"StdAfx.h" /FD /GZ /c +# ADD CPP /nologo /W3 /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "_UNICODE" /D "UNICODE" /D "WIN32" /D "_CONSOLE" /Yu"StdAfx.h" /FD /GZ /c +# ADD BASE RSC /l 0x419 /d "_DEBUG" +# ADD RSC /l 0x419 /d "_DEBUG" +BSC32=bscmake.exe +# ADD BASE BSC32 /nologo +# ADD BSC32 /nologo +LINK32=link.exe +# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:console /debug /machine:I386 /out:"c:\UTIL\7za2.exe" /pdbtype:sept +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:console /debug /machine:I386 /out:"c:\UTIL\lzma.exe" /pdbtype:sept + +!ENDIF + +# Begin Target + +# Name "AloneLZMA - Win32 Release" +# Name "AloneLZMA - Win32 Debug" +# Name "AloneLZMA - Win32 ReleaseU" +# Name "AloneLZMA - Win32 DebugU" +# Begin Group "Spec" + +# PROP Default_Filter "" +# Begin Source File + +SOURCE=.\StdAfx.cpp +# ADD CPP /Yc"StdAfx.h" +# End Source File +# Begin Source File + +SOURCE=.\StdAfx.h +# End Source File +# End Group +# Begin Group "Compress" + +# PROP Default_Filter "" +# Begin Group "LZMA" + +# PROP Default_Filter "" +# Begin Source File + +SOURCE=..\LZMA\LZMA.h +# End Source File +# Begin Source File + +SOURCE=..\LZMA\LZMADecoder.cpp +# End Source File +# Begin Source File + +SOURCE=..\LZMA\LZMADecoder.h +# End Source File +# Begin Source File + +SOURCE=..\LZMA\LZMAEncoder.cpp +# End Source File +# Begin Source File + +SOURCE=..\LZMA\LZMAEncoder.h +# End Source File +# End Group +# Begin Group "RangeCoder" + +# PROP Default_Filter "" +# Begin Source File + +SOURCE=..\RangeCoder\RangeCoder.h +# End Source File +# Begin Source File + +SOURCE=..\RangeCoder\RangeCoderBit.cpp +# End Source File +# Begin Source File + +SOURCE=..\RangeCoder\RangeCoderBit.h +# End Source File +# Begin Source File + +SOURCE=..\RangeCoder\RangeCoderBitTree.h +# End Source File +# Begin Source File + +SOURCE=..\RangeCoder\RangeCoderOpt.h +# End Source File +# End Group +# Begin Group "LZ" + +# PROP Default_Filter "" +# Begin Group "BT" + +# PROP Default_Filter "" +# Begin Source File + +SOURCE=..\LZ\BinTree\BinTree.h +# End Source File +# Begin Source File + +SOURCE=..\LZ\BinTree\BinTree2.h +# End Source File +# Begin Source File + +SOURCE=..\LZ\BinTree\BinTree3.h +# End Source File +# Begin Source File + +SOURCE=..\LZ\BinTree\BinTree3Z.h +# End Source File +# Begin Source File + +SOURCE=..\LZ\BinTree\BinTree4.h +# End Source File +# Begin Source File + +SOURCE=..\LZ\BinTree\BinTreeMain.h +# End Source File +# End Group +# Begin Group "HC" + +# PROP Default_Filter "" +# Begin Source File + +SOURCE=..\LZ\HashChain\HC4.h +# End Source File +# Begin Source File + +SOURCE=..\LZ\HashChain\HCMain.h +# End Source File +# End Group +# Begin Source File + +SOURCE=..\LZ\IMatchFinder.h +# End Source File +# Begin Source File + +SOURCE=..\LZ\LZInWindow.cpp +# End Source File +# Begin Source File + +SOURCE=..\LZ\LZInWindow.h +# End Source File +# Begin Source File + +SOURCE=..\LZ\LZOutWindow.cpp +# End Source File +# Begin Source File + +SOURCE=..\LZ\LZOutWindow.h +# End Source File +# End Group +# Begin Group "Branch" + +# PROP Default_Filter "" +# Begin Source File + +SOURCE=..\Branch\BranchTypes.h +# End Source File +# Begin Source File + +SOURCE=..\Branch\BranchX86.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\Branch\BranchX86.h +# End Source File +# End Group +# Begin Group "LZMA_C" + +# PROP Default_Filter "" +# Begin Source File + +SOURCE=..\LZMA_C\LzmaDecode.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\LZMA_C\LzmaDecode.h +# End Source File +# Begin Source File + +SOURCE=..\LZMA_C\LzmaTypes.h +# End Source File +# End Group +# End Group +# Begin Group "Windows" + +# PROP Default_Filter "" +# Begin Source File + +SOURCE=..\..\..\Windows\FileIO.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\FileIO.h +# End Source File +# End Group +# Begin Group "Common" + +# PROP Default_Filter "" +# Begin Source File + +SOURCE=..\..\..\Common\Alloc.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\Alloc.h +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\CommandLineParser.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\CommandLineParser.h +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\CRC.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\CRC.h +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\Defs.h +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\Defs.h +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\MyCom.h +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\MyWindows.h +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\NewHandler.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\NewHandler.h +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\String.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\String.h +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\StringConvert.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\StringConvert.h +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\StringToInt.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\StringToInt.h +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\Types.h +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\Vector.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\Vector.h +# End Source File +# End Group +# Begin Group "7zip Common" + +# PROP Default_Filter "" +# Begin Source File + +SOURCE=..\..\Common\FileStreams.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\Common\FileStreams.h +# End Source File +# Begin Source File + +SOURCE=..\..\Common\InBuffer.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\Common\InBuffer.h +# End Source File +# Begin Source File + +SOURCE=..\..\Common\OutBuffer.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\Common\OutBuffer.h +# End Source File +# Begin Source File + +SOURCE=..\..\Common\StreamUtils.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\Common\StreamUtils.h +# End Source File +# End Group +# Begin Source File + +SOURCE=..\..\ICoder.h +# End Source File +# Begin Source File + +SOURCE=.\LzmaAlone.cpp +# End Source File +# Begin Source File + +SOURCE=.\LzmaBench.cpp +# End Source File +# Begin Source File + +SOURCE=.\LzmaBench.h +# End Source File +# Begin Source File + +SOURCE=.\LzmaRam.cpp +# End Source File +# Begin Source File + +SOURCE=.\LzmaRam.h +# End Source File +# Begin Source File + +SOURCE=.\LzmaRamDecode.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=.\LzmaRamDecode.h +# End Source File +# End Target +# End Project diff --git a/lzma443/C/7zip/Compress/LZMA_Alone/AloneLZMA.dsw b/lzma443/C/7zip/Compress/LZMA_Alone/AloneLZMA.dsw new file mode 100755 index 0000000..7402f29 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA_Alone/AloneLZMA.dsw @@ -0,0 +1,29 @@ +Microsoft Developer Studio Workspace File, Format Version 6.00 +# WARNING: DO NOT EDIT OR DELETE THIS WORKSPACE FILE! + +############################################################################### + +Project: "AloneLZMA"=.\AloneLZMA.dsp - Package Owner=<4> + +Package=<5> +{{{ +}}} + +Package=<4> +{{{ +}}} + +############################################################################### + +Global: + +Package=<5> +{{{ +}}} + +Package=<3> +{{{ +}}} + +############################################################################### + diff --git a/lzma443/C/7zip/Compress/LZMA_Alone/LzmaAlone.cpp b/lzma443/C/7zip/Compress/LZMA_Alone/LzmaAlone.cpp new file mode 100755 index 0000000..cce01eb --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA_Alone/LzmaAlone.cpp @@ -0,0 +1,524 @@ +// LzmaAlone.cpp + +#include "StdAfx.h" + +#include "../../../Common/MyWindows.h" +#include "../../../Common/MyInitGuid.h" + +#include + +#if defined(_WIN32) || defined(OS2) || defined(MSDOS) +#include +#include +#define MY_SET_BINARY_MODE(file) setmode(fileno(file),O_BINARY) +#else +#define MY_SET_BINARY_MODE(file) +#endif + +#include "../../../Common/CommandLineParser.h" +#include "../../../Common/StringConvert.h" +#include "../../../Common/StringToInt.h" + +#include "../../Common/FileStreams.h" +#include "../../Common/StreamUtils.h" + +#include "../LZMA/LZMADecoder.h" +#include "../LZMA/LZMAEncoder.h" + +#include "LzmaBench.h" +#include "LzmaRam.h" + +extern "C" +{ +#include "LzmaRamDecode.h" +} + +using namespace NCommandLineParser; + +#ifdef _WIN32 +bool g_IsNT = false; +static inline bool IsItWindowsNT() +{ + OSVERSIONINFO versionInfo; + versionInfo.dwOSVersionInfoSize = sizeof(versionInfo); + if (!::GetVersionEx(&versionInfo)) + return false; + return (versionInfo.dwPlatformId == VER_PLATFORM_WIN32_NT); +} +#endif + +static const char *kCantAllocate = "Can not allocate memory"; +static const char *kReadError = "Read error"; +static const char *kWriteError = "Write error"; + +namespace NKey { +enum Enum +{ + kHelp1 = 0, + kHelp2, + kMode, + kDictionary, + kFastBytes, + kMatchFinderCycles, + kLitContext, + kLitPos, + kPosBits, + kMatchFinder, + kEOS, + kStdIn, + kStdOut, + kFilter86 +}; +} + +static const CSwitchForm kSwitchForms[] = +{ + { L"?", NSwitchType::kSimple, false }, + { L"H", NSwitchType::kSimple, false }, + { L"A", NSwitchType::kUnLimitedPostString, false, 1 }, + { L"D", NSwitchType::kUnLimitedPostString, false, 1 }, + { L"FB", NSwitchType::kUnLimitedPostString, false, 1 }, + { L"MC", NSwitchType::kUnLimitedPostString, false, 1 }, + { L"LC", NSwitchType::kUnLimitedPostString, false, 1 }, + { L"LP", NSwitchType::kUnLimitedPostString, false, 1 }, + { L"PB", NSwitchType::kUnLimitedPostString, false, 1 }, + { L"MF", NSwitchType::kUnLimitedPostString, false, 1 }, + { L"EOS", NSwitchType::kSimple, false }, + { L"SI", NSwitchType::kSimple, false }, + { L"SO", NSwitchType::kSimple, false }, + { L"F86", NSwitchType::kSimple, false } +}; + +static const int kNumSwitches = sizeof(kSwitchForms) / sizeof(kSwitchForms[0]); + +static void PrintHelp() +{ + fprintf(stderr, "\nUsage: LZMA inputFile outputFile [...]\n" + " e: encode file\n" + " d: decode file\n" + " b: Benchmark\n" + "\n" + " -a{N}: set compression mode - [0, 1], default: 1 (max)\n" + " -d{N}: set dictionary - [0,30], default: 23 (8MB)\n" + " -fb{N}: set number of fast bytes - [5, 273], default: 128\n" + " -mc{N}: set number of cycles for match finder\n" + " -lc{N}: set number of literal context bits - [0, 8], default: 3\n" + " -lp{N}: set number of literal pos bits - [0, 4], default: 0\n" + " -pb{N}: set number of pos bits - [0, 4], default: 2\n" + " -mf{MF_ID}: set Match Finder: [bt2, bt3, bt4, hc4], default: bt4\n" + " -eos: write End Of Stream marker\n" + " -si: read data from stdin\n" + " -so: write data to stdout\n" + ); +} + +static void PrintHelpAndExit(const char *s) +{ + fprintf(stderr, "\nError: %s\n\n", s); + PrintHelp(); + throw -1; +} + +static void IncorrectCommand() +{ + PrintHelpAndExit("Incorrect command"); +} + +static void WriteArgumentsToStringList(int numArguments, const char *arguments[], + UStringVector &strings) +{ + for(int i = 1; i < numArguments; i++) + strings.Add(MultiByteToUnicodeString(arguments[i])); +} + +static bool GetNumber(const wchar_t *s, UInt32 &value) +{ + value = 0; + if (MyStringLen(s) == 0) + return false; + const wchar_t *end; + UInt64 res = ConvertStringToUInt64(s, &end); + if (*end != L'\0') + return false; + if (res > 0xFFFFFFFF) + return false; + value = UInt32(res); + return true; +} + +int main2(int n, const char *args[]) +{ + #ifdef _WIN32 + g_IsNT = IsItWindowsNT(); + #endif + + fprintf(stderr, "\nLZMA 4.43 Copyright (c) 1999-2006 Igor Pavlov 2006-06-04\n"); + + if (n == 1) + { + PrintHelp(); + return 0; + } + + if (sizeof(Byte) != 1 || sizeof(UInt32) < 4 || sizeof(UInt64) < 4) + { + fprintf(stderr, "Unsupported base types. Edit Common/Types.h and recompile"); + return 1; + } + + UStringVector commandStrings; + WriteArgumentsToStringList(n, args, commandStrings); + CParser parser(kNumSwitches); + try + { + parser.ParseStrings(kSwitchForms, commandStrings); + } + catch(...) + { + IncorrectCommand(); + } + + if(parser[NKey::kHelp1].ThereIs || parser[NKey::kHelp2].ThereIs) + { + PrintHelp(); + return 0; + } + const UStringVector &nonSwitchStrings = parser.NonSwitchStrings; + + int paramIndex = 0; + if (paramIndex >= nonSwitchStrings.Size()) + IncorrectCommand(); + const UString &command = nonSwitchStrings[paramIndex++]; + + bool dictionaryIsDefined = false; + UInt32 dictionary = 1 << 21; + if(parser[NKey::kDictionary].ThereIs) + { + UInt32 dicLog; + if (!GetNumber(parser[NKey::kDictionary].PostStrings[0], dicLog)) + IncorrectCommand(); + dictionary = 1 << dicLog; + dictionaryIsDefined = true; + } + UString mf = L"BT4"; + if (parser[NKey::kMatchFinder].ThereIs) + mf = parser[NKey::kMatchFinder].PostStrings[0]; + + if (command.CompareNoCase(L"b") == 0) + { + const UInt32 kNumDefaultItereations = 10; + UInt32 numIterations = kNumDefaultItereations; + { + if (paramIndex < nonSwitchStrings.Size()) + if (!GetNumber(nonSwitchStrings[paramIndex++], numIterations)) + numIterations = kNumDefaultItereations; + } + return LzmaBenchmark(stderr, numIterations, dictionary); + } + + bool encodeMode = false; + if (command.CompareNoCase(L"e") == 0) + encodeMode = true; + else if (command.CompareNoCase(L"d") == 0) + encodeMode = false; + else + IncorrectCommand(); + + bool stdInMode = parser[NKey::kStdIn].ThereIs; + bool stdOutMode = parser[NKey::kStdOut].ThereIs; + + CMyComPtr inStream; + CInFileStream *inStreamSpec = 0; + if (stdInMode) + { + inStream = new CStdInFileStream; + MY_SET_BINARY_MODE(stdin); + } + else + { + if (paramIndex >= nonSwitchStrings.Size()) + IncorrectCommand(); + const UString &inputName = nonSwitchStrings[paramIndex++]; + inStreamSpec = new CInFileStream; + inStream = inStreamSpec; + if (!inStreamSpec->Open(GetSystemString(inputName))) + { + fprintf(stderr, "\nError: can not open input file %s\n", + (const char *)GetOemString(inputName)); + return 1; + } + } + + CMyComPtr outStream; + if (stdOutMode) + { + outStream = new CStdOutFileStream; + MY_SET_BINARY_MODE(stdout); + } + else + { + if (paramIndex >= nonSwitchStrings.Size()) + IncorrectCommand(); + const UString &outputName = nonSwitchStrings[paramIndex++]; + COutFileStream *outStreamSpec = new COutFileStream; + outStream = outStreamSpec; + if (!outStreamSpec->Create(GetSystemString(outputName), true)) + { + fprintf(stderr, "\nError: can not open output file %s\n", + (const char *)GetOemString(outputName)); + return 1; + } + } + + if (parser[NKey::kFilter86].ThereIs) + { + // -f86 switch is for x86 filtered mode: BCJ + LZMA. + if (parser[NKey::kEOS].ThereIs || stdInMode) + throw "Can not use stdin in this mode"; + UInt64 fileSize; + inStreamSpec->File.GetLength(fileSize); + if (fileSize > 0xF0000000) + throw "File is too big"; + UInt32 inSize = (UInt32)fileSize; + Byte *inBuffer = 0; + if (inSize != 0) + { + inBuffer = (Byte *)MyAlloc((size_t)inSize); + if (inBuffer == 0) + throw kCantAllocate; + } + + UInt32 processedSize; + if (ReadStream(inStream, inBuffer, (UInt32)inSize, &processedSize) != S_OK) + throw "Can not read"; + if ((UInt32)inSize != processedSize) + throw "Read size error"; + + Byte *outBuffer = 0; + size_t outSizeProcessed; + if (encodeMode) + { + // we allocate 105% of original size for output buffer + size_t outSize = (size_t)fileSize / 20 * 21 + (1 << 16); + if (outSize != 0) + { + outBuffer = (Byte *)MyAlloc((size_t)outSize); + if (outBuffer == 0) + throw kCantAllocate; + } + if (!dictionaryIsDefined) + dictionary = 1 << 23; + int res = LzmaRamEncode(inBuffer, inSize, outBuffer, outSize, &outSizeProcessed, + dictionary, SZ_FILTER_AUTO); + if (res != 0) + { + fprintf(stderr, "\nEncoder error = %d\n", (int)res); + return 1; + } + } + else + { + size_t outSize; + if (LzmaRamGetUncompressedSize(inBuffer, inSize, &outSize) != 0) + throw "data error"; + if (outSize != 0) + { + outBuffer = (Byte *)MyAlloc(outSize); + if (outBuffer == 0) + throw kCantAllocate; + } + int res = LzmaRamDecompress(inBuffer, inSize, outBuffer, outSize, &outSizeProcessed, malloc, free); + if (res != 0) + throw "LzmaDecoder error"; + } + if (WriteStream(outStream, outBuffer, (UInt32)outSizeProcessed, &processedSize) != S_OK) + throw kWriteError; + MyFree(outBuffer); + MyFree(inBuffer); + return 0; + } + + + UInt64 fileSize; + if (encodeMode) + { + NCompress::NLZMA::CEncoder *encoderSpec = + new NCompress::NLZMA::CEncoder; + CMyComPtr encoder = encoderSpec; + + if (!dictionaryIsDefined) + dictionary = 1 << 23; + + UInt32 posStateBits = 2; + UInt32 litContextBits = 3; // for normal files + // UInt32 litContextBits = 0; // for 32-bit data + UInt32 litPosBits = 0; + // UInt32 litPosBits = 2; // for 32-bit data + UInt32 algorithm = 2; + UInt32 numFastBytes = 128; + UInt32 matchFinderCycles = 16 + numFastBytes / 2; + bool matchFinderCyclesDefined = false; + + bool eos = parser[NKey::kEOS].ThereIs || stdInMode; + + if(parser[NKey::kMode].ThereIs) + if (!GetNumber(parser[NKey::kMode].PostStrings[0], algorithm)) + IncorrectCommand(); + + if(parser[NKey::kFastBytes].ThereIs) + if (!GetNumber(parser[NKey::kFastBytes].PostStrings[0], numFastBytes)) + IncorrectCommand(); + if (matchFinderCyclesDefined = parser[NKey::kMatchFinderCycles].ThereIs) + if (!GetNumber(parser[NKey::kMatchFinderCycles].PostStrings[0], matchFinderCycles)) + IncorrectCommand(); + if(parser[NKey::kLitContext].ThereIs) + if (!GetNumber(parser[NKey::kLitContext].PostStrings[0], litContextBits)) + IncorrectCommand(); + if(parser[NKey::kLitPos].ThereIs) + if (!GetNumber(parser[NKey::kLitPos].PostStrings[0], litPosBits)) + IncorrectCommand(); + if(parser[NKey::kPosBits].ThereIs) + if (!GetNumber(parser[NKey::kPosBits].PostStrings[0], posStateBits)) + IncorrectCommand(); + + PROPID propIDs[] = + { + NCoderPropID::kDictionarySize, + NCoderPropID::kPosStateBits, + NCoderPropID::kLitContextBits, + NCoderPropID::kLitPosBits, + NCoderPropID::kAlgorithm, + NCoderPropID::kNumFastBytes, + NCoderPropID::kMatchFinder, + NCoderPropID::kEndMarker, + NCoderPropID::kMatchFinderCycles + }; + const int kNumPropsMax = sizeof(propIDs) / sizeof(propIDs[0]); + /* + NWindows::NCOM::CPropVariant properties[kNumProps]; + properties[0] = UInt32(dictionary); + properties[1] = UInt32(posStateBits); + properties[2] = UInt32(litContextBits); + + properties[3] = UInt32(litPosBits); + properties[4] = UInt32(algorithm); + properties[5] = UInt32(numFastBytes); + properties[6] = mf; + properties[7] = eos; + */ + PROPVARIANT properties[kNumPropsMax]; + for (int p = 0; p < 6; p++) + properties[p].vt = VT_UI4; + + properties[0].ulVal = UInt32(dictionary); + properties[1].ulVal = UInt32(posStateBits); + properties[2].ulVal = UInt32(litContextBits); + properties[3].ulVal = UInt32(litPosBits); + properties[4].ulVal = UInt32(algorithm); + properties[5].ulVal = UInt32(numFastBytes); + + properties[8].vt = VT_UI4; + properties[8].ulVal = UInt32(matchFinderCycles); + + properties[6].vt = VT_BSTR; + properties[6].bstrVal = (BSTR)(const wchar_t *)mf; + + properties[7].vt = VT_BOOL; + properties[7].boolVal = eos ? VARIANT_TRUE : VARIANT_FALSE; + + int numProps = kNumPropsMax; + if (!matchFinderCyclesDefined) + numProps--; + + if (encoderSpec->SetCoderProperties(propIDs, properties, numProps) != S_OK) + IncorrectCommand(); + encoderSpec->WriteCoderProperties(outStream); + + if (eos || stdInMode) + fileSize = (UInt64)(Int64)-1; + else + inStreamSpec->File.GetLength(fileSize); + + for (int i = 0; i < 8; i++) + { + Byte b = Byte(fileSize >> (8 * i)); + if (outStream->Write(&b, 1, 0) != S_OK) + { + fprintf(stderr, kWriteError); + return 1; + } + } + HRESULT result = encoder->Code(inStream, outStream, 0, 0, 0); + if (result == E_OUTOFMEMORY) + { + fprintf(stderr, "\nError: Can not allocate memory\n"); + return 1; + } + else if (result != S_OK) + { + fprintf(stderr, "\nEncoder error = %X\n", (unsigned int)result); + return 1; + } + } + else + { + NCompress::NLZMA::CDecoder *decoderSpec = + new NCompress::NLZMA::CDecoder; + CMyComPtr decoder = decoderSpec; + const UInt32 kPropertiesSize = 5; + Byte properties[kPropertiesSize]; + UInt32 processedSize; + if (ReadStream(inStream, properties, kPropertiesSize, &processedSize) != S_OK) + { + fprintf(stderr, kReadError); + return 1; + } + if (processedSize != kPropertiesSize) + { + fprintf(stderr, kReadError); + return 1; + } + if (decoderSpec->SetDecoderProperties2(properties, kPropertiesSize) != S_OK) + { + fprintf(stderr, "SetDecoderProperties error"); + return 1; + } + fileSize = 0; + for (int i = 0; i < 8; i++) + { + Byte b; + if (inStream->Read(&b, 1, &processedSize) != S_OK) + { + fprintf(stderr, kReadError); + return 1; + } + if (processedSize != 1) + { + fprintf(stderr, kReadError); + return 1; + } + fileSize |= ((UInt64)b) << (8 * i); + } + if (decoder->Code(inStream, outStream, 0, &fileSize, 0) != S_OK) + { + fprintf(stderr, "Decoder error"); + return 1; + } + } + return 0; +} + +int main(int n, const char *args[]) +{ + try { return main2(n, args); } + catch(const char *s) + { + fprintf(stderr, "\nError: %s\n", s); + return 1; + } + catch(...) + { + fprintf(stderr, "\nError\n"); + return 1; + } +} diff --git a/lzma443/C/7zip/Compress/LZMA_Alone/LzmaBench.cpp b/lzma443/C/7zip/Compress/LZMA_Alone/LzmaBench.cpp new file mode 100755 index 0000000..84d9a5a --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA_Alone/LzmaBench.cpp @@ -0,0 +1,506 @@ +// LzmaBench.cpp + +#include "StdAfx.h" + +#include "LzmaBench.h" + +#ifndef _WIN32 +#include +#endif + +#include "../../../Common/CRC.h" +#include "../LZMA/LZMADecoder.h" +#include "../LZMA/LZMAEncoder.h" + +static const UInt32 kAdditionalSize = +#ifdef _WIN32_WCE +(1 << 20); +#else +(6 << 20); +#endif + +static const UInt32 kCompressedAdditionalSize = (1 << 10); +static const UInt32 kMaxLzmaPropSize = 10; + +class CRandomGenerator +{ + UInt32 A1; + UInt32 A2; +public: + CRandomGenerator() { Init(); } + void Init() { A1 = 362436069; A2 = 521288629;} + UInt32 GetRnd() + { + return + ((A1 = 36969 * (A1 & 0xffff) + (A1 >> 16)) << 16) ^ + ((A2 = 18000 * (A2 & 0xffff) + (A2 >> 16)) ); + } +}; + +class CBitRandomGenerator +{ + CRandomGenerator RG; + UInt32 Value; + int NumBits; +public: + void Init() + { + Value = 0; + NumBits = 0; + } + UInt32 GetRnd(int numBits) + { + if (NumBits > numBits) + { + UInt32 result = Value & ((1 << numBits) - 1); + Value >>= numBits; + NumBits -= numBits; + return result; + } + numBits -= NumBits; + UInt32 result = (Value << numBits); + Value = RG.GetRnd(); + result |= Value & ((1 << numBits) - 1); + Value >>= numBits; + NumBits = 32 - numBits; + return result; + } +}; + +class CBenchRandomGenerator +{ + CBitRandomGenerator RG; + UInt32 Pos; + UInt32 Rep0; +public: + UInt32 BufferSize; + Byte *Buffer; + CBenchRandomGenerator(): Buffer(0) {} + ~CBenchRandomGenerator() { Free(); } + void Free() + { + ::MidFree(Buffer); + Buffer = 0; + } + bool Alloc(UInt32 bufferSize) + { + if (Buffer != 0 && BufferSize == bufferSize) + return true; + Free(); + Buffer = (Byte *)::MidAlloc(bufferSize); + Pos = 0; + BufferSize = bufferSize; + return (Buffer != 0); + } + UInt32 GetRndBit() { return RG.GetRnd(1); } + /* + UInt32 GetLogRand(int maxLen) + { + UInt32 len = GetRnd() % (maxLen + 1); + return GetRnd() & ((1 << len) - 1); + } + */ + UInt32 GetLogRandBits(int numBits) + { + UInt32 len = RG.GetRnd(numBits); + return RG.GetRnd(len); + } + UInt32 GetOffset() + { + if (GetRndBit() == 0) + return GetLogRandBits(4); + return (GetLogRandBits(4) << 10) | RG.GetRnd(10); + } + UInt32 GetLen1() { return RG.GetRnd(1 + (int)RG.GetRnd(2)); } + UInt32 GetLen2() { return RG.GetRnd(2 + (int)RG.GetRnd(2)); } + void Generate() + { + RG.Init(); + Rep0 = 1; + while(Pos < BufferSize) + { + if (GetRndBit() == 0 || Pos < 1) + Buffer[Pos++] = (Byte)RG.GetRnd(8); + else + { + UInt32 len; + if (RG.GetRnd(3) == 0) + len = 1 + GetLen1(); + else + { + do + Rep0 = GetOffset(); + while (Rep0 >= Pos); + Rep0++; + len = 2 + GetLen2(); + } + for (UInt32 i = 0; i < len && Pos < BufferSize; i++, Pos++) + Buffer[Pos] = Buffer[Pos - Rep0]; + } + } + } +}; + +class CBenchmarkInStream: + public ISequentialInStream, + public CMyUnknownImp +{ + const Byte *Data; + UInt32 Pos; + UInt32 Size; +public: + MY_UNKNOWN_IMP + void Init(const Byte *data, UInt32 size) + { + Data = data; + Size = size; + Pos = 0; + } + STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); +}; + +STDMETHODIMP CBenchmarkInStream::Read(void *data, UInt32 size, UInt32 *processedSize) +{ + UInt32 remain = Size - Pos; + if (size > remain) + size = remain; + for (UInt32 i = 0; i < size; i++) + ((Byte *)data)[i] = Data[Pos + i]; + Pos += size; + if(processedSize != NULL) + *processedSize = size; + return S_OK; +} + +class CBenchmarkOutStream: + public ISequentialOutStream, + public CMyUnknownImp +{ + UInt32 BufferSize; + FILE *_f; +public: + UInt32 Pos; + Byte *Buffer; + CBenchmarkOutStream(): _f(0), Buffer(0) {} + virtual ~CBenchmarkOutStream() { delete []Buffer; } + void Init(FILE *f, UInt32 bufferSize) + { + delete []Buffer; + Buffer = 0; + Buffer = new Byte[bufferSize]; + Pos = 0; + BufferSize = bufferSize; + _f = f; + } + MY_UNKNOWN_IMP + STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize); +}; + +STDMETHODIMP CBenchmarkOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize) +{ + UInt32 i; + for (i = 0; i < size && Pos < BufferSize; i++) + Buffer[Pos++] = ((const Byte *)data)[i]; + if(processedSize != NULL) + *processedSize = i; + if (i != size) + { + fprintf(_f, "\nERROR: Buffer is full\n"); + return E_FAIL; + } + return S_OK; +} + +class CCrcOutStream: + public ISequentialOutStream, + public CMyUnknownImp +{ +public: + CCRC CRC; + MY_UNKNOWN_IMP + void Init() { CRC.Init(); } + STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize); +}; + +STDMETHODIMP CCrcOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize) +{ + CRC.Update(data, size); + if(processedSize != NULL) + *processedSize = size; + return S_OK; +} + +static UInt64 GetTimeCount() +{ + #ifdef _WIN32 + LARGE_INTEGER value; + if (::QueryPerformanceCounter(&value)) + return value.QuadPart; + return GetTickCount(); + #else + return clock(); + #endif +} + +static UInt64 GetFreq() +{ + #ifdef _WIN32 + LARGE_INTEGER value; + if (::QueryPerformanceFrequency(&value)) + return value.QuadPart; + return 1000; + #else + return CLOCKS_PER_SEC; + #endif +} + +struct CProgressInfo: + public ICompressProgressInfo, + public CMyUnknownImp +{ + UInt64 ApprovedStart; + UInt64 InSize; + UInt64 Time; + void Init() + { + InSize = 0; + Time = 0; + } + MY_UNKNOWN_IMP + STDMETHOD(SetRatioInfo)(const UInt64 *inSize, const UInt64 *outSize); +}; + +STDMETHODIMP CProgressInfo::SetRatioInfo(const UInt64 *inSize, const UInt64 *outSize) +{ + if (*inSize >= ApprovedStart && InSize == 0) + { + Time = ::GetTimeCount(); + InSize = *inSize; + } + return S_OK; +} + +static const int kSubBits = 8; + +static UInt32 GetLogSize(UInt32 size) +{ + for (int i = kSubBits; i < 32; i++) + for (UInt32 j = 0; j < (1 << kSubBits); j++) + if (size <= (((UInt32)1) << i) + (j << (i - kSubBits))) + return (i << kSubBits) + j; + return (32 << kSubBits); +} + +static UInt64 MyMultDiv64(UInt64 value, UInt64 elapsedTime) +{ + UInt64 freq = GetFreq(); + UInt64 elTime = elapsedTime; + while(freq > 1000000) + { + freq >>= 1; + elTime >>= 1; + } + if (elTime == 0) + elTime = 1; + return value * freq / elTime; +} + +static UInt64 GetCompressRating(UInt32 dictionarySize, UInt64 elapsedTime, UInt64 size) +{ + UInt64 t = GetLogSize(dictionarySize) - (18 << kSubBits); + UInt64 numCommandsForOne = 1060 + ((t * t * 10) >> (2 * kSubBits)); + UInt64 numCommands = (UInt64)(size) * numCommandsForOne; + return MyMultDiv64(numCommands, elapsedTime); +} + +static UInt64 GetDecompressRating(UInt64 elapsedTime, + UInt64 outSize, UInt64 inSize) +{ + UInt64 numCommands = inSize * 220 + outSize * 20; + return MyMultDiv64(numCommands, elapsedTime); +} + +/* +static UInt64 GetTotalRating( + UInt32 dictionarySize, + bool isBT4, + UInt64 elapsedTimeEn, UInt64 sizeEn, + UInt64 elapsedTimeDe, + UInt64 inSizeDe, UInt64 outSizeDe) +{ + return (GetCompressRating(dictionarySize, isBT4, elapsedTimeEn, sizeEn) + + GetDecompressRating(elapsedTimeDe, inSizeDe, outSizeDe)) / 2; +} +*/ + +static void PrintRating(FILE *f, UInt64 rating) +{ + fprintf(f, "%5d MIPS", (unsigned int)(rating / 1000000)); +} + +static void PrintResults( + FILE *f, + UInt32 dictionarySize, + UInt64 elapsedTime, + UInt64 size, + bool decompressMode, UInt64 secondSize) +{ + UInt64 speed = MyMultDiv64(size, elapsedTime); + fprintf(f, "%6d KB/s ", (unsigned int)(speed / 1024)); + UInt64 rating; + if (decompressMode) + rating = GetDecompressRating(elapsedTime, size, secondSize); + else + rating = GetCompressRating(dictionarySize, elapsedTime, size); + PrintRating(f, rating); +} + +static void ThrowError(FILE *f, HRESULT result, const char *s) +{ + fprintf(f, "\nError: "); + if (result == E_ABORT) + fprintf(f, "User break"); + if (result == E_OUTOFMEMORY) + fprintf(f, "Can not allocate memory"); + else + fputs(s,f); + fprintf(f, "\n"); +} + +const wchar_t *bt2 = L"BT2"; +const wchar_t *bt4 = L"BT4"; + +int LzmaBenchmark(FILE *f, UInt32 numIterations, UInt32 dictionarySize) +{ + if (numIterations == 0) + return 0; + if (dictionarySize < (1 << 18)) + { + fprintf(f, "\nError: dictionary size for benchmark must be >= 19 (512 KB)\n"); + return 1; + } + fprintf(f, "\n Compressing Decompressing\n\n"); + NCompress::NLZMA::CEncoder *encoderSpec = new NCompress::NLZMA::CEncoder; + CMyComPtr encoder = encoderSpec; + + NCompress::NLZMA::CDecoder *decoderSpec = new NCompress::NLZMA::CDecoder; + CMyComPtr decoder = decoderSpec; + + CBenchmarkOutStream *propStreamSpec = new CBenchmarkOutStream; + CMyComPtr propStream = propStreamSpec; + propStreamSpec->Init(f, kMaxLzmaPropSize); + + PROPID propIDs[] = + { + NCoderPropID::kDictionarySize + }; + const int kNumProps = sizeof(propIDs) / sizeof(propIDs[0]); + PROPVARIANT properties[kNumProps]; + properties[0].vt = VT_UI4; + properties[0].ulVal = UInt32(dictionarySize); + + const UInt32 kBufferSize = dictionarySize + kAdditionalSize; + const UInt32 kCompressedBufferSize = (kBufferSize / 2) + kCompressedAdditionalSize; + + if (encoderSpec->SetCoderProperties(propIDs, properties, kNumProps) != S_OK) + { + fprintf(f, "\nError: Incorrect command\n"); + return 1; + } + encoderSpec->WriteCoderProperties(propStream); + + CBenchRandomGenerator rg; + if (!rg.Alloc(kBufferSize)) + { + fprintf(f, "\nError: Can't allocate memory\n"); + return 1; + } + + rg.Generate(); + CCRC crc; + crc.Update(rg.Buffer, rg.BufferSize); + + CProgressInfo *progressInfoSpec = new CProgressInfo; + CMyComPtr progressInfo = progressInfoSpec; + + progressInfoSpec->ApprovedStart = dictionarySize; + + UInt64 totalBenchSize = 0; + UInt64 totalEncodeTime = 0; + UInt64 totalDecodeTime = 0; + UInt64 totalCompressedSize = 0; + + for (UInt32 i = 0; i < numIterations; i++) + { + progressInfoSpec->Init(); + CBenchmarkInStream *inStreamSpec = new CBenchmarkInStream; + inStreamSpec->Init(rg.Buffer, rg.BufferSize); + CMyComPtr inStream = inStreamSpec; + CBenchmarkOutStream *outStreamSpec = new CBenchmarkOutStream; + outStreamSpec->Init(f, kCompressedBufferSize); + CMyComPtr outStream = outStreamSpec; + HRESULT result = encoder->Code(inStream, outStream, 0, 0, progressInfo); + UInt64 encodeTime = ::GetTimeCount() - progressInfoSpec->Time; + UInt32 compressedSize = outStreamSpec->Pos; + if(result != S_OK) + { + ThrowError(f, result, "Encoder Error"); + return 1; + } + if (progressInfoSpec->InSize == 0) + { + fprintf(f, "\nError: Internal ERROR 1282\n"); + return 1; + } + + /////////////////////// + // Decompressing + + CCrcOutStream *crcOutStreamSpec = new CCrcOutStream; + CMyComPtr crcOutStream = crcOutStreamSpec; + + UInt64 decodeTime; + for (int j = 0; j < 2; j++) + { + inStreamSpec->Init(outStreamSpec->Buffer, compressedSize); + crcOutStreamSpec->Init(); + + if (decoderSpec->SetDecoderProperties2(propStreamSpec->Buffer, propStreamSpec->Pos) != S_OK) + { + fprintf(f, "\nError: Set Decoder Properties Error\n"); + return 1; + } + UInt64 outSize = kBufferSize; + UInt64 startTime = ::GetTimeCount(); + result = decoder->Code(inStream, crcOutStream, 0, &outSize, 0); + decodeTime = ::GetTimeCount() - startTime; + if(result != S_OK) + { + ThrowError(f, result, "Decode Error"); + return 1; + } + if (crcOutStreamSpec->CRC.GetDigest() != crc.GetDigest()) + { + fprintf(f, "\nError: CRC Error\n"); + return 1; + } + } + UInt64 benchSize = kBufferSize - progressInfoSpec->InSize; + PrintResults(f, dictionarySize, encodeTime, benchSize, false, 0); + fprintf(f, " "); + PrintResults(f, dictionarySize, decodeTime, kBufferSize, true, compressedSize); + fprintf(f, "\n"); + + totalBenchSize += benchSize; + totalEncodeTime += encodeTime; + totalDecodeTime += decodeTime; + totalCompressedSize += compressedSize; + } + fprintf(f, "---------------------------------------------------\n"); + PrintResults(f, dictionarySize, totalEncodeTime, totalBenchSize, false, 0); + fprintf(f, " "); + PrintResults(f, dictionarySize, totalDecodeTime, + kBufferSize * numIterations, true, totalCompressedSize); + fprintf(f, " Average\n"); + return 0; +} diff --git a/lzma443/C/7zip/Compress/LZMA_Alone/LzmaBench.h b/lzma443/C/7zip/Compress/LZMA_Alone/LzmaBench.h new file mode 100755 index 0000000..a6a0e82 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA_Alone/LzmaBench.h @@ -0,0 +1,11 @@ +// LzmaBench.h + +#ifndef __LzmaBench_h +#define __LzmaBench_h + +#include +#include "../../../Common/Types.h" + +int LzmaBenchmark(FILE *f, UInt32 numIterations, UInt32 dictionarySize); + +#endif diff --git a/lzma443/C/7zip/Compress/LZMA_Alone/LzmaRam.cpp b/lzma443/C/7zip/Compress/LZMA_Alone/LzmaRam.cpp new file mode 100755 index 0000000..090d73d --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA_Alone/LzmaRam.cpp @@ -0,0 +1,228 @@ +// LzmaRam.cpp + +#include "StdAfx.h" +#include "../../../Common/Types.h" +#include "../LZMA/LZMADecoder.h" +#include "../LZMA/LZMAEncoder.h" +#include "LzmaRam.h" + +extern "C" +{ +#include "../Branch/BranchX86.h" +} + +class CInStreamRam: + public ISequentialInStream, + public CMyUnknownImp +{ + const Byte *Data; + size_t Size; + size_t Pos; +public: + MY_UNKNOWN_IMP + void Init(const Byte *data, size_t size) + { + Data = data; + Size = size; + Pos = 0; + } + STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); +}; + +STDMETHODIMP CInStreamRam::Read(void *data, UInt32 size, UInt32 *processedSize) +{ + UInt32 remain = Size - Pos; + if (size > remain) + size = remain; + for (UInt32 i = 0; i < size; i++) + ((Byte *)data)[i] = Data[Pos + i]; + Pos += size; + if(processedSize != NULL) + *processedSize = size; + return S_OK; +} + +class COutStreamRam: + public ISequentialOutStream, + public CMyUnknownImp +{ + size_t Size; +public: + Byte *Data; + size_t Pos; + bool Overflow; + void Init(Byte *data, size_t size) + { + Data = data; + Size = size; + Pos = 0; + Overflow = false; + } + void SetPos(size_t pos) + { + Overflow = false; + Pos = pos; + } + MY_UNKNOWN_IMP + HRESULT WriteByte(Byte b) + { + if (Pos >= Size) + { + Overflow = true; + return E_FAIL; + } + Data[Pos++] = b; + return S_OK; + } + STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize); +}; + +STDMETHODIMP COutStreamRam::Write(const void *data, UInt32 size, UInt32 *processedSize) +{ + UInt32 i; + for (i = 0; i < size && Pos < Size; i++) + Data[Pos++] = ((const Byte *)data)[i]; + if(processedSize != NULL) + *processedSize = i; + if (i != size) + { + Overflow = true; + return E_FAIL; + } + return S_OK; +} + +#define SZE_FAIL (1) +#define SZE_OUTOFMEMORY (2) +#define SZE_OUT_OVERFLOW (3) + +int LzmaRamEncode( + const Byte *inBuffer, size_t inSize, + Byte *outBuffer, size_t outSize, size_t *outSizeProcessed, + UInt32 dictionarySize, ESzFilterMode filterMode) +{ + #ifndef _NO_EXCEPTIONS + try { + #endif + + *outSizeProcessed = 0; + const size_t kIdSize = 1; + const size_t kLzmaPropsSize = 5; + const size_t kMinDestSize = kIdSize + kLzmaPropsSize + 8; + if (outSize < kMinDestSize) + return SZE_OUT_OVERFLOW; + NCompress::NLZMA::CEncoder *encoderSpec = new NCompress::NLZMA::CEncoder; + CMyComPtr encoder = encoderSpec; + + PROPID propIDs[] = + { + NCoderPropID::kAlgorithm, + NCoderPropID::kDictionarySize, + NCoderPropID::kNumFastBytes, + }; + const int kNumProps = sizeof(propIDs) / sizeof(propIDs[0]); + PROPVARIANT properties[kNumProps]; + properties[0].vt = VT_UI4; + properties[1].vt = VT_UI4; + properties[2].vt = VT_UI4; + properties[0].ulVal = (UInt32)2; + properties[1].ulVal = (UInt32)dictionarySize; + properties[2].ulVal = (UInt32)64; + + if (encoderSpec->SetCoderProperties(propIDs, properties, kNumProps) != S_OK) + return 1; + + COutStreamRam *outStreamSpec = new COutStreamRam; + if (outStreamSpec == 0) + return SZE_OUTOFMEMORY; + CMyComPtr outStream = outStreamSpec; + CInStreamRam *inStreamSpec = new CInStreamRam; + if (inStreamSpec == 0) + return SZE_OUTOFMEMORY; + CMyComPtr inStream = inStreamSpec; + + outStreamSpec->Init(outBuffer, outSize); + if (outStreamSpec->WriteByte(0) != S_OK) + return SZE_OUT_OVERFLOW; + + if (encoderSpec->WriteCoderProperties(outStream) != S_OK) + return SZE_OUT_OVERFLOW; + if (outStreamSpec->Pos != kIdSize + kLzmaPropsSize) + return 1; + + int i; + for (i = 0; i < 8; i++) + { + UInt64 t = (UInt64)(inSize); + if (outStreamSpec->WriteByte((Byte)((t) >> (8 * i))) != S_OK) + return SZE_OUT_OVERFLOW; + } + + Byte *filteredStream = 0; + + bool useFilter = (filterMode != SZ_FILTER_NO); + if (useFilter) + { + if (inSize != 0) + { + filteredStream = (Byte *)MyAlloc(inSize); + if (filteredStream == 0) + return SZE_OUTOFMEMORY; + memmove(filteredStream, inBuffer, inSize); + } + UInt32 _prevMask; + UInt32 _prevPos; + x86_Convert_Init(_prevMask, _prevPos); + x86_Convert(filteredStream, (UInt32)inSize, 0, &_prevMask, &_prevPos, 1); + } + + UInt32 minSize = 0; + int numPasses = (filterMode == SZ_FILTER_AUTO) ? 3 : 1; + bool bestIsFiltered = false; + int mainResult = 0; + size_t startPos = outStreamSpec->Pos; + for (i = 0; i < numPasses; i++) + { + if (numPasses > 1 && i == numPasses - 1 && !bestIsFiltered) + break; + outStreamSpec->SetPos(startPos); + bool curModeIsFiltered = false; + if (useFilter && i == 0) + curModeIsFiltered = true; + if (numPasses > 1 && i == numPasses - 1) + curModeIsFiltered = true; + + inStreamSpec->Init(curModeIsFiltered ? filteredStream : inBuffer, inSize); + + HRESULT lzmaResult = encoder->Code(inStream, outStream, 0, 0, 0); + + mainResult = 0; + if (lzmaResult == E_OUTOFMEMORY) + { + mainResult = SZE_OUTOFMEMORY; + break; + } + if (i == 0 || outStreamSpec->Pos <= minSize) + { + minSize = outStreamSpec->Pos; + bestIsFiltered = curModeIsFiltered; + } + if (outStreamSpec->Overflow) + mainResult = SZE_OUT_OVERFLOW; + else if (lzmaResult != S_OK) + { + mainResult = SZE_FAIL; + break; + } + } + *outSizeProcessed = outStreamSpec->Pos; + if (bestIsFiltered) + outBuffer[0] = 1; + if (useFilter) + MyFree(filteredStream); + return mainResult; + + #ifndef _NO_EXCEPTIONS + } catch(...) { return SZE_OUTOFMEMORY; } + #endif +} diff --git a/lzma443/C/7zip/Compress/LZMA_Alone/LzmaRam.h b/lzma443/C/7zip/Compress/LZMA_Alone/LzmaRam.h new file mode 100755 index 0000000..1244dc8 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA_Alone/LzmaRam.h @@ -0,0 +1,46 @@ +// LzmaRam.h + +#ifndef __LzmaRam_h +#define __LzmaRam_h + +#include +#include "../../../Common/Types.h" + +/* +LzmaRamEncode: BCJ + LZMA RAM->RAM compressing. +It uses .lzma format, but it writes one additional byte to .lzma file: + 0: - no filter + 1: - x86(BCJ) filter. + +To provide best compression ratio dictionarySize mustbe >= inSize + +LzmaRamEncode allocates Data with MyAlloc/BigAlloc functions. +RAM Requirements: + RamSize = dictionarySize * 9.5 + 6MB + FilterBlockSize + FilterBlockSize = 0, if useFilter == false + FilterBlockSize = inSize, if useFilter == true + + Return code: + 0 - OK + 1 - Unspecified Error + 2 - Memory allocating error + 3 - Output buffer OVERFLOW + +If you use SZ_FILTER_AUTO mode, then encoder will use 2 or 3 passes: + 2 passes when FILTER_NO provides better compression. + 3 passes when FILTER_YES provides better compression. +*/ + +enum ESzFilterMode +{ + SZ_FILTER_NO, + SZ_FILTER_YES, + SZ_FILTER_AUTO +}; + +int LzmaRamEncode( + const Byte *inBuffer, size_t inSize, + Byte *outBuffer, size_t outSize, size_t *outSizeProcessed, + UInt32 dictionarySize, ESzFilterMode filterMode); + +#endif diff --git a/lzma443/C/7zip/Compress/LZMA_Alone/LzmaRamDecode.c b/lzma443/C/7zip/Compress/LZMA_Alone/LzmaRamDecode.c new file mode 100755 index 0000000..ed1784d --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA_Alone/LzmaRamDecode.c @@ -0,0 +1,79 @@ +/* LzmaRamDecode.c */ + +#include "LzmaRamDecode.h" +#ifdef _SZ_ONE_DIRECTORY +#include "LzmaDecode.h" +#include "BranchX86.h" +#else +#include "../LZMA_C/LzmaDecode.h" +#include "../Branch/BranchX86.h" +#endif + +#define LZMA_PROPS_SIZE 14 +#define LZMA_SIZE_OFFSET 6 + +int LzmaRamGetUncompressedSize( + const unsigned char *inBuffer, + size_t inSize, + size_t *outSize) +{ + unsigned int i; + if (inSize < LZMA_PROPS_SIZE) + return 1; + *outSize = 0; + for(i = 0; i < sizeof(size_t); i++) + *outSize += ((size_t)inBuffer[LZMA_SIZE_OFFSET + i]) << (8 * i); + for(; i < 8; i++) + if (inBuffer[LZMA_SIZE_OFFSET + i] != 0) + return 1; + return 0; +} + +#define SZE_DATA_ERROR (1) +#define SZE_OUTOFMEMORY (2) + +int LzmaRamDecompress( + const unsigned char *inBuffer, + size_t inSize, + unsigned char *outBuffer, + size_t outSize, + size_t *outSizeProcessed, + void * (*allocFunc)(size_t size), + void (*freeFunc)(void *)) +{ + CLzmaDecoderState state; /* it's about 24 bytes structure, if int is 32-bit */ + int result; + SizeT outSizeProcessedLoc; + SizeT inProcessed; + int useFilter; + + if (inSize < LZMA_PROPS_SIZE) + return 1; + useFilter = inBuffer[0]; + + *outSizeProcessed = 0; + if (useFilter > 1) + return 1; + + if (LzmaDecodeProperties(&state.Properties, inBuffer + 1, LZMA_PROPERTIES_SIZE) != LZMA_RESULT_OK) + return 1; + state.Probs = (CProb *)allocFunc(LzmaGetNumProbs(&state.Properties) * sizeof(CProb)); + if (state.Probs == 0) + return SZE_OUTOFMEMORY; + + result = LzmaDecode(&state, + inBuffer + LZMA_PROPS_SIZE, (SizeT)inSize - LZMA_PROPS_SIZE, &inProcessed, + outBuffer, (SizeT)outSize, &outSizeProcessedLoc); + freeFunc(state.Probs); + if (result != LZMA_RESULT_OK) + return 1; + *outSizeProcessed = (size_t)outSizeProcessedLoc; + if (useFilter == 1) + { + UInt32 _prevMask; + UInt32 _prevPos; + x86_Convert_Init(_prevMask, _prevPos); + x86_Convert(outBuffer, (UInt32)outSizeProcessedLoc, 0, &_prevMask, &_prevPos, 0); + } + return 0; +} diff --git a/lzma443/C/7zip/Compress/LZMA_Alone/LzmaRamDecode.h b/lzma443/C/7zip/Compress/LZMA_Alone/LzmaRamDecode.h new file mode 100755 index 0000000..7e641c5 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA_Alone/LzmaRamDecode.h @@ -0,0 +1,55 @@ +/* LzmaRamDecode.h */ + +#ifndef __LzmaRamDecode_h +#define __LzmaRamDecode_h + +#include + +/* +LzmaRamGetUncompressedSize: + In: + inBuffer - input data + inSize - input data size + Out: + outSize - uncompressed size + Return code: + 0 - OK + 1 - Error in headers +*/ + +int LzmaRamGetUncompressedSize( + const unsigned char *inBuffer, + size_t inSize, + size_t *outSize); + + +/* +LzmaRamDecompress: + In: + inBuffer - input data + inSize - input data size + outBuffer - output data + outSize - output size + allocFunc - alloc function (can be malloc) + freeFunc - free function (can be free) + Out: + outSizeProcessed - processed size + Return code: + 0 - OK + 1 - Error in headers / data stream + 2 - Memory allocating error + +Memory requirements depend from properties of LZMA stream. +With default lzma settings it's about 16 KB. +*/ + +int LzmaRamDecompress( + const unsigned char *inBuffer, + size_t inSize, + unsigned char *outBuffer, + size_t outSize, + size_t *outSizeProcessed, + void * (*allocFunc)(size_t size), + void (*freeFunc)(void *)); + +#endif diff --git a/lzma443/C/7zip/Compress/LZMA_Alone/StdAfx.cpp b/lzma443/C/7zip/Compress/LZMA_Alone/StdAfx.cpp new file mode 100755 index 0000000..d0feea8 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA_Alone/StdAfx.cpp @@ -0,0 +1,3 @@ +// StdAfx.cpp + +#include "StdAfx.h" diff --git a/lzma443/C/7zip/Compress/LZMA_Alone/StdAfx.h b/lzma443/C/7zip/Compress/LZMA_Alone/StdAfx.h new file mode 100755 index 0000000..e7fb698 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA_Alone/StdAfx.h @@ -0,0 +1,8 @@ +// StdAfx.h + +#ifndef __STDAFX_H +#define __STDAFX_H + +#include "../../../Common/MyWindows.h" + +#endif diff --git a/lzma443/C/7zip/Compress/LZMA_Alone/lzma b/lzma443/C/7zip/Compress/LZMA_Alone/lzma new file mode 100755 index 0000000..665873c Binary files /dev/null and b/lzma443/C/7zip/Compress/LZMA_Alone/lzma differ diff --git a/lzma443/C/7zip/Compress/LZMA_Alone/makefile.gcc b/lzma443/C/7zip/Compress/LZMA_Alone/makefile.gcc new file mode 100755 index 0000000..c43237b --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA_Alone/makefile.gcc @@ -0,0 +1,113 @@ +PROG = lzma +CXX = g++ -O2 -Wall +CXX_C = gcc -O2 -Wall +LIB = -lm +RM = rm -f +CFLAGS = -c -I ../../../ + +OBJS = \ + LzmaAlone.o \ + LzmaBench.o \ + LzmaRam.o \ + LzmaRamDecode.o \ + LzmaDecode.o \ + BranchX86.o \ + LZMADecoder.o \ + LZMAEncoder.o \ + LZInWindow.o \ + LZOutWindow.o \ + RangeCoderBit.o \ + InBuffer.o \ + OutBuffer.o \ + FileStreams.o \ + StreamUtils.o \ + Alloc.o \ + C_FileIO.o \ + CommandLineParser.o \ + CRC.o \ + String.o \ + StringConvert.o \ + StringToInt.o \ + Vector.o \ + + +all: $(PROG) + +$(PROG): $(OBJS) + $(CXX) -o $(PROG) $(LDFLAGS) $(OBJS) $(LIB) + +LzmaAlone.o: LzmaAlone.cpp + $(CXX) $(CFLAGS) LzmaAlone.cpp + +LzmaBench.o: LzmaBench.cpp + $(CXX) $(CFLAGS) LzmaBench.cpp + +LzmaRam.o: LzmaRam.cpp + $(CXX) $(CFLAGS) LzmaRam.cpp + +LzmaRamDecode.o: LzmaRamDecode.c + $(CXX_C) $(CFLAGS) LzmaRamDecode.c + +LzmaDecode.o: ../LZMA_C/LzmaDecode.c + $(CXX_C) $(CFLAGS) ../LZMA_C/LzmaDecode.c + +BranchX86.o: ../Branch/BranchX86.c + $(CXX_C) $(CFLAGS) ../Branch/BranchX86.c + +LZMADecoder.o: ../LZMA/LZMADecoder.cpp + $(CXX) $(CFLAGS) ../LZMA/LZMADecoder.cpp + +LZMAEncoder.o: ../LZMA/LZMAEncoder.cpp + $(CXX) $(CFLAGS) ../LZMA/LZMAEncoder.cpp + +LZInWindow.o: ../LZ/LZInWindow.cpp + $(CXX) $(CFLAGS) ../LZ/LZInWindow.cpp + +LZOutWindow.o: ../LZ/LZOutWindow.cpp + $(CXX) $(CFLAGS) ../LZ/LZOutWindow.cpp + +RangeCoderBit.o: ../RangeCoder/RangeCoderBit.cpp + $(CXX) $(CFLAGS) ../RangeCoder/RangeCoderBit.cpp + +InBuffer.o: ../../Common/InBuffer.cpp + $(CXX) $(CFLAGS) ../../Common/InBuffer.cpp + +OutBuffer.o: ../../Common/OutBuffer.cpp + $(CXX) $(CFLAGS) ../../Common/OutBuffer.cpp + +FileStreams.o: ../../Common/FileStreams.cpp + $(CXX) $(CFLAGS) ../../Common/FileStreams.cpp + +StreamUtils.o: ../../Common/StreamUtils.cpp + $(CXX) $(CFLAGS) ../../Common/StreamUtils.cpp + +Alloc.o: ../../../Common/Alloc.cpp + $(CXX) $(CFLAGS) ../../../Common/Alloc.cpp + +C_FileIO.o: ../../../Common/C_FileIO.cpp + $(CXX) $(CFLAGS) ../../../Common/C_FileIO.cpp + +CommandLineParser.o: ../../../Common/CommandLineParser.cpp + $(CXX) $(CFLAGS) ../../../Common/CommandLineParser.cpp + +CRC.o: ../../../Common/CRC.cpp + $(CXX) $(CFLAGS) ../../../Common/CRC.cpp + +MyWindows.o: ../../../Common/MyWindows.cpp + $(CXX) $(CFLAGS) ../../../Common/MyWindows.cpp + +String.o: ../../../Common/String.cpp + $(CXX) $(CFLAGS) ../../../Common/String.cpp + +StringConvert.o: ../../../Common/StringConvert.cpp + $(CXX) $(CFLAGS) ../../../Common/StringConvert.cpp + +StringToInt.o: ../../../Common/StringToInt.cpp + $(CXX) $(CFLAGS) ../../../Common/StringToInt.cpp + +Vector.o: ../../../Common/Vector.cpp + $(CXX) $(CFLAGS) ../../../Common/Vector.cpp + +clean: + -$(RM) $(PROG) $(OBJS) + diff --git a/lzma443/C/7zip/Compress/LZMA_C/LzmaDecode.c b/lzma443/C/7zip/Compress/LZMA_C/LzmaDecode.c new file mode 100755 index 0000000..cb83453 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA_C/LzmaDecode.c @@ -0,0 +1,584 @@ +/* + LzmaDecode.c + LZMA Decoder (optimized for Speed version) + + LZMA SDK 4.40 Copyright (c) 1999-2006 Igor Pavlov (2006-05-01) + http://www.7-zip.org/ + + LZMA SDK is licensed under two licenses: + 1) GNU Lesser General Public License (GNU LGPL) + 2) Common Public License (CPL) + It means that you can select one of these two licenses and + follow rules of that license. + + SPECIAL EXCEPTION: + Igor Pavlov, as the author of this Code, expressly permits you to + statically or dynamically link your Code (or bind by name) to the + interfaces of this file without subjecting your linked Code to the + terms of the CPL or GNU LGPL. Any modifications or additions + to this file, however, are subject to the LGPL or CPL terms. +*/ + +#include "LzmaDecode.h" + +#define kNumTopBits 24 +#define kTopValue ((UInt32)1 << kNumTopBits) + +#define kNumBitModelTotalBits 11 +#define kBitModelTotal (1 << kNumBitModelTotalBits) +#define kNumMoveBits 5 + +#define RC_READ_BYTE (*Buffer++) + +#define RC_INIT2 Code = 0; Range = 0xFFFFFFFF; \ + { int i; for(i = 0; i < 5; i++) { RC_TEST; Code = (Code << 8) | RC_READ_BYTE; }} + +#ifdef _LZMA_IN_CB + +#define RC_TEST { if (Buffer == BufferLim) \ + { SizeT size; int result = InCallback->Read(InCallback, &Buffer, &size); if (result != LZMA_RESULT_OK) return result; \ + BufferLim = Buffer + size; if (size == 0) return LZMA_RESULT_DATA_ERROR; }} + +#define RC_INIT Buffer = BufferLim = 0; RC_INIT2 + +#else + +#define RC_TEST { if (Buffer == BufferLim) return LZMA_RESULT_DATA_ERROR; } + +#define RC_INIT(buffer, bufferSize) Buffer = buffer; BufferLim = buffer + bufferSize; RC_INIT2 + +#endif + +#define RC_NORMALIZE if (Range < kTopValue) { RC_TEST; Range <<= 8; Code = (Code << 8) | RC_READ_BYTE; } + +#define IfBit0(p) RC_NORMALIZE; bound = (Range >> kNumBitModelTotalBits) * *(p); if (Code < bound) +#define UpdateBit0(p) Range = bound; *(p) += (kBitModelTotal - *(p)) >> kNumMoveBits; +#define UpdateBit1(p) Range -= bound; Code -= bound; *(p) -= (*(p)) >> kNumMoveBits; + +#define RC_GET_BIT2(p, mi, A0, A1) IfBit0(p) \ + { UpdateBit0(p); mi <<= 1; A0; } else \ + { UpdateBit1(p); mi = (mi + mi) + 1; A1; } + +#define RC_GET_BIT(p, mi) RC_GET_BIT2(p, mi, ; , ;) + +#define RangeDecoderBitTreeDecode(probs, numLevels, res) \ + { int i = numLevels; res = 1; \ + do { CProb *p = probs + res; RC_GET_BIT(p, res) } while(--i != 0); \ + res -= (1 << numLevels); } + + +#define kNumPosBitsMax 4 +#define kNumPosStatesMax (1 << kNumPosBitsMax) + +#define kLenNumLowBits 3 +#define kLenNumLowSymbols (1 << kLenNumLowBits) +#define kLenNumMidBits 3 +#define kLenNumMidSymbols (1 << kLenNumMidBits) +#define kLenNumHighBits 8 +#define kLenNumHighSymbols (1 << kLenNumHighBits) + +#define LenChoice 0 +#define LenChoice2 (LenChoice + 1) +#define LenLow (LenChoice2 + 1) +#define LenMid (LenLow + (kNumPosStatesMax << kLenNumLowBits)) +#define LenHigh (LenMid + (kNumPosStatesMax << kLenNumMidBits)) +#define kNumLenProbs (LenHigh + kLenNumHighSymbols) + + +#define kNumStates 12 +#define kNumLitStates 7 + +#define kStartPosModelIndex 4 +#define kEndPosModelIndex 14 +#define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) + +#define kNumPosSlotBits 6 +#define kNumLenToPosStates 4 + +#define kNumAlignBits 4 +#define kAlignTableSize (1 << kNumAlignBits) + +#define kMatchMinLen 2 + +#define IsMatch 0 +#define IsRep (IsMatch + (kNumStates << kNumPosBitsMax)) +#define IsRepG0 (IsRep + kNumStates) +#define IsRepG1 (IsRepG0 + kNumStates) +#define IsRepG2 (IsRepG1 + kNumStates) +#define IsRep0Long (IsRepG2 + kNumStates) +#define PosSlot (IsRep0Long + (kNumStates << kNumPosBitsMax)) +#define SpecPos (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) +#define Align (SpecPos + kNumFullDistances - kEndPosModelIndex) +#define LenCoder (Align + kAlignTableSize) +#define RepLenCoder (LenCoder + kNumLenProbs) +#define Literal (RepLenCoder + kNumLenProbs) + +#if Literal != LZMA_BASE_SIZE +StopCompilingDueBUG +#endif + +int LzmaDecodeProperties(CLzmaProperties *propsRes, const unsigned char *propsData, int size) +{ + unsigned char prop0; + if (size < LZMA_PROPERTIES_SIZE) + return LZMA_RESULT_DATA_ERROR; + prop0 = propsData[0]; + if (prop0 >= (9 * 5 * 5)) + return LZMA_RESULT_DATA_ERROR; + { + for (propsRes->pb = 0; prop0 >= (9 * 5); propsRes->pb++, prop0 -= (9 * 5)); + for (propsRes->lp = 0; prop0 >= 9; propsRes->lp++, prop0 -= 9); + propsRes->lc = prop0; + /* + unsigned char remainder = (unsigned char)(prop0 / 9); + propsRes->lc = prop0 % 9; + propsRes->pb = remainder / 5; + propsRes->lp = remainder % 5; + */ + } + + #ifdef _LZMA_OUT_READ + { + int i; + propsRes->DictionarySize = 0; + for (i = 0; i < 4; i++) + propsRes->DictionarySize += (UInt32)(propsData[1 + i]) << (i * 8); + if (propsRes->DictionarySize == 0) + propsRes->DictionarySize = 1; + } + #endif + return LZMA_RESULT_OK; +} + +#define kLzmaStreamWasFinishedId (-1) + +int LzmaDecode(CLzmaDecoderState *vs, + #ifdef _LZMA_IN_CB + ILzmaInCallback *InCallback, + #else + const unsigned char *inStream, SizeT inSize, SizeT *inSizeProcessed, + #endif + unsigned char *outStream, SizeT outSize, SizeT *outSizeProcessed) +{ + CProb *p = vs->Probs; + SizeT nowPos = 0; + Byte previousByte = 0; + UInt32 posStateMask = (1 << (vs->Properties.pb)) - 1; + UInt32 literalPosMask = (1 << (vs->Properties.lp)) - 1; + int lc = vs->Properties.lc; + + #ifdef _LZMA_OUT_READ + + UInt32 Range = vs->Range; + UInt32 Code = vs->Code; + #ifdef _LZMA_IN_CB + const Byte *Buffer = vs->Buffer; + const Byte *BufferLim = vs->BufferLim; + #else + const Byte *Buffer = inStream; + const Byte *BufferLim = inStream + inSize; + #endif + int state = vs->State; + UInt32 rep0 = vs->Reps[0], rep1 = vs->Reps[1], rep2 = vs->Reps[2], rep3 = vs->Reps[3]; + int len = vs->RemainLen; + UInt32 globalPos = vs->GlobalPos; + UInt32 distanceLimit = vs->DistanceLimit; + + Byte *dictionary = vs->Dictionary; + UInt32 dictionarySize = vs->Properties.DictionarySize; + UInt32 dictionaryPos = vs->DictionaryPos; + + Byte tempDictionary[4]; + + #ifndef _LZMA_IN_CB + *inSizeProcessed = 0; + #endif + *outSizeProcessed = 0; + if (len == kLzmaStreamWasFinishedId) + return LZMA_RESULT_OK; + + if (dictionarySize == 0) + { + dictionary = tempDictionary; + dictionarySize = 1; + tempDictionary[0] = vs->TempDictionary[0]; + } + + if (len == kLzmaNeedInitId) + { + { + UInt32 numProbs = Literal + ((UInt32)LZMA_LIT_SIZE << (lc + vs->Properties.lp)); + UInt32 i; + for (i = 0; i < numProbs; i++) + p[i] = kBitModelTotal >> 1; + rep0 = rep1 = rep2 = rep3 = 1; + state = 0; + globalPos = 0; + distanceLimit = 0; + dictionaryPos = 0; + dictionary[dictionarySize - 1] = 0; + #ifdef _LZMA_IN_CB + RC_INIT; + #else + RC_INIT(inStream, inSize); + #endif + } + len = 0; + } + while(len != 0 && nowPos < outSize) + { + UInt32 pos = dictionaryPos - rep0; + if (pos >= dictionarySize) + pos += dictionarySize; + outStream[nowPos++] = dictionary[dictionaryPos] = dictionary[pos]; + if (++dictionaryPos == dictionarySize) + dictionaryPos = 0; + len--; + } + if (dictionaryPos == 0) + previousByte = dictionary[dictionarySize - 1]; + else + previousByte = dictionary[dictionaryPos - 1]; + + #else /* if !_LZMA_OUT_READ */ + + int state = 0; + UInt32 rep0 = 1, rep1 = 1, rep2 = 1, rep3 = 1; + int len = 0; + const Byte *Buffer; + const Byte *BufferLim; + UInt32 Range; + UInt32 Code; + + #ifndef _LZMA_IN_CB + *inSizeProcessed = 0; + #endif + *outSizeProcessed = 0; + + { + UInt32 i; + UInt32 numProbs = Literal + ((UInt32)LZMA_LIT_SIZE << (lc + vs->Properties.lp)); + for (i = 0; i < numProbs; i++) + p[i] = kBitModelTotal >> 1; + } + + #ifdef _LZMA_IN_CB + RC_INIT; + #else + RC_INIT(inStream, inSize); + #endif + + #endif /* _LZMA_OUT_READ */ + + while(nowPos < outSize) + { + CProb *prob; + UInt32 bound; + int posState = (int)( + (nowPos + #ifdef _LZMA_OUT_READ + + globalPos + #endif + ) + & posStateMask); + + prob = p + IsMatch + (state << kNumPosBitsMax) + posState; + IfBit0(prob) + { + int symbol = 1; + UpdateBit0(prob) + prob = p + Literal + (LZMA_LIT_SIZE * + ((( + (nowPos + #ifdef _LZMA_OUT_READ + + globalPos + #endif + ) + & literalPosMask) << lc) + (previousByte >> (8 - lc)))); + + if (state >= kNumLitStates) + { + int matchByte; + #ifdef _LZMA_OUT_READ + UInt32 pos = dictionaryPos - rep0; + if (pos >= dictionarySize) + pos += dictionarySize; + matchByte = dictionary[pos]; + #else + matchByte = outStream[nowPos - rep0]; + #endif + do + { + int bit; + CProb *probLit; + matchByte <<= 1; + bit = (matchByte & 0x100); + probLit = prob + 0x100 + bit + symbol; + RC_GET_BIT2(probLit, symbol, if (bit != 0) break, if (bit == 0) break) + } + while (symbol < 0x100); + } + while (symbol < 0x100) + { + CProb *probLit = prob + symbol; + RC_GET_BIT(probLit, symbol) + } + previousByte = (Byte)symbol; + + outStream[nowPos++] = previousByte; + #ifdef _LZMA_OUT_READ + if (distanceLimit < dictionarySize) + distanceLimit++; + + dictionary[dictionaryPos] = previousByte; + if (++dictionaryPos == dictionarySize) + dictionaryPos = 0; + #endif + if (state < 4) state = 0; + else if (state < 10) state -= 3; + else state -= 6; + } + else + { + UpdateBit1(prob); + prob = p + IsRep + state; + IfBit0(prob) + { + UpdateBit0(prob); + rep3 = rep2; + rep2 = rep1; + rep1 = rep0; + state = state < kNumLitStates ? 0 : 3; + prob = p + LenCoder; + } + else + { + UpdateBit1(prob); + prob = p + IsRepG0 + state; + IfBit0(prob) + { + UpdateBit0(prob); + prob = p + IsRep0Long + (state << kNumPosBitsMax) + posState; + IfBit0(prob) + { + #ifdef _LZMA_OUT_READ + UInt32 pos; + #endif + UpdateBit0(prob); + + #ifdef _LZMA_OUT_READ + if (distanceLimit == 0) + #else + if (nowPos == 0) + #endif + return LZMA_RESULT_DATA_ERROR; + + state = state < kNumLitStates ? 9 : 11; + #ifdef _LZMA_OUT_READ + pos = dictionaryPos - rep0; + if (pos >= dictionarySize) + pos += dictionarySize; + previousByte = dictionary[pos]; + dictionary[dictionaryPos] = previousByte; + if (++dictionaryPos == dictionarySize) + dictionaryPos = 0; + #else + previousByte = outStream[nowPos - rep0]; + #endif + outStream[nowPos++] = previousByte; + #ifdef _LZMA_OUT_READ + if (distanceLimit < dictionarySize) + distanceLimit++; + #endif + + continue; + } + else + { + UpdateBit1(prob); + } + } + else + { + UInt32 distance; + UpdateBit1(prob); + prob = p + IsRepG1 + state; + IfBit0(prob) + { + UpdateBit0(prob); + distance = rep1; + } + else + { + UpdateBit1(prob); + prob = p + IsRepG2 + state; + IfBit0(prob) + { + UpdateBit0(prob); + distance = rep2; + } + else + { + UpdateBit1(prob); + distance = rep3; + rep3 = rep2; + } + rep2 = rep1; + } + rep1 = rep0; + rep0 = distance; + } + state = state < kNumLitStates ? 8 : 11; + prob = p + RepLenCoder; + } + { + int numBits, offset; + CProb *probLen = prob + LenChoice; + IfBit0(probLen) + { + UpdateBit0(probLen); + probLen = prob + LenLow + (posState << kLenNumLowBits); + offset = 0; + numBits = kLenNumLowBits; + } + else + { + UpdateBit1(probLen); + probLen = prob + LenChoice2; + IfBit0(probLen) + { + UpdateBit0(probLen); + probLen = prob + LenMid + (posState << kLenNumMidBits); + offset = kLenNumLowSymbols; + numBits = kLenNumMidBits; + } + else + { + UpdateBit1(probLen); + probLen = prob + LenHigh; + offset = kLenNumLowSymbols + kLenNumMidSymbols; + numBits = kLenNumHighBits; + } + } + RangeDecoderBitTreeDecode(probLen, numBits, len); + len += offset; + } + + if (state < 4) + { + int posSlot; + state += kNumLitStates; + prob = p + PosSlot + + ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << + kNumPosSlotBits); + RangeDecoderBitTreeDecode(prob, kNumPosSlotBits, posSlot); + if (posSlot >= kStartPosModelIndex) + { + int numDirectBits = ((posSlot >> 1) - 1); + rep0 = (2 | ((UInt32)posSlot & 1)); + if (posSlot < kEndPosModelIndex) + { + rep0 <<= numDirectBits; + prob = p + SpecPos + rep0 - posSlot - 1; + } + else + { + numDirectBits -= kNumAlignBits; + do + { + RC_NORMALIZE + Range >>= 1; + rep0 <<= 1; + if (Code >= Range) + { + Code -= Range; + rep0 |= 1; + } + } + while (--numDirectBits != 0); + prob = p + Align; + rep0 <<= kNumAlignBits; + numDirectBits = kNumAlignBits; + } + { + int i = 1; + int mi = 1; + do + { + CProb *prob3 = prob + mi; + RC_GET_BIT2(prob3, mi, ; , rep0 |= i); + i <<= 1; + } + while(--numDirectBits != 0); + } + } + else + rep0 = posSlot; + if (++rep0 == (UInt32)(0)) + { + /* it's for stream version */ + len = kLzmaStreamWasFinishedId; + break; + } + } + + len += kMatchMinLen; + #ifdef _LZMA_OUT_READ + if (rep0 > distanceLimit) + #else + if (rep0 > nowPos) + #endif + return LZMA_RESULT_DATA_ERROR; + + #ifdef _LZMA_OUT_READ + if (dictionarySize - distanceLimit > (UInt32)len) + distanceLimit += len; + else + distanceLimit = dictionarySize; + #endif + + do + { + #ifdef _LZMA_OUT_READ + UInt32 pos = dictionaryPos - rep0; + if (pos >= dictionarySize) + pos += dictionarySize; + previousByte = dictionary[pos]; + dictionary[dictionaryPos] = previousByte; + if (++dictionaryPos == dictionarySize) + dictionaryPos = 0; + #else + previousByte = outStream[nowPos - rep0]; + #endif + len--; + outStream[nowPos++] = previousByte; + } + while(len != 0 && nowPos < outSize); + } + } + RC_NORMALIZE; + + #ifdef _LZMA_OUT_READ + vs->Range = Range; + vs->Code = Code; + vs->DictionaryPos = dictionaryPos; + vs->GlobalPos = globalPos + (UInt32)nowPos; + vs->DistanceLimit = distanceLimit; + vs->Reps[0] = rep0; + vs->Reps[1] = rep1; + vs->Reps[2] = rep2; + vs->Reps[3] = rep3; + vs->State = state; + vs->RemainLen = len; + vs->TempDictionary[0] = tempDictionary[0]; + #endif + + #ifdef _LZMA_IN_CB + vs->Buffer = Buffer; + vs->BufferLim = BufferLim; + #else + *inSizeProcessed = (SizeT)(Buffer - inStream); + #endif + *outSizeProcessed = nowPos; + return LZMA_RESULT_OK; +} diff --git a/lzma443/C/7zip/Compress/LZMA_C/LzmaDecode.h b/lzma443/C/7zip/Compress/LZMA_C/LzmaDecode.h new file mode 100755 index 0000000..2870eeb --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA_C/LzmaDecode.h @@ -0,0 +1,113 @@ +/* + LzmaDecode.h + LZMA Decoder interface + + LZMA SDK 4.40 Copyright (c) 1999-2006 Igor Pavlov (2006-05-01) + http://www.7-zip.org/ + + LZMA SDK is licensed under two licenses: + 1) GNU Lesser General Public License (GNU LGPL) + 2) Common Public License (CPL) + It means that you can select one of these two licenses and + follow rules of that license. + + SPECIAL EXCEPTION: + Igor Pavlov, as the author of this code, expressly permits you to + statically or dynamically link your code (or bind by name) to the + interfaces of this file without subjecting your linked code to the + terms of the CPL or GNU LGPL. Any modifications or additions + to this file, however, are subject to the LGPL or CPL terms. +*/ + +#ifndef __LZMADECODE_H +#define __LZMADECODE_H + +#include "LzmaTypes.h" + +/* #define _LZMA_IN_CB */ +/* Use callback for input data */ + +/* #define _LZMA_OUT_READ */ +/* Use read function for output data */ + +/* #define _LZMA_PROB32 */ +/* It can increase speed on some 32-bit CPUs, + but memory usage will be doubled in that case */ + +/* #define _LZMA_LOC_OPT */ +/* Enable local speed optimizations inside code */ + +#ifdef _LZMA_PROB32 +#define CProb UInt32 +#else +#define CProb UInt16 +#endif + +#define LZMA_RESULT_OK 0 +#define LZMA_RESULT_DATA_ERROR 1 + +#ifdef _LZMA_IN_CB +typedef struct _ILzmaInCallback +{ + int (*Read)(void *object, const unsigned char **buffer, SizeT *bufferSize); +} ILzmaInCallback; +#endif + +#define LZMA_BASE_SIZE 1846 +#define LZMA_LIT_SIZE 768 + +#define LZMA_PROPERTIES_SIZE 5 + +typedef struct _CLzmaProperties +{ + int lc; + int lp; + int pb; + #ifdef _LZMA_OUT_READ + UInt32 DictionarySize; + #endif +}CLzmaProperties; + +int LzmaDecodeProperties(CLzmaProperties *propsRes, const unsigned char *propsData, int size); + +#define LzmaGetNumProbs(Properties) (LZMA_BASE_SIZE + (LZMA_LIT_SIZE << ((Properties)->lc + (Properties)->lp))) + +#define kLzmaNeedInitId (-2) + +typedef struct _CLzmaDecoderState +{ + CLzmaProperties Properties; + CProb *Probs; + + #ifdef _LZMA_IN_CB + const unsigned char *Buffer; + const unsigned char *BufferLim; + #endif + + #ifdef _LZMA_OUT_READ + unsigned char *Dictionary; + UInt32 Range; + UInt32 Code; + UInt32 DictionaryPos; + UInt32 GlobalPos; + UInt32 DistanceLimit; + UInt32 Reps[4]; + int State; + int RemainLen; + unsigned char TempDictionary[4]; + #endif +} CLzmaDecoderState; + +#ifdef _LZMA_OUT_READ +#define LzmaDecoderInit(vs) { (vs)->RemainLen = kLzmaNeedInitId; } +#endif + +int LzmaDecode(CLzmaDecoderState *vs, + #ifdef _LZMA_IN_CB + ILzmaInCallback *inCallback, + #else + const unsigned char *inStream, SizeT inSize, SizeT *inSizeProcessed, + #endif + unsigned char *outStream, SizeT outSize, SizeT *outSizeProcessed); + +#endif diff --git a/lzma443/C/7zip/Compress/LZMA_C/LzmaDecodeSize.c b/lzma443/C/7zip/Compress/LZMA_C/LzmaDecodeSize.c new file mode 100755 index 0000000..a3a5eb9 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA_C/LzmaDecodeSize.c @@ -0,0 +1,712 @@ +/* + LzmaDecodeSize.c + LZMA Decoder (optimized for Size version) + + LZMA SDK 4.40 Copyright (c) 1999-2006 Igor Pavlov (2006-05-01) + http://www.7-zip.org/ + + LZMA SDK is licensed under two licenses: + 1) GNU Lesser General Public License (GNU LGPL) + 2) Common Public License (CPL) + It means that you can select one of these two licenses and + follow rules of that license. + + SPECIAL EXCEPTION: + Igor Pavlov, as the author of this code, expressly permits you to + statically or dynamically link your code (or bind by name) to the + interfaces of this file without subjecting your linked code to the + terms of the CPL or GNU LGPL. Any modifications or additions + to this file, however, are subject to the LGPL or CPL terms. +*/ + +#include "LzmaDecode.h" + +#define kNumTopBits 24 +#define kTopValue ((UInt32)1 << kNumTopBits) + +#define kNumBitModelTotalBits 11 +#define kBitModelTotal (1 << kNumBitModelTotalBits) +#define kNumMoveBits 5 + +typedef struct _CRangeDecoder +{ + const Byte *Buffer; + const Byte *BufferLim; + UInt32 Range; + UInt32 Code; + #ifdef _LZMA_IN_CB + ILzmaInCallback *InCallback; + int Result; + #endif + int ExtraBytes; +} CRangeDecoder; + +Byte RangeDecoderReadByte(CRangeDecoder *rd) +{ + if (rd->Buffer == rd->BufferLim) + { + #ifdef _LZMA_IN_CB + SizeT size; + rd->Result = rd->InCallback->Read(rd->InCallback, &rd->Buffer, &size); + rd->BufferLim = rd->Buffer + size; + if (size == 0) + #endif + { + rd->ExtraBytes = 1; + return 0xFF; + } + } + return (*rd->Buffer++); +} + +/* #define ReadByte (*rd->Buffer++) */ +#define ReadByte (RangeDecoderReadByte(rd)) + +void RangeDecoderInit(CRangeDecoder *rd + #ifndef _LZMA_IN_CB + , const Byte *stream, SizeT bufferSize + #endif + ) +{ + int i; + #ifdef _LZMA_IN_CB + rd->Buffer = rd->BufferLim = 0; + #else + rd->Buffer = stream; + rd->BufferLim = stream + bufferSize; + #endif + rd->ExtraBytes = 0; + rd->Code = 0; + rd->Range = (0xFFFFFFFF); + for(i = 0; i < 5; i++) + rd->Code = (rd->Code << 8) | ReadByte; +} + +#define RC_INIT_VAR UInt32 range = rd->Range; UInt32 code = rd->Code; +#define RC_FLUSH_VAR rd->Range = range; rd->Code = code; +#define RC_NORMALIZE if (range < kTopValue) { range <<= 8; code = (code << 8) | ReadByte; } + +UInt32 RangeDecoderDecodeDirectBits(CRangeDecoder *rd, int numTotalBits) +{ + RC_INIT_VAR + UInt32 result = 0; + int i; + for (i = numTotalBits; i != 0; i--) + { + /* UInt32 t; */ + range >>= 1; + + result <<= 1; + if (code >= range) + { + code -= range; + result |= 1; + } + /* + t = (code - range) >> 31; + t &= 1; + code -= range & (t - 1); + result = (result + result) | (1 - t); + */ + RC_NORMALIZE + } + RC_FLUSH_VAR + return result; +} + +int RangeDecoderBitDecode(CProb *prob, CRangeDecoder *rd) +{ + UInt32 bound = (rd->Range >> kNumBitModelTotalBits) * *prob; + if (rd->Code < bound) + { + rd->Range = bound; + *prob += (kBitModelTotal - *prob) >> kNumMoveBits; + if (rd->Range < kTopValue) + { + rd->Code = (rd->Code << 8) | ReadByte; + rd->Range <<= 8; + } + return 0; + } + else + { + rd->Range -= bound; + rd->Code -= bound; + *prob -= (*prob) >> kNumMoveBits; + if (rd->Range < kTopValue) + { + rd->Code = (rd->Code << 8) | ReadByte; + rd->Range <<= 8; + } + return 1; + } +} + +#define RC_GET_BIT2(prob, mi, A0, A1) \ + UInt32 bound = (range >> kNumBitModelTotalBits) * *prob; \ + if (code < bound) \ + { A0; range = bound; *prob += (kBitModelTotal - *prob) >> kNumMoveBits; mi <<= 1; } \ + else \ + { A1; range -= bound; code -= bound; *prob -= (*prob) >> kNumMoveBits; mi = (mi + mi) + 1; } \ + RC_NORMALIZE + +#define RC_GET_BIT(prob, mi) RC_GET_BIT2(prob, mi, ; , ;) + +int RangeDecoderBitTreeDecode(CProb *probs, int numLevels, CRangeDecoder *rd) +{ + int mi = 1; + int i; + #ifdef _LZMA_LOC_OPT + RC_INIT_VAR + #endif + for(i = numLevels; i != 0; i--) + { + #ifdef _LZMA_LOC_OPT + CProb *prob = probs + mi; + RC_GET_BIT(prob, mi) + #else + mi = (mi + mi) + RangeDecoderBitDecode(probs + mi, rd); + #endif + } + #ifdef _LZMA_LOC_OPT + RC_FLUSH_VAR + #endif + return mi - (1 << numLevels); +} + +int RangeDecoderReverseBitTreeDecode(CProb *probs, int numLevels, CRangeDecoder *rd) +{ + int mi = 1; + int i; + int symbol = 0; + #ifdef _LZMA_LOC_OPT + RC_INIT_VAR + #endif + for(i = 0; i < numLevels; i++) + { + #ifdef _LZMA_LOC_OPT + CProb *prob = probs + mi; + RC_GET_BIT2(prob, mi, ; , symbol |= (1 << i)) + #else + int bit = RangeDecoderBitDecode(probs + mi, rd); + mi = mi + mi + bit; + symbol |= (bit << i); + #endif + } + #ifdef _LZMA_LOC_OPT + RC_FLUSH_VAR + #endif + return symbol; +} + +Byte LzmaLiteralDecode(CProb *probs, CRangeDecoder *rd) +{ + int symbol = 1; + #ifdef _LZMA_LOC_OPT + RC_INIT_VAR + #endif + do + { + #ifdef _LZMA_LOC_OPT + CProb *prob = probs + symbol; + RC_GET_BIT(prob, symbol) + #else + symbol = (symbol + symbol) | RangeDecoderBitDecode(probs + symbol, rd); + #endif + } + while (symbol < 0x100); + #ifdef _LZMA_LOC_OPT + RC_FLUSH_VAR + #endif + return symbol; +} + +Byte LzmaLiteralDecodeMatch(CProb *probs, CRangeDecoder *rd, Byte matchByte) +{ + int symbol = 1; + #ifdef _LZMA_LOC_OPT + RC_INIT_VAR + #endif + do + { + int bit; + int matchBit = (matchByte >> 7) & 1; + matchByte <<= 1; + #ifdef _LZMA_LOC_OPT + { + CProb *prob = probs + 0x100 + (matchBit << 8) + symbol; + RC_GET_BIT2(prob, symbol, bit = 0, bit = 1) + } + #else + bit = RangeDecoderBitDecode(probs + 0x100 + (matchBit << 8) + symbol, rd); + symbol = (symbol << 1) | bit; + #endif + if (matchBit != bit) + { + while (symbol < 0x100) + { + #ifdef _LZMA_LOC_OPT + CProb *prob = probs + symbol; + RC_GET_BIT(prob, symbol) + #else + symbol = (symbol + symbol) | RangeDecoderBitDecode(probs + symbol, rd); + #endif + } + break; + } + } + while (symbol < 0x100); + #ifdef _LZMA_LOC_OPT + RC_FLUSH_VAR + #endif + return symbol; +} + +#define kNumPosBitsMax 4 +#define kNumPosStatesMax (1 << kNumPosBitsMax) + +#define kLenNumLowBits 3 +#define kLenNumLowSymbols (1 << kLenNumLowBits) +#define kLenNumMidBits 3 +#define kLenNumMidSymbols (1 << kLenNumMidBits) +#define kLenNumHighBits 8 +#define kLenNumHighSymbols (1 << kLenNumHighBits) + +#define LenChoice 0 +#define LenChoice2 (LenChoice + 1) +#define LenLow (LenChoice2 + 1) +#define LenMid (LenLow + (kNumPosStatesMax << kLenNumLowBits)) +#define LenHigh (LenMid + (kNumPosStatesMax << kLenNumMidBits)) +#define kNumLenProbs (LenHigh + kLenNumHighSymbols) + +int LzmaLenDecode(CProb *p, CRangeDecoder *rd, int posState) +{ + if(RangeDecoderBitDecode(p + LenChoice, rd) == 0) + return RangeDecoderBitTreeDecode(p + LenLow + + (posState << kLenNumLowBits), kLenNumLowBits, rd); + if(RangeDecoderBitDecode(p + LenChoice2, rd) == 0) + return kLenNumLowSymbols + RangeDecoderBitTreeDecode(p + LenMid + + (posState << kLenNumMidBits), kLenNumMidBits, rd); + return kLenNumLowSymbols + kLenNumMidSymbols + + RangeDecoderBitTreeDecode(p + LenHigh, kLenNumHighBits, rd); +} + +#define kNumStates 12 +#define kNumLitStates 7 + +#define kStartPosModelIndex 4 +#define kEndPosModelIndex 14 +#define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) + +#define kNumPosSlotBits 6 +#define kNumLenToPosStates 4 + +#define kNumAlignBits 4 +#define kAlignTableSize (1 << kNumAlignBits) + +#define kMatchMinLen 2 + +#define IsMatch 0 +#define IsRep (IsMatch + (kNumStates << kNumPosBitsMax)) +#define IsRepG0 (IsRep + kNumStates) +#define IsRepG1 (IsRepG0 + kNumStates) +#define IsRepG2 (IsRepG1 + kNumStates) +#define IsRep0Long (IsRepG2 + kNumStates) +#define PosSlot (IsRep0Long + (kNumStates << kNumPosBitsMax)) +#define SpecPos (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) +#define Align (SpecPos + kNumFullDistances - kEndPosModelIndex) +#define LenCoder (Align + kAlignTableSize) +#define RepLenCoder (LenCoder + kNumLenProbs) +#define Literal (RepLenCoder + kNumLenProbs) + +#if Literal != LZMA_BASE_SIZE +StopCompilingDueBUG +#endif + +int LzmaDecodeProperties(CLzmaProperties *propsRes, const unsigned char *propsData, int size) +{ + unsigned char prop0; + if (size < LZMA_PROPERTIES_SIZE) + return LZMA_RESULT_DATA_ERROR; + prop0 = propsData[0]; + if (prop0 >= (9 * 5 * 5)) + return LZMA_RESULT_DATA_ERROR; + { + for (propsRes->pb = 0; prop0 >= (9 * 5); propsRes->pb++, prop0 -= (9 * 5)); + for (propsRes->lp = 0; prop0 >= 9; propsRes->lp++, prop0 -= 9); + propsRes->lc = prop0; + /* + unsigned char remainder = (unsigned char)(prop0 / 9); + propsRes->lc = prop0 % 9; + propsRes->pb = remainder / 5; + propsRes->lp = remainder % 5; + */ + } + + #ifdef _LZMA_OUT_READ + { + int i; + propsRes->DictionarySize = 0; + for (i = 0; i < 4; i++) + propsRes->DictionarySize += (UInt32)(propsData[1 + i]) << (i * 8); + if (propsRes->DictionarySize == 0) + propsRes->DictionarySize = 1; + } + #endif + return LZMA_RESULT_OK; +} + +#define kLzmaStreamWasFinishedId (-1) + +int LzmaDecode(CLzmaDecoderState *vs, + #ifdef _LZMA_IN_CB + ILzmaInCallback *InCallback, + #else + const unsigned char *inStream, SizeT inSize, SizeT *inSizeProcessed, + #endif + unsigned char *outStream, SizeT outSize, SizeT *outSizeProcessed) +{ + CProb *p = vs->Probs; + SizeT nowPos = 0; + Byte previousByte = 0; + UInt32 posStateMask = (1 << (vs->Properties.pb)) - 1; + UInt32 literalPosMask = (1 << (vs->Properties.lp)) - 1; + int lc = vs->Properties.lc; + CRangeDecoder rd; + + #ifdef _LZMA_OUT_READ + + int state = vs->State; + UInt32 rep0 = vs->Reps[0], rep1 = vs->Reps[1], rep2 = vs->Reps[2], rep3 = vs->Reps[3]; + int len = vs->RemainLen; + UInt32 globalPos = vs->GlobalPos; + UInt32 distanceLimit = vs->DistanceLimit; + + Byte *dictionary = vs->Dictionary; + UInt32 dictionarySize = vs->Properties.DictionarySize; + UInt32 dictionaryPos = vs->DictionaryPos; + + Byte tempDictionary[4]; + + rd.Range = vs->Range; + rd.Code = vs->Code; + #ifdef _LZMA_IN_CB + rd.InCallback = InCallback; + rd.Buffer = vs->Buffer; + rd.BufferLim = vs->BufferLim; + #else + rd.Buffer = inStream; + rd.BufferLim = inStream + inSize; + #endif + + #ifndef _LZMA_IN_CB + *inSizeProcessed = 0; + #endif + *outSizeProcessed = 0; + if (len == kLzmaStreamWasFinishedId) + return LZMA_RESULT_OK; + + if (dictionarySize == 0) + { + dictionary = tempDictionary; + dictionarySize = 1; + tempDictionary[0] = vs->TempDictionary[0]; + } + + if (len == kLzmaNeedInitId) + { + { + UInt32 numProbs = Literal + ((UInt32)LZMA_LIT_SIZE << (lc + vs->Properties.lp)); + UInt32 i; + for (i = 0; i < numProbs; i++) + p[i] = kBitModelTotal >> 1; + rep0 = rep1 = rep2 = rep3 = 1; + state = 0; + globalPos = 0; + distanceLimit = 0; + dictionaryPos = 0; + dictionary[dictionarySize - 1] = 0; + RangeDecoderInit(&rd + #ifndef _LZMA_IN_CB + , inStream, inSize + #endif + ); + #ifdef _LZMA_IN_CB + if (rd.Result != LZMA_RESULT_OK) + return rd.Result; + #endif + if (rd.ExtraBytes != 0) + return LZMA_RESULT_DATA_ERROR; + } + len = 0; + } + while(len != 0 && nowPos < outSize) + { + UInt32 pos = dictionaryPos - rep0; + if (pos >= dictionarySize) + pos += dictionarySize; + outStream[nowPos++] = dictionary[dictionaryPos] = dictionary[pos]; + if (++dictionaryPos == dictionarySize) + dictionaryPos = 0; + len--; + } + if (dictionaryPos == 0) + previousByte = dictionary[dictionarySize - 1]; + else + previousByte = dictionary[dictionaryPos - 1]; + + #ifdef _LZMA_IN_CB + rd.Result = LZMA_RESULT_OK; + #endif + rd.ExtraBytes = 0; + + #else /* if !_LZMA_OUT_READ */ + + int state = 0; + UInt32 rep0 = 1, rep1 = 1, rep2 = 1, rep3 = 1; + int len = 0; + + #ifndef _LZMA_IN_CB + *inSizeProcessed = 0; + #endif + *outSizeProcessed = 0; + + { + UInt32 i; + UInt32 numProbs = Literal + ((UInt32)LZMA_LIT_SIZE << (lc + vs->Properties.lp)); + for (i = 0; i < numProbs; i++) + p[i] = kBitModelTotal >> 1; + } + + #ifdef _LZMA_IN_CB + rd.InCallback = InCallback; + #endif + RangeDecoderInit(&rd + #ifndef _LZMA_IN_CB + , inStream, inSize + #endif + ); + + #ifdef _LZMA_IN_CB + if (rd.Result != LZMA_RESULT_OK) + return rd.Result; + #endif + if (rd.ExtraBytes != 0) + return LZMA_RESULT_DATA_ERROR; + + #endif /* _LZMA_OUT_READ */ + + + while(nowPos < outSize) + { + int posState = (int)( + (nowPos + #ifdef _LZMA_OUT_READ + + globalPos + #endif + ) + & posStateMask); + #ifdef _LZMA_IN_CB + if (rd.Result != LZMA_RESULT_OK) + return rd.Result; + #endif + if (rd.ExtraBytes != 0) + return LZMA_RESULT_DATA_ERROR; + if (RangeDecoderBitDecode(p + IsMatch + (state << kNumPosBitsMax) + posState, &rd) == 0) + { + CProb *probs = p + Literal + (LZMA_LIT_SIZE * + ((( + (nowPos + #ifdef _LZMA_OUT_READ + + globalPos + #endif + ) + & literalPosMask) << lc) + (previousByte >> (8 - lc)))); + + if (state >= kNumLitStates) + { + Byte matchByte; + #ifdef _LZMA_OUT_READ + UInt32 pos = dictionaryPos - rep0; + if (pos >= dictionarySize) + pos += dictionarySize; + matchByte = dictionary[pos]; + #else + matchByte = outStream[nowPos - rep0]; + #endif + previousByte = LzmaLiteralDecodeMatch(probs, &rd, matchByte); + } + else + previousByte = LzmaLiteralDecode(probs, &rd); + outStream[nowPos++] = previousByte; + #ifdef _LZMA_OUT_READ + if (distanceLimit < dictionarySize) + distanceLimit++; + + dictionary[dictionaryPos] = previousByte; + if (++dictionaryPos == dictionarySize) + dictionaryPos = 0; + #endif + if (state < 4) state = 0; + else if (state < 10) state -= 3; + else state -= 6; + } + else + { + if (RangeDecoderBitDecode(p + IsRep + state, &rd) == 1) + { + if (RangeDecoderBitDecode(p + IsRepG0 + state, &rd) == 0) + { + if (RangeDecoderBitDecode(p + IsRep0Long + (state << kNumPosBitsMax) + posState, &rd) == 0) + { + #ifdef _LZMA_OUT_READ + UInt32 pos; + #endif + + #ifdef _LZMA_OUT_READ + if (distanceLimit == 0) + #else + if (nowPos == 0) + #endif + return LZMA_RESULT_DATA_ERROR; + + state = state < 7 ? 9 : 11; + #ifdef _LZMA_OUT_READ + pos = dictionaryPos - rep0; + if (pos >= dictionarySize) + pos += dictionarySize; + previousByte = dictionary[pos]; + dictionary[dictionaryPos] = previousByte; + if (++dictionaryPos == dictionarySize) + dictionaryPos = 0; + #else + previousByte = outStream[nowPos - rep0]; + #endif + outStream[nowPos++] = previousByte; + + #ifdef _LZMA_OUT_READ + if (distanceLimit < dictionarySize) + distanceLimit++; + #endif + continue; + } + } + else + { + UInt32 distance; + if(RangeDecoderBitDecode(p + IsRepG1 + state, &rd) == 0) + distance = rep1; + else + { + if(RangeDecoderBitDecode(p + IsRepG2 + state, &rd) == 0) + distance = rep2; + else + { + distance = rep3; + rep3 = rep2; + } + rep2 = rep1; + } + rep1 = rep0; + rep0 = distance; + } + len = LzmaLenDecode(p + RepLenCoder, &rd, posState); + state = state < 7 ? 8 : 11; + } + else + { + int posSlot; + rep3 = rep2; + rep2 = rep1; + rep1 = rep0; + state = state < 7 ? 7 : 10; + len = LzmaLenDecode(p + LenCoder, &rd, posState); + posSlot = RangeDecoderBitTreeDecode(p + PosSlot + + ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << + kNumPosSlotBits), kNumPosSlotBits, &rd); + if (posSlot >= kStartPosModelIndex) + { + int numDirectBits = ((posSlot >> 1) - 1); + rep0 = ((2 | ((UInt32)posSlot & 1)) << numDirectBits); + if (posSlot < kEndPosModelIndex) + { + rep0 += RangeDecoderReverseBitTreeDecode( + p + SpecPos + rep0 - posSlot - 1, numDirectBits, &rd); + } + else + { + rep0 += RangeDecoderDecodeDirectBits(&rd, + numDirectBits - kNumAlignBits) << kNumAlignBits; + rep0 += RangeDecoderReverseBitTreeDecode(p + Align, kNumAlignBits, &rd); + } + } + else + rep0 = posSlot; + if (++rep0 == (UInt32)(0)) + { + /* it's for stream version */ + len = kLzmaStreamWasFinishedId; + break; + } + } + + len += kMatchMinLen; + #ifdef _LZMA_OUT_READ + if (rep0 > distanceLimit) + #else + if (rep0 > nowPos) + #endif + return LZMA_RESULT_DATA_ERROR; + + #ifdef _LZMA_OUT_READ + if (dictionarySize - distanceLimit > (UInt32)len) + distanceLimit += len; + else + distanceLimit = dictionarySize; + #endif + + do + { + #ifdef _LZMA_OUT_READ + UInt32 pos = dictionaryPos - rep0; + if (pos >= dictionarySize) + pos += dictionarySize; + previousByte = dictionary[pos]; + dictionary[dictionaryPos] = previousByte; + if (++dictionaryPos == dictionarySize) + dictionaryPos = 0; + #else + previousByte = outStream[nowPos - rep0]; + #endif + len--; + outStream[nowPos++] = previousByte; + } + while(len != 0 && nowPos < outSize); + } + } + + + #ifdef _LZMA_OUT_READ + vs->Range = rd.Range; + vs->Code = rd.Code; + vs->DictionaryPos = dictionaryPos; + vs->GlobalPos = globalPos + (UInt32)nowPos; + vs->DistanceLimit = distanceLimit; + vs->Reps[0] = rep0; + vs->Reps[1] = rep1; + vs->Reps[2] = rep2; + vs->Reps[3] = rep3; + vs->State = state; + vs->RemainLen = len; + vs->TempDictionary[0] = tempDictionary[0]; + #endif + + #ifdef _LZMA_IN_CB + vs->Buffer = rd.Buffer; + vs->BufferLim = rd.BufferLim; + #else + *inSizeProcessed = (SizeT)(rd.Buffer - inStream); + #endif + *outSizeProcessed = nowPos; + return LZMA_RESULT_OK; +} diff --git a/lzma443/C/7zip/Compress/LZMA_C/LzmaStateDecode.c b/lzma443/C/7zip/Compress/LZMA_C/LzmaStateDecode.c new file mode 100755 index 0000000..e50f88b --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA_C/LzmaStateDecode.c @@ -0,0 +1,521 @@ +/* + LzmaStateDecode.c + LZMA Decoder (State version) + + LZMA SDK 4.40 Copyright (c) 1999-2006 Igor Pavlov (2006-05-01) + http://www.7-zip.org/ + + LZMA SDK is licensed under two licenses: + 1) GNU Lesser General Public License (GNU LGPL) + 2) Common Public License (CPL) + It means that you can select one of these two licenses and + follow rules of that license. + + SPECIAL EXCEPTION: + Igor Pavlov, as the author of this Code, expressly permits you to + statically or dynamically link your Code (or bind by name) to the + interfaces of this file without subjecting your linked Code to the + terms of the CPL or GNU LGPL. Any modifications or additions + to this file, however, are subject to the LGPL or CPL terms. +*/ + +#include "LzmaStateDecode.h" + +#define kNumTopBits 24 +#define kTopValue ((UInt32)1 << kNumTopBits) + +#define kNumBitModelTotalBits 11 +#define kBitModelTotal (1 << kNumBitModelTotalBits) +#define kNumMoveBits 5 + +#define RC_READ_BYTE (*Buffer++) + +#define RC_INIT Code = 0; Range = 0xFFFFFFFF; \ + { int i; for(i = 0; i < 5; i++) { Code = (Code << 8) | RC_READ_BYTE; }} + +#define RC_NORMALIZE if (Range < kTopValue) { Range <<= 8; Code = (Code << 8) | RC_READ_BYTE; } + +#define IfBit0(p) RC_NORMALIZE; bound = (Range >> kNumBitModelTotalBits) * *(p); if (Code < bound) +#define UpdateBit0(p) Range = bound; *(p) += (kBitModelTotal - *(p)) >> kNumMoveBits; +#define UpdateBit1(p) Range -= bound; Code -= bound; *(p) -= (*(p)) >> kNumMoveBits; + +#define RC_GET_BIT2(p, mi, A0, A1) IfBit0(p) \ + { UpdateBit0(p); mi <<= 1; A0; } else \ + { UpdateBit1(p); mi = (mi + mi) + 1; A1; } + +#define RC_GET_BIT(p, mi) RC_GET_BIT2(p, mi, ; , ;) + +#define RangeDecoderBitTreeDecode(probs, numLevels, res) \ + { int i = numLevels; res = 1; \ + do { CProb *p = probs + res; RC_GET_BIT(p, res) } while(--i != 0); \ + res -= (1 << numLevels); } + + +#define kNumPosBitsMax 4 +#define kNumPosStatesMax (1 << kNumPosBitsMax) + +#define kLenNumLowBits 3 +#define kLenNumLowSymbols (1 << kLenNumLowBits) +#define kLenNumMidBits 3 +#define kLenNumMidSymbols (1 << kLenNumMidBits) +#define kLenNumHighBits 8 +#define kLenNumHighSymbols (1 << kLenNumHighBits) + +#define LenChoice 0 +#define LenChoice2 (LenChoice + 1) +#define LenLow (LenChoice2 + 1) +#define LenMid (LenLow + (kNumPosStatesMax << kLenNumLowBits)) +#define LenHigh (LenMid + (kNumPosStatesMax << kLenNumMidBits)) +#define kNumLenProbs (LenHigh + kLenNumHighSymbols) + + +#define kNumStates 12 +#define kNumLitStates 7 + +#define kStartPosModelIndex 4 +#define kEndPosModelIndex 14 +#define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) + +#define kNumPosSlotBits 6 +#define kNumLenToPosStates 4 + +#define kNumAlignBits 4 +#define kAlignTableSize (1 << kNumAlignBits) + +#define kMatchMinLen 2 + +#define IsMatch 0 +#define IsRep (IsMatch + (kNumStates << kNumPosBitsMax)) +#define IsRepG0 (IsRep + kNumStates) +#define IsRepG1 (IsRepG0 + kNumStates) +#define IsRepG2 (IsRepG1 + kNumStates) +#define IsRep0Long (IsRepG2 + kNumStates) +#define PosSlot (IsRep0Long + (kNumStates << kNumPosBitsMax)) +#define SpecPos (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) +#define Align (SpecPos + kNumFullDistances - kEndPosModelIndex) +#define LenCoder (Align + kAlignTableSize) +#define RepLenCoder (LenCoder + kNumLenProbs) +#define Literal (RepLenCoder + kNumLenProbs) + +#if Literal != LZMA_BASE_SIZE +StopCompilingDueBUG +#endif + +/* kRequiredInBufferSize = number of required input bytes for worst case: + longest match with longest distance. + kLzmaInBufferSize must be larger than kRequiredInBufferSize + 23 bits = 2 (match select) + 10 (len) + 6 (distance) + 4(align) + 1 (RC_NORMALIZE) +*/ + +#define kRequiredInBufferSize ((23 * (kNumBitModelTotalBits - kNumMoveBits + 1) + 26 + 9) / 8) + +#define kLzmaStreamWasFinishedId (-1) + +int LzmaDecodeProperties(CLzmaProperties *propsRes, const unsigned char *propsData, int size) +{ + unsigned char prop0; + if (size < LZMA_PROPERTIES_SIZE) + return LZMA_RESULT_DATA_ERROR; + prop0 = propsData[0]; + if (prop0 >= (9 * 5 * 5)) + return LZMA_RESULT_DATA_ERROR; + { + for (propsRes->pb = 0; prop0 >= (9 * 5); propsRes->pb++, prop0 -= (9 * 5)); + for (propsRes->lp = 0; prop0 >= 9; propsRes->lp++, prop0 -= 9); + propsRes->lc = prop0; + /* + unsigned char remainder = (unsigned char)(prop0 / 9); + propsRes->lc = prop0 % 9; + propsRes->pb = remainder / 5; + propsRes->lp = remainder % 5; + */ + } + + { + int i; + propsRes->DictionarySize = 0; + for (i = 0; i < 4; i++) + propsRes->DictionarySize += (UInt32)(propsData[1 + i]) << (i * 8); + if (propsRes->DictionarySize == 0) + propsRes->DictionarySize = 1; + return LZMA_RESULT_OK; + } +} + +int LzmaDecode( + CLzmaDecoderState *vs, + const unsigned char *inStream, SizeT inSize, SizeT *inSizeProcessed, + unsigned char *outStream, SizeT outSize, SizeT *outSizeProcessed, + int finishDecoding) +{ + UInt32 Range = vs->Range; + UInt32 Code = vs->Code; + + unsigned char *Buffer = vs->Buffer; + int BufferSize = vs->BufferSize; /* don't change it to unsigned int */ + CProb *p = vs->Probs; + + int state = vs->State; + unsigned char previousByte; + UInt32 rep0 = vs->Reps[0], rep1 = vs->Reps[1], rep2 = vs->Reps[2], rep3 = vs->Reps[3]; + SizeT nowPos = 0; + UInt32 posStateMask = (1 << (vs->Properties.pb)) - 1; + UInt32 literalPosMask = (1 << (vs->Properties.lp)) - 1; + int lc = vs->Properties.lc; + int len = vs->RemainLen; + UInt32 globalPos = vs->GlobalPos; + UInt32 distanceLimit = vs->DistanceLimit; + + unsigned char *dictionary = vs->Dictionary; + UInt32 dictionarySize = vs->Properties.DictionarySize; + UInt32 dictionaryPos = vs->DictionaryPos; + + unsigned char tempDictionary[4]; + + (*inSizeProcessed) = 0; + (*outSizeProcessed) = 0; + if (len == kLzmaStreamWasFinishedId) + return LZMA_RESULT_OK; + + if (dictionarySize == 0) + { + dictionary = tempDictionary; + dictionarySize = 1; + tempDictionary[0] = vs->TempDictionary[0]; + } + + if (len == kLzmaNeedInitId) + { + while (inSize > 0 && BufferSize < kLzmaInBufferSize) + { + Buffer[BufferSize++] = *inStream++; + (*inSizeProcessed)++; + inSize--; + } + if (BufferSize < 5) + { + vs->BufferSize = BufferSize; + return finishDecoding ? LZMA_RESULT_DATA_ERROR : LZMA_RESULT_OK; + } + { + UInt32 numProbs = Literal + ((UInt32)LZMA_LIT_SIZE << (lc + vs->Properties.lp)); + UInt32 i; + for (i = 0; i < numProbs; i++) + p[i] = kBitModelTotal >> 1; + rep0 = rep1 = rep2 = rep3 = 1; + state = 0; + globalPos = 0; + distanceLimit = 0; + dictionaryPos = 0; + dictionary[dictionarySize - 1] = 0; + RC_INIT; + } + len = 0; + } + while(len != 0 && nowPos < outSize) + { + UInt32 pos = dictionaryPos - rep0; + if (pos >= dictionarySize) + pos += dictionarySize; + outStream[nowPos++] = dictionary[dictionaryPos] = dictionary[pos]; + if (++dictionaryPos == dictionarySize) + dictionaryPos = 0; + len--; + } + if (dictionaryPos == 0) + previousByte = dictionary[dictionarySize - 1]; + else + previousByte = dictionary[dictionaryPos - 1]; + + while(1) + { + int bufferPos = (int)(Buffer - vs->Buffer); + if (BufferSize - bufferPos < kRequiredInBufferSize) + { + int i; + BufferSize -= bufferPos; + if (BufferSize < 0) + return LZMA_RESULT_DATA_ERROR; + for (i = 0; i < BufferSize; i++) + vs->Buffer[i] = Buffer[i]; + Buffer = vs->Buffer; + while (inSize > 0 && BufferSize < kLzmaInBufferSize) + { + Buffer[BufferSize++] = *inStream++; + (*inSizeProcessed)++; + inSize--; + } + if (BufferSize < kRequiredInBufferSize && !finishDecoding) + break; + } + if (nowPos >= outSize) + break; + { + CProb *prob; + UInt32 bound; + int posState = (int)((nowPos + globalPos) & posStateMask); + + prob = p + IsMatch + (state << kNumPosBitsMax) + posState; + IfBit0(prob) + { + int symbol = 1; + UpdateBit0(prob) + prob = p + Literal + (LZMA_LIT_SIZE * + ((((nowPos + globalPos)& literalPosMask) << lc) + (previousByte >> (8 - lc)))); + + if (state >= kNumLitStates) + { + int matchByte; + UInt32 pos = dictionaryPos - rep0; + if (pos >= dictionarySize) + pos += dictionarySize; + matchByte = dictionary[pos]; + do + { + int bit; + CProb *probLit; + matchByte <<= 1; + bit = (matchByte & 0x100); + probLit = prob + 0x100 + bit + symbol; + RC_GET_BIT2(probLit, symbol, if (bit != 0) break, if (bit == 0) break) + } + while (symbol < 0x100); + } + while (symbol < 0x100) + { + CProb *probLit = prob + symbol; + RC_GET_BIT(probLit, symbol) + } + previousByte = (unsigned char)symbol; + + outStream[nowPos++] = previousByte; + if (distanceLimit < dictionarySize) + distanceLimit++; + + dictionary[dictionaryPos] = previousByte; + if (++dictionaryPos == dictionarySize) + dictionaryPos = 0; + if (state < 4) state = 0; + else if (state < 10) state -= 3; + else state -= 6; + } + else + { + UpdateBit1(prob); + prob = p + IsRep + state; + IfBit0(prob) + { + UpdateBit0(prob); + rep3 = rep2; + rep2 = rep1; + rep1 = rep0; + state = state < kNumLitStates ? 0 : 3; + prob = p + LenCoder; + } + else + { + UpdateBit1(prob); + prob = p + IsRepG0 + state; + IfBit0(prob) + { + UpdateBit0(prob); + prob = p + IsRep0Long + (state << kNumPosBitsMax) + posState; + IfBit0(prob) + { + UInt32 pos; + UpdateBit0(prob); + if (distanceLimit == 0) + return LZMA_RESULT_DATA_ERROR; + if (distanceLimit < dictionarySize) + distanceLimit++; + state = state < kNumLitStates ? 9 : 11; + pos = dictionaryPos - rep0; + if (pos >= dictionarySize) + pos += dictionarySize; + previousByte = dictionary[pos]; + dictionary[dictionaryPos] = previousByte; + if (++dictionaryPos == dictionarySize) + dictionaryPos = 0; + outStream[nowPos++] = previousByte; + continue; + } + else + { + UpdateBit1(prob); + } + } + else + { + UInt32 distance; + UpdateBit1(prob); + prob = p + IsRepG1 + state; + IfBit0(prob) + { + UpdateBit0(prob); + distance = rep1; + } + else + { + UpdateBit1(prob); + prob = p + IsRepG2 + state; + IfBit0(prob) + { + UpdateBit0(prob); + distance = rep2; + } + else + { + UpdateBit1(prob); + distance = rep3; + rep3 = rep2; + } + rep2 = rep1; + } + rep1 = rep0; + rep0 = distance; + } + state = state < kNumLitStates ? 8 : 11; + prob = p + RepLenCoder; + } + { + int numBits, offset; + CProb *probLen = prob + LenChoice; + IfBit0(probLen) + { + UpdateBit0(probLen); + probLen = prob + LenLow + (posState << kLenNumLowBits); + offset = 0; + numBits = kLenNumLowBits; + } + else + { + UpdateBit1(probLen); + probLen = prob + LenChoice2; + IfBit0(probLen) + { + UpdateBit0(probLen); + probLen = prob + LenMid + (posState << kLenNumMidBits); + offset = kLenNumLowSymbols; + numBits = kLenNumMidBits; + } + else + { + UpdateBit1(probLen); + probLen = prob + LenHigh; + offset = kLenNumLowSymbols + kLenNumMidSymbols; + numBits = kLenNumHighBits; + } + } + RangeDecoderBitTreeDecode(probLen, numBits, len); + len += offset; + } + + if (state < 4) + { + int posSlot; + state += kNumLitStates; + prob = p + PosSlot + + ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << + kNumPosSlotBits); + RangeDecoderBitTreeDecode(prob, kNumPosSlotBits, posSlot); + if (posSlot >= kStartPosModelIndex) + { + int numDirectBits = ((posSlot >> 1) - 1); + rep0 = (2 | ((UInt32)posSlot & 1)); + if (posSlot < kEndPosModelIndex) + { + rep0 <<= numDirectBits; + prob = p + SpecPos + rep0 - posSlot - 1; + } + else + { + numDirectBits -= kNumAlignBits; + do + { + RC_NORMALIZE + Range >>= 1; + rep0 <<= 1; + if (Code >= Range) + { + Code -= Range; + rep0 |= 1; + } + } + while (--numDirectBits != 0); + prob = p + Align; + rep0 <<= kNumAlignBits; + numDirectBits = kNumAlignBits; + } + { + int i = 1; + int mi = 1; + do + { + CProb *prob3 = prob + mi; + RC_GET_BIT2(prob3, mi, ; , rep0 |= i); + i <<= 1; + } + while(--numDirectBits != 0); + } + } + else + rep0 = posSlot; + if (++rep0 == (UInt32)(0)) + { + /* it's for stream version */ + len = kLzmaStreamWasFinishedId; + break; + } + } + + len += kMatchMinLen; + if (rep0 > distanceLimit) + return LZMA_RESULT_DATA_ERROR; + if (dictionarySize - distanceLimit > (UInt32)len) + distanceLimit += len; + else + distanceLimit = dictionarySize; + + do + { + UInt32 pos = dictionaryPos - rep0; + if (pos >= dictionarySize) + pos += dictionarySize; + previousByte = dictionary[pos]; + dictionary[dictionaryPos] = previousByte; + if (++dictionaryPos == dictionarySize) + dictionaryPos = 0; + len--; + outStream[nowPos++] = previousByte; + } + while(len != 0 && nowPos < outSize); + } + } + } + RC_NORMALIZE; + + BufferSize -= (int)(Buffer - vs->Buffer); + if (BufferSize < 0) + return LZMA_RESULT_DATA_ERROR; + { + int i; + for (i = 0; i < BufferSize; i++) + vs->Buffer[i] = Buffer[i]; + } + vs->BufferSize = BufferSize; + vs->Range = Range; + vs->Code = Code; + vs->DictionaryPos = dictionaryPos; + vs->GlobalPos = (UInt32)(globalPos + nowPos); + vs->DistanceLimit = distanceLimit; + vs->Reps[0] = rep0; + vs->Reps[1] = rep1; + vs->Reps[2] = rep2; + vs->Reps[3] = rep3; + vs->State = state; + vs->RemainLen = len; + vs->TempDictionary[0] = tempDictionary[0]; + + (*outSizeProcessed) = nowPos; + return LZMA_RESULT_OK; +} diff --git a/lzma443/C/7zip/Compress/LZMA_C/LzmaStateDecode.h b/lzma443/C/7zip/Compress/LZMA_C/LzmaStateDecode.h new file mode 100755 index 0000000..26490d6 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA_C/LzmaStateDecode.h @@ -0,0 +1,96 @@ +/* + LzmaStateDecode.h + LZMA Decoder interface (State version) + + LZMA SDK 4.40 Copyright (c) 1999-2006 Igor Pavlov (2006-05-01) + http://www.7-zip.org/ + + LZMA SDK is licensed under two licenses: + 1) GNU Lesser General Public License (GNU LGPL) + 2) Common Public License (CPL) + It means that you can select one of these two licenses and + follow rules of that license. + + SPECIAL EXCEPTION: + Igor Pavlov, as the author of this code, expressly permits you to + statically or dynamically link your code (or bind by name) to the + interfaces of this file without subjecting your linked code to the + terms of the CPL or GNU LGPL. Any modifications or additions + to this file, however, are subject to the LGPL or CPL terms. +*/ + +#ifndef __LZMASTATEDECODE_H +#define __LZMASTATEDECODE_H + +#include "LzmaTypes.h" + +/* #define _LZMA_PROB32 */ +/* It can increase speed on some 32-bit CPUs, + but memory usage will be doubled in that case */ + +#ifdef _LZMA_PROB32 +#define CProb UInt32 +#else +#define CProb UInt16 +#endif + +#define LZMA_RESULT_OK 0 +#define LZMA_RESULT_DATA_ERROR 1 + +#define LZMA_BASE_SIZE 1846 +#define LZMA_LIT_SIZE 768 + +#define LZMA_PROPERTIES_SIZE 5 + +typedef struct _CLzmaProperties +{ + int lc; + int lp; + int pb; + UInt32 DictionarySize; +}CLzmaProperties; + +int LzmaDecodeProperties(CLzmaProperties *propsRes, const unsigned char *propsData, int size); + +#define LzmaGetNumProbs(lzmaProps) (LZMA_BASE_SIZE + (LZMA_LIT_SIZE << ((lzmaProps)->lc + (lzmaProps)->lp))) + +#define kLzmaInBufferSize 64 /* don't change it. it must be larger than kRequiredInBufferSize */ + +#define kLzmaNeedInitId (-2) + +typedef struct _CLzmaDecoderState +{ + CLzmaProperties Properties; + CProb *Probs; + unsigned char *Dictionary; + + unsigned char Buffer[kLzmaInBufferSize]; + int BufferSize; + + UInt32 Range; + UInt32 Code; + UInt32 DictionaryPos; + UInt32 GlobalPos; + UInt32 DistanceLimit; + UInt32 Reps[4]; + int State; + int RemainLen; /* -2: decoder needs internal initialization + -1: stream was finished, + 0: ok + > 0: need to write RemainLen bytes as match Reps[0], + */ + unsigned char TempDictionary[4]; /* it's required when DictionarySize = 0 */ +} CLzmaDecoderState; + +#define LzmaDecoderInit(vs) { (vs)->RemainLen = kLzmaNeedInitId; (vs)->BufferSize = 0; } + +/* LzmaDecode: decoding from input stream to output stream. + If finishDecoding != 0, then there are no more bytes in input stream + after inStream[inSize - 1]. */ + +int LzmaDecode(CLzmaDecoderState *vs, + const unsigned char *inStream, SizeT inSize, SizeT *inSizeProcessed, + unsigned char *outStream, SizeT outSize, SizeT *outSizeProcessed, + int finishDecoding); + +#endif diff --git a/lzma443/C/7zip/Compress/LZMA_C/LzmaStateTest.c b/lzma443/C/7zip/Compress/LZMA_C/LzmaStateTest.c new file mode 100755 index 0000000..5df4e43 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA_C/LzmaStateTest.c @@ -0,0 +1,195 @@ +/* +LzmaStateTest.c +Test application for LZMA Decoder (State version) + +This file written and distributed to public domain by Igor Pavlov. +This file is part of LZMA SDK 4.26 (2005-08-02) +*/ + +#include +#include +#include + +#include "LzmaStateDecode.h" + +const char *kCantReadMessage = "Can not read input file"; +const char *kCantWriteMessage = "Can not write output file"; +const char *kCantAllocateMessage = "Can not allocate memory"; + +#define kInBufferSize (1 << 15) +#define kOutBufferSize (1 << 15) + +unsigned char g_InBuffer[kInBufferSize]; +unsigned char g_OutBuffer[kOutBufferSize]; + +size_t MyReadFile(FILE *file, void *data, size_t size) + { return fread(data, 1, size, file); } + +int MyReadFileAndCheck(FILE *file, void *data, size_t size) + { return (MyReadFile(file, data, size) == size); } + +int PrintError(char *buffer, const char *message) +{ + sprintf(buffer + strlen(buffer), "\nError: "); + sprintf(buffer + strlen(buffer), message); + return 1; +} + +int main3(FILE *inFile, FILE *outFile, char *rs) +{ + /* We use two 32-bit integers to construct 64-bit integer for file size. + You can remove outSizeHigh, if you don't need >= 4GB supporting, + or you can use UInt64 outSize, if your compiler supports 64-bit integers*/ + UInt32 outSize = 0; + UInt32 outSizeHigh = 0; + + int waitEOS = 1; + /* waitEOS = 1, if there is no uncompressed size in headers, + so decoder will wait EOS (End of Stream Marker) in compressed stream */ + + int i; + int res = 0; + CLzmaDecoderState state; /* it's about 140 bytes structure, if int is 32-bit */ + unsigned char properties[LZMA_PROPERTIES_SIZE]; + SizeT inAvail = 0; + unsigned char *inBuffer = 0; + + if (sizeof(UInt32) < 4) + return PrintError(rs, "LZMA decoder needs correct UInt32"); + + /* Read LZMA properties for compressed stream */ + + if (!MyReadFileAndCheck(inFile, properties, sizeof(properties))) + return PrintError(rs, kCantReadMessage); + + /* Read uncompressed size */ + + for (i = 0; i < 8; i++) + { + unsigned char b; + if (!MyReadFileAndCheck(inFile, &b, 1)) + return PrintError(rs, kCantReadMessage); + if (b != 0xFF) + waitEOS = 0; + if (i < 4) + outSize += (UInt32)(b) << (i * 8); + else + outSizeHigh += (UInt32)(b) << ((i - 4) * 8); + } + + /* Decode LZMA properties and allocate memory */ + + if (LzmaDecodeProperties(&state.Properties, properties, LZMA_PROPERTIES_SIZE) != LZMA_RESULT_OK) + return PrintError(rs, "Incorrect stream properties"); + state.Probs = (CProb *)malloc(LzmaGetNumProbs(&state.Properties) * sizeof(CProb)); + if (state.Probs == 0) + return PrintError(rs, kCantAllocateMessage); + + if (state.Properties.DictionarySize == 0) + state.Dictionary = 0; + else + { + state.Dictionary = (unsigned char *)malloc(state.Properties.DictionarySize); + if (state.Dictionary == 0) + { + free(state.Probs); + return PrintError(rs, kCantAllocateMessage); + } + } + + /* Decompress */ + + LzmaDecoderInit(&state); + + do + { + SizeT inProcessed, outProcessed; + int finishDecoding; + UInt32 outAvail = kOutBufferSize; + if (!waitEOS && outSizeHigh == 0 && outAvail > outSize) + outAvail = outSize; + if (inAvail == 0) + { + inAvail = (SizeT)MyReadFile(inFile, g_InBuffer, kInBufferSize); + inBuffer = g_InBuffer; + } + finishDecoding = (inAvail == 0); + res = LzmaDecode(&state, + inBuffer, inAvail, &inProcessed, + g_OutBuffer, outAvail, &outProcessed, + finishDecoding); + if (res != 0) + { + sprintf(rs + strlen(rs), "\nDecoding error = %d\n", res); + res = 1; + break; + } + inAvail -= inProcessed; + inBuffer += inProcessed; + + if (outFile != 0) + if (fwrite(g_OutBuffer, 1, outProcessed, outFile) != outProcessed) + { + PrintError(rs, kCantWriteMessage); + res = 1; + break; + } + + if (outSize < outProcessed) + outSizeHigh--; + outSize -= (UInt32)outProcessed; + outSize &= 0xFFFFFFFF; + + if (outProcessed == 0 && finishDecoding) + { + if (!waitEOS && (outSize != 0 || outSizeHigh != 0)) + res = 1; + break; + } + } + while ((outSize != 0 && outSizeHigh == 0) || outSizeHigh != 0 || waitEOS); + + free(state.Dictionary); + free(state.Probs); + return res; +} + +int main2(int numArgs, const char *args[], char *rs) +{ + FILE *inFile = 0; + FILE *outFile = 0; + int res; + + sprintf(rs + strlen(rs), "\nLZMA Decoder 4.26 Copyright (c) 1999-2005 Igor Pavlov 2005-08-02\n"); + if (numArgs < 2 || numArgs > 3) + { + sprintf(rs + strlen(rs), "\nUsage: lzmadec file.lzma [outFile]\n"); + return 1; + } + + inFile = fopen(args[1], "rb"); + if (inFile == 0) + return PrintError(rs, "Can not open input file"); + + if (numArgs > 2) + { + outFile = fopen(args[2], "wb+"); + if (outFile == 0) + return PrintError(rs, "Can not open output file"); + } + + res = main3(inFile, outFile, rs); + + if (outFile != 0) + fclose(outFile); + fclose(inFile); + return res; +} + +int main(int numArgs, const char *args[]) +{ + char rs[800] = { 0 }; + int res = main2(numArgs, args, rs); + printf(rs); + return res; +} diff --git a/lzma443/C/7zip/Compress/LZMA_C/LzmaTest.c b/lzma443/C/7zip/Compress/LZMA_C/LzmaTest.c new file mode 100755 index 0000000..f95a753 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA_C/LzmaTest.c @@ -0,0 +1,342 @@ +/* +LzmaTest.c +Test application for LZMA Decoder + +This file written and distributed to public domain by Igor Pavlov. +This file is part of LZMA SDK 4.26 (2005-08-05) +*/ + +#include +#include +#include + +#include "LzmaDecode.h" + +const char *kCantReadMessage = "Can not read input file"; +const char *kCantWriteMessage = "Can not write output file"; +const char *kCantAllocateMessage = "Can not allocate memory"; + +size_t MyReadFile(FILE *file, void *data, size_t size) +{ + if (size == 0) + return 0; + return fread(data, 1, size, file); +} + +int MyReadFileAndCheck(FILE *file, void *data, size_t size) + { return (MyReadFile(file, data, size) == size);} + +size_t MyWriteFile(FILE *file, const void *data, size_t size) +{ + if (size == 0) + return 0; + return fwrite(data, 1, size, file); +} + +int MyWriteFileAndCheck(FILE *file, const void *data, size_t size) + { return (MyWriteFile(file, data, size) == size); } + +#ifdef _LZMA_IN_CB +#define kInBufferSize (1 << 15) +typedef struct _CBuffer +{ + ILzmaInCallback InCallback; + FILE *File; + unsigned char Buffer[kInBufferSize]; +} CBuffer; + +int LzmaReadCompressed(void *object, const unsigned char **buffer, SizeT *size) +{ + CBuffer *b = (CBuffer *)object; + *buffer = b->Buffer; + *size = (SizeT)MyReadFile(b->File, b->Buffer, kInBufferSize); + return LZMA_RESULT_OK; +} +CBuffer g_InBuffer; + +#endif + +#ifdef _LZMA_OUT_READ +#define kOutBufferSize (1 << 15) +unsigned char g_OutBuffer[kOutBufferSize]; +#endif + +int PrintError(char *buffer, const char *message) +{ + sprintf(buffer + strlen(buffer), "\nError: "); + sprintf(buffer + strlen(buffer), message); + return 1; +} + +int main3(FILE *inFile, FILE *outFile, char *rs) +{ + /* We use two 32-bit integers to construct 64-bit integer for file size. + You can remove outSizeHigh, if you don't need >= 4GB supporting, + or you can use UInt64 outSize, if your compiler supports 64-bit integers*/ + UInt32 outSize = 0; + UInt32 outSizeHigh = 0; + #ifndef _LZMA_OUT_READ + SizeT outSizeFull; + unsigned char *outStream; + #endif + + int waitEOS = 1; + /* waitEOS = 1, if there is no uncompressed size in headers, + so decoder will wait EOS (End of Stream Marker) in compressed stream */ + + #ifndef _LZMA_IN_CB + SizeT compressedSize; + unsigned char *inStream; + #endif + + CLzmaDecoderState state; /* it's about 24-80 bytes structure, if int is 32-bit */ + unsigned char properties[LZMA_PROPERTIES_SIZE]; + + int res; + + #ifdef _LZMA_IN_CB + g_InBuffer.File = inFile; + #endif + + if (sizeof(UInt32) < 4) + return PrintError(rs, "LZMA decoder needs correct UInt32"); + + #ifndef _LZMA_IN_CB + { + long length; + fseek(inFile, 0, SEEK_END); + length = ftell(inFile); + fseek(inFile, 0, SEEK_SET); + if ((long)(SizeT)length != length) + return PrintError(rs, "Too big compressed stream"); + compressedSize = (SizeT)(length - (LZMA_PROPERTIES_SIZE + 8)); + } + #endif + + /* Read LZMA properties for compressed stream */ + + if (!MyReadFileAndCheck(inFile, properties, sizeof(properties))) + return PrintError(rs, kCantReadMessage); + + /* Read uncompressed size */ + + { + int i; + for (i = 0; i < 8; i++) + { + unsigned char b; + if (!MyReadFileAndCheck(inFile, &b, 1)) + return PrintError(rs, kCantReadMessage); + if (b != 0xFF) + waitEOS = 0; + if (i < 4) + outSize += (UInt32)(b) << (i * 8); + else + outSizeHigh += (UInt32)(b) << ((i - 4) * 8); + } + + #ifndef _LZMA_OUT_READ + if (waitEOS) + return PrintError(rs, "Stream with EOS marker is not supported"); + outSizeFull = (SizeT)outSize; + if (sizeof(SizeT) >= 8) + outSizeFull |= (((SizeT)outSizeHigh << 16) << 16); + else if (outSizeHigh != 0 || (UInt32)(SizeT)outSize != outSize) + return PrintError(rs, "Too big uncompressed stream"); + #endif + } + + /* Decode LZMA properties and allocate memory */ + + if (LzmaDecodeProperties(&state.Properties, properties, LZMA_PROPERTIES_SIZE) != LZMA_RESULT_OK) + return PrintError(rs, "Incorrect stream properties"); + state.Probs = (CProb *)malloc(LzmaGetNumProbs(&state.Properties) * sizeof(CProb)); + + #ifdef _LZMA_OUT_READ + if (state.Properties.DictionarySize == 0) + state.Dictionary = 0; + else + state.Dictionary = (unsigned char *)malloc(state.Properties.DictionarySize); + #else + if (outSizeFull == 0) + outStream = 0; + else + outStream = (unsigned char *)malloc(outSizeFull); + #endif + + #ifndef _LZMA_IN_CB + if (compressedSize == 0) + inStream = 0; + else + inStream = (unsigned char *)malloc(compressedSize); + #endif + + if (state.Probs == 0 + #ifdef _LZMA_OUT_READ + || (state.Dictionary == 0 && state.Properties.DictionarySize != 0) + #else + || (outStream == 0 && outSizeFull != 0) + #endif + #ifndef _LZMA_IN_CB + || (inStream == 0 && compressedSize != 0) + #endif + ) + { + free(state.Probs); + #ifdef _LZMA_OUT_READ + free(state.Dictionary); + #else + free(outStream); + #endif + #ifndef _LZMA_IN_CB + free(inStream); + #endif + return PrintError(rs, kCantAllocateMessage); + } + + /* Decompress */ + + #ifdef _LZMA_IN_CB + g_InBuffer.InCallback.Read = LzmaReadCompressed; + #else + if (!MyReadFileAndCheck(inFile, inStream, compressedSize)) + return PrintError(rs, kCantReadMessage); + #endif + + #ifdef _LZMA_OUT_READ + { + #ifndef _LZMA_IN_CB + SizeT inAvail = compressedSize; + const unsigned char *inBuffer = inStream; + #endif + LzmaDecoderInit(&state); + do + { + #ifndef _LZMA_IN_CB + SizeT inProcessed; + #endif + SizeT outProcessed; + SizeT outAvail = kOutBufferSize; + if (!waitEOS && outSizeHigh == 0 && outAvail > outSize) + outAvail = (SizeT)outSize; + res = LzmaDecode(&state, + #ifdef _LZMA_IN_CB + &g_InBuffer.InCallback, + #else + inBuffer, inAvail, &inProcessed, + #endif + g_OutBuffer, outAvail, &outProcessed); + if (res != 0) + { + sprintf(rs + strlen(rs), "\nDecoding error = %d\n", res); + res = 1; + break; + } + #ifndef _LZMA_IN_CB + inAvail -= inProcessed; + inBuffer += inProcessed; + #endif + + if (outFile != 0) + if (!MyWriteFileAndCheck(outFile, g_OutBuffer, (size_t)outProcessed)) + { + PrintError(rs, kCantWriteMessage); + res = 1; + break; + } + + if (outSize < outProcessed) + outSizeHigh--; + outSize -= (UInt32)outProcessed; + outSize &= 0xFFFFFFFF; + + if (outProcessed == 0) + { + if (!waitEOS && (outSize != 0 || outSizeHigh != 0)) + res = 1; + break; + } + } + while ((outSize != 0 && outSizeHigh == 0) || outSizeHigh != 0 || waitEOS); + } + + #else + { + #ifndef _LZMA_IN_CB + SizeT inProcessed; + #endif + SizeT outProcessed; + res = LzmaDecode(&state, + #ifdef _LZMA_IN_CB + &g_InBuffer.InCallback, + #else + inStream, compressedSize, &inProcessed, + #endif + outStream, outSizeFull, &outProcessed); + if (res != 0) + { + sprintf(rs + strlen(rs), "\nDecoding error = %d\n", res); + res = 1; + } + else if (outFile != 0) + { + if (!MyWriteFileAndCheck(outFile, outStream, (size_t)outProcessed)) + { + PrintError(rs, kCantWriteMessage); + res = 1; + } + } + } + #endif + + free(state.Probs); + #ifdef _LZMA_OUT_READ + free(state.Dictionary); + #else + free(outStream); + #endif + #ifndef _LZMA_IN_CB + free(inStream); + #endif + return res; +} + +int main2(int numArgs, const char *args[], char *rs) +{ + FILE *inFile = 0; + FILE *outFile = 0; + int res; + + sprintf(rs + strlen(rs), "\nLZMA Decoder 4.26 Copyright (c) 1999-2005 Igor Pavlov 2005-08-05\n"); + if (numArgs < 2 || numArgs > 3) + { + sprintf(rs + strlen(rs), "\nUsage: lzmadec file.lzma [outFile]\n"); + return 1; + } + + inFile = fopen(args[1], "rb"); + if (inFile == 0) + return PrintError(rs, "Can not open input file"); + + if (numArgs > 2) + { + outFile = fopen(args[2], "wb+"); + if (outFile == 0) + return PrintError(rs, "Can not open output file"); + } + + res = main3(inFile, outFile, rs); + + if (outFile != 0) + fclose(outFile); + fclose(inFile); + return res; +} + +int main(int numArgs, const char *args[]) +{ + char rs[800] = { 0 }; + int res = main2(numArgs, args, rs); + printf(rs); + return res; +} diff --git a/lzma443/C/7zip/Compress/LZMA_C/LzmaTypes.h b/lzma443/C/7zip/Compress/LZMA_C/LzmaTypes.h new file mode 100755 index 0000000..288c5e4 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA_C/LzmaTypes.h @@ -0,0 +1,45 @@ +/* +LzmaTypes.h + +Types for LZMA Decoder + +This file written and distributed to public domain by Igor Pavlov. +This file is part of LZMA SDK 4.40 (2006-05-01) +*/ + +#ifndef __LZMATYPES_H +#define __LZMATYPES_H + +#ifndef _7ZIP_BYTE_DEFINED +#define _7ZIP_BYTE_DEFINED +typedef unsigned char Byte; +#endif + +#ifndef _7ZIP_UINT16_DEFINED +#define _7ZIP_UINT16_DEFINED +typedef unsigned short UInt16; +#endif + +#ifndef _7ZIP_UINT32_DEFINED +#define _7ZIP_UINT32_DEFINED +#ifdef _LZMA_UINT32_IS_ULONG +typedef unsigned long UInt32; +#else +typedef unsigned int UInt32; +#endif +#endif + +/* #define _LZMA_SYSTEM_SIZE_T */ +/* Use system's size_t. You can use it to enable 64-bit sizes supporting */ + +#ifndef _7ZIP_SIZET_DEFINED +#define _7ZIP_SIZET_DEFINED +#ifdef _LZMA_SYSTEM_SIZE_T +#include +typedef size_t SizeT; +#else +typedef UInt32 SizeT; +#endif +#endif + +#endif diff --git a/lzma443/C/7zip/Compress/LZMA_C/makefile.gcc b/lzma443/C/7zip/Compress/LZMA_C/makefile.gcc new file mode 100755 index 0000000..43bbef1 --- /dev/null +++ b/lzma443/C/7zip/Compress/LZMA_C/makefile.gcc @@ -0,0 +1,23 @@ +PROG = lzmadec +CXX = gcc +LIB = +RM = rm -f +CFLAGS = -c -O2 -Wall -pedantic -D _LZMA_PROB32 + +OBJS = LzmaTest.o LzmaDecode.o + +all: $(PROG) + +$(PROG): $(OBJS) + $(CXX) -o $(PROG) $(LDFLAGS) $(OBJS) $(LIB) + +LzmaTest.o: LzmaTest.c + $(CXX) $(CFLAGS) LzmaTest.c + +LzmaDecode.o: LzmaDecode.c + $(CXX) $(CFLAGS) LzmaDecode.c + + +clean: + -$(RM) $(PROG) $(OBJS) + diff --git a/lzma443/C/7zip/Compress/RangeCoder/RangeCoder.h b/lzma443/C/7zip/Compress/RangeCoder/RangeCoder.h new file mode 100755 index 0000000..bbb2ba8 --- /dev/null +++ b/lzma443/C/7zip/Compress/RangeCoder/RangeCoder.h @@ -0,0 +1,205 @@ +// Compress/RangeCoder/RangeCoder.h + +#ifndef __COMPRESS_RANGECODER_H +#define __COMPRESS_RANGECODER_H + +#include "../../Common/InBuffer.h" +#include "../../Common/OutBuffer.h" + +namespace NCompress { +namespace NRangeCoder { + +const int kNumTopBits = 24; +const UInt32 kTopValue = (1 << kNumTopBits); + +class CEncoder +{ + UInt32 _cacheSize; + Byte _cache; +public: + UInt64 Low; + UInt32 Range; + COutBuffer Stream; + bool Create(UInt32 bufferSize) { return Stream.Create(bufferSize); } + + void SetStream(ISequentialOutStream *stream) { Stream.SetStream(stream); } + void Init() + { + Stream.Init(); + Low = 0; + Range = 0xFFFFFFFF; + _cacheSize = 1; + _cache = 0; + } + + void FlushData() + { + // Low += 1; + for(int i = 0; i < 5; i++) + ShiftLow(); + } + + HRESULT FlushStream() { return Stream.Flush(); } + + void ReleaseStream() { Stream.ReleaseStream(); } + + void Encode(UInt32 start, UInt32 size, UInt32 total) + { + Low += start * (Range /= total); + Range *= size; + while (Range < kTopValue) + { + Range <<= 8; + ShiftLow(); + } + } + + void ShiftLow() + { + if ((UInt32)Low < (UInt32)0xFF000000 || (int)(Low >> 32) != 0) + { + Byte temp = _cache; + do + { + Stream.WriteByte((Byte)(temp + (Byte)(Low >> 32))); + temp = 0xFF; + } + while(--_cacheSize != 0); + _cache = (Byte)((UInt32)Low >> 24); + } + _cacheSize++; + Low = (UInt32)Low << 8; + } + + void EncodeDirectBits(UInt32 value, int numTotalBits) + { + for (int i = numTotalBits - 1; i >= 0; i--) + { + Range >>= 1; + if (((value >> i) & 1) == 1) + Low += Range; + if (Range < kTopValue) + { + Range <<= 8; + ShiftLow(); + } + } + } + + void EncodeBit(UInt32 size0, UInt32 numTotalBits, UInt32 symbol) + { + UInt32 newBound = (Range >> numTotalBits) * size0; + if (symbol == 0) + Range = newBound; + else + { + Low += newBound; + Range -= newBound; + } + while (Range < kTopValue) + { + Range <<= 8; + ShiftLow(); + } + } + + UInt64 GetProcessedSize() { return Stream.GetProcessedSize() + _cacheSize + 4; } +}; + +class CDecoder +{ +public: + CInBuffer Stream; + UInt32 Range; + UInt32 Code; + bool Create(UInt32 bufferSize) { return Stream.Create(bufferSize); } + + void Normalize() + { + while (Range < kTopValue) + { + Code = (Code << 8) | Stream.ReadByte(); + Range <<= 8; + } + } + + void SetStream(ISequentialInStream *stream) { Stream.SetStream(stream); } + void Init() + { + Stream.Init(); + Code = 0; + Range = 0xFFFFFFFF; + for(int i = 0; i < 5; i++) + Code = (Code << 8) | Stream.ReadByte(); + } + + void ReleaseStream() { Stream.ReleaseStream(); } + + UInt32 GetThreshold(UInt32 total) + { + return (Code) / ( Range /= total); + } + + void Decode(UInt32 start, UInt32 size) + { + Code -= start * Range; + Range *= size; + Normalize(); + } + + UInt32 DecodeDirectBits(int numTotalBits) + { + UInt32 range = Range; + UInt32 code = Code; + UInt32 result = 0; + for (int i = numTotalBits; i != 0; i--) + { + range >>= 1; + /* + result <<= 1; + if (code >= range) + { + code -= range; + result |= 1; + } + */ + UInt32 t = (code - range) >> 31; + code -= range & (t - 1); + result = (result << 1) | (1 - t); + + if (range < kTopValue) + { + code = (code << 8) | Stream.ReadByte(); + range <<= 8; + } + } + Range = range; + Code = code; + return result; + } + + UInt32 DecodeBit(UInt32 size0, UInt32 numTotalBits) + { + UInt32 newBound = (Range >> numTotalBits) * size0; + UInt32 symbol; + if (Code < newBound) + { + symbol = 0; + Range = newBound; + } + else + { + symbol = 1; + Code -= newBound; + Range -= newBound; + } + Normalize(); + return symbol; + } + + UInt64 GetProcessedSize() {return Stream.GetProcessedSize(); } +}; + +}} + +#endif diff --git a/lzma443/C/7zip/Compress/RangeCoder/RangeCoderBit.cpp b/lzma443/C/7zip/Compress/RangeCoder/RangeCoderBit.cpp new file mode 100755 index 0000000..8e4c4d3 --- /dev/null +++ b/lzma443/C/7zip/Compress/RangeCoder/RangeCoderBit.cpp @@ -0,0 +1,80 @@ +// Compress/RangeCoder/RangeCoderBit.cpp + +#include "StdAfx.h" + +#include "RangeCoderBit.h" + +namespace NCompress { +namespace NRangeCoder { + +UInt32 CPriceTables::ProbPrices[kBitModelTotal >> kNumMoveReducingBits]; +static CPriceTables g_PriceTables; + +CPriceTables::CPriceTables() { Init(); } + +void CPriceTables::Init() +{ + const int kNumBits = (kNumBitModelTotalBits - kNumMoveReducingBits); + for(int i = kNumBits - 1; i >= 0; i--) + { + UInt32 start = 1 << (kNumBits - i - 1); + UInt32 end = 1 << (kNumBits - i); + for (UInt32 j = start; j < end; j++) + ProbPrices[j] = (i << kNumBitPriceShiftBits) + + (((end - j) << kNumBitPriceShiftBits) >> (kNumBits - i - 1)); + } + + /* + // simplest: bad solution + for(UInt32 i = 1; i < (kBitModelTotal >> kNumMoveReducingBits) - 1; i++) + ProbPrices[i] = kBitPrice; + */ + + /* + const double kDummyMultMid = (1.0 / kBitPrice) / 2; + const double kDummyMultMid = 0; + // float solution + double ln2 = log(double(2)); + double lnAll = log(double(kBitModelTotal >> kNumMoveReducingBits)); + for(UInt32 i = 1; i < (kBitModelTotal >> kNumMoveReducingBits) - 1; i++) + ProbPrices[i] = UInt32((fabs(lnAll - log(double(i))) / ln2 + kDummyMultMid) * kBitPrice); + */ + + /* + // experimental, slow, solution: + for(UInt32 i = 1; i < (kBitModelTotal >> kNumMoveReducingBits) - 1; i++) + { + const int kCyclesBits = 5; + const UInt32 kCycles = (1 << kCyclesBits); + + UInt32 range = UInt32(-1); + UInt32 bitCount = 0; + for (UInt32 j = 0; j < kCycles; j++) + { + range >>= (kNumBitModelTotalBits - kNumMoveReducingBits); + range *= i; + while(range < (1 << 31)) + { + range <<= 1; + bitCount++; + } + } + bitCount <<= kNumBitPriceShiftBits; + range -= (1 << 31); + for (int k = kNumBitPriceShiftBits - 1; k >= 0; k--) + { + range <<= 1; + if (range > (1 << 31)) + { + bitCount += (1 << k); + range -= (1 << 31); + } + } + ProbPrices[i] = (bitCount + // + (1 << (kCyclesBits - 1)) + ) >> kCyclesBits; + } + */ +} + +}} diff --git a/lzma443/C/7zip/Compress/RangeCoder/RangeCoderBit.h b/lzma443/C/7zip/Compress/RangeCoder/RangeCoderBit.h new file mode 100755 index 0000000..624f887 --- /dev/null +++ b/lzma443/C/7zip/Compress/RangeCoder/RangeCoderBit.h @@ -0,0 +1,120 @@ +// Compress/RangeCoder/RangeCoderBit.h + +#ifndef __COMPRESS_RANGECODER_BIT_H +#define __COMPRESS_RANGECODER_BIT_H + +#include "RangeCoder.h" + +namespace NCompress { +namespace NRangeCoder { + +const int kNumBitModelTotalBits = 11; +const UInt32 kBitModelTotal = (1 << kNumBitModelTotalBits); + +const int kNumMoveReducingBits = 2; + +const int kNumBitPriceShiftBits = 6; +const UInt32 kBitPrice = 1 << kNumBitPriceShiftBits; + +class CPriceTables +{ +public: + static UInt32 ProbPrices[kBitModelTotal >> kNumMoveReducingBits]; + static void Init(); + CPriceTables(); +}; + +template +class CBitModel +{ +public: + UInt32 Prob; + void UpdateModel(UInt32 symbol) + { + /* + Prob -= (Prob + ((symbol - 1) & ((1 << numMoveBits) - 1))) >> numMoveBits; + Prob += (1 - symbol) << (kNumBitModelTotalBits - numMoveBits); + */ + if (symbol == 0) + Prob += (kBitModelTotal - Prob) >> numMoveBits; + else + Prob -= (Prob) >> numMoveBits; + } +public: + void Init() { Prob = kBitModelTotal / 2; } +}; + +template +class CBitEncoder: public CBitModel +{ +public: + void Encode(CEncoder *encoder, UInt32 symbol) + { + /* + encoder->EncodeBit(this->Prob, kNumBitModelTotalBits, symbol); + this->UpdateModel(symbol); + */ + UInt32 newBound = (encoder->Range >> kNumBitModelTotalBits) * this->Prob; + if (symbol == 0) + { + encoder->Range = newBound; + this->Prob += (kBitModelTotal - this->Prob) >> numMoveBits; + } + else + { + encoder->Low += newBound; + encoder->Range -= newBound; + this->Prob -= (this->Prob) >> numMoveBits; + } + if (encoder->Range < kTopValue) + { + encoder->Range <<= 8; + encoder->ShiftLow(); + } + } + UInt32 GetPrice(UInt32 symbol) const + { + return CPriceTables::ProbPrices[ + (((this->Prob - symbol) ^ ((-(int)symbol))) & (kBitModelTotal - 1)) >> kNumMoveReducingBits]; + } + UInt32 GetPrice0() const { return CPriceTables::ProbPrices[this->Prob >> kNumMoveReducingBits]; } + UInt32 GetPrice1() const { return CPriceTables::ProbPrices[(kBitModelTotal - this->Prob) >> kNumMoveReducingBits]; } +}; + + +template +class CBitDecoder: public CBitModel +{ +public: + UInt32 Decode(CDecoder *decoder) + { + UInt32 newBound = (decoder->Range >> kNumBitModelTotalBits) * this->Prob; + if (decoder->Code < newBound) + { + decoder->Range = newBound; + this->Prob += (kBitModelTotal - this->Prob) >> numMoveBits; + if (decoder->Range < kTopValue) + { + decoder->Code = (decoder->Code << 8) | decoder->Stream.ReadByte(); + decoder->Range <<= 8; + } + return 0; + } + else + { + decoder->Range -= newBound; + decoder->Code -= newBound; + this->Prob -= (this->Prob) >> numMoveBits; + if (decoder->Range < kTopValue) + { + decoder->Code = (decoder->Code << 8) | decoder->Stream.ReadByte(); + decoder->Range <<= 8; + } + return 1; + } + } +}; + +}} + +#endif diff --git a/lzma443/C/7zip/Compress/RangeCoder/RangeCoderBitTree.h b/lzma443/C/7zip/Compress/RangeCoder/RangeCoderBitTree.h new file mode 100755 index 0000000..4f0c78b --- /dev/null +++ b/lzma443/C/7zip/Compress/RangeCoder/RangeCoderBitTree.h @@ -0,0 +1,161 @@ +// Compress/RangeCoder/RangeCoderBitTree.h + +#ifndef __COMPRESS_RANGECODER_BIT_TREE_H +#define __COMPRESS_RANGECODER_BIT_TREE_H + +#include "RangeCoderBit.h" +#include "RangeCoderOpt.h" + +namespace NCompress { +namespace NRangeCoder { + +template +class CBitTreeEncoder +{ + CBitEncoder Models[1 << NumBitLevels]; +public: + void Init() + { + for(UInt32 i = 1; i < (1 << NumBitLevels); i++) + Models[i].Init(); + } + void Encode(CEncoder *rangeEncoder, UInt32 symbol) + { + UInt32 modelIndex = 1; + for (int bitIndex = NumBitLevels; bitIndex != 0 ;) + { + bitIndex--; + UInt32 bit = (symbol >> bitIndex) & 1; + Models[modelIndex].Encode(rangeEncoder, bit); + modelIndex = (modelIndex << 1) | bit; + } + }; + void ReverseEncode(CEncoder *rangeEncoder, UInt32 symbol) + { + UInt32 modelIndex = 1; + for (int i = 0; i < NumBitLevels; i++) + { + UInt32 bit = symbol & 1; + Models[modelIndex].Encode(rangeEncoder, bit); + modelIndex = (modelIndex << 1) | bit; + symbol >>= 1; + } + } + UInt32 GetPrice(UInt32 symbol) const + { + symbol |= (1 << NumBitLevels); + UInt32 price = 0; + while (symbol != 1) + { + price += Models[symbol >> 1].GetPrice(symbol & 1); + symbol >>= 1; + } + return price; + } + UInt32 ReverseGetPrice(UInt32 symbol) const + { + UInt32 price = 0; + UInt32 modelIndex = 1; + for (int i = NumBitLevels; i != 0; i--) + { + UInt32 bit = symbol & 1; + symbol >>= 1; + price += Models[modelIndex].GetPrice(bit); + modelIndex = (modelIndex << 1) | bit; + } + return price; + } +}; + +template +class CBitTreeDecoder +{ + CBitDecoder Models[1 << NumBitLevels]; +public: + void Init() + { + for(UInt32 i = 1; i < (1 << NumBitLevels); i++) + Models[i].Init(); + } + UInt32 Decode(CDecoder *rangeDecoder) + { + UInt32 modelIndex = 1; + RC_INIT_VAR + for(int bitIndex = NumBitLevels; bitIndex != 0; bitIndex--) + { + // modelIndex = (modelIndex << 1) + Models[modelIndex].Decode(rangeDecoder); + RC_GETBIT(numMoveBits, Models[modelIndex].Prob, modelIndex) + } + RC_FLUSH_VAR + return modelIndex - (1 << NumBitLevels); + }; + UInt32 ReverseDecode(CDecoder *rangeDecoder) + { + UInt32 modelIndex = 1; + UInt32 symbol = 0; + RC_INIT_VAR + for(int bitIndex = 0; bitIndex < NumBitLevels; bitIndex++) + { + // UInt32 bit = Models[modelIndex].Decode(rangeDecoder); + // modelIndex <<= 1; + // modelIndex += bit; + // symbol |= (bit << bitIndex); + RC_GETBIT2(numMoveBits, Models[modelIndex].Prob, modelIndex, ; , symbol |= (1 << bitIndex)) + } + RC_FLUSH_VAR + return symbol; + } +}; + +template +void ReverseBitTreeEncode(CBitEncoder *Models, + CEncoder *rangeEncoder, int NumBitLevels, UInt32 symbol) +{ + UInt32 modelIndex = 1; + for (int i = 0; i < NumBitLevels; i++) + { + UInt32 bit = symbol & 1; + Models[modelIndex].Encode(rangeEncoder, bit); + modelIndex = (modelIndex << 1) | bit; + symbol >>= 1; + } +} + +template +UInt32 ReverseBitTreeGetPrice(CBitEncoder *Models, + UInt32 NumBitLevels, UInt32 symbol) +{ + UInt32 price = 0; + UInt32 modelIndex = 1; + for (int i = NumBitLevels; i != 0; i--) + { + UInt32 bit = symbol & 1; + symbol >>= 1; + price += Models[modelIndex].GetPrice(bit); + modelIndex = (modelIndex << 1) | bit; + } + return price; +} + +template +UInt32 ReverseBitTreeDecode(CBitDecoder *Models, + CDecoder *rangeDecoder, int NumBitLevels) +{ + UInt32 modelIndex = 1; + UInt32 symbol = 0; + RC_INIT_VAR + for(int bitIndex = 0; bitIndex < NumBitLevels; bitIndex++) + { + // UInt32 bit = Models[modelIndex].Decode(rangeDecoder); + // modelIndex <<= 1; + // modelIndex += bit; + // symbol |= (bit << bitIndex); + RC_GETBIT2(numMoveBits, Models[modelIndex].Prob, modelIndex, ; , symbol |= (1 << bitIndex)) + } + RC_FLUSH_VAR + return symbol; +} + +}} + +#endif diff --git a/lzma443/C/7zip/Compress/RangeCoder/RangeCoderOpt.h b/lzma443/C/7zip/Compress/RangeCoder/RangeCoderOpt.h new file mode 100755 index 0000000..668b9a5 --- /dev/null +++ b/lzma443/C/7zip/Compress/RangeCoder/RangeCoderOpt.h @@ -0,0 +1,31 @@ +// Compress/RangeCoder/RangeCoderOpt.h + +#ifndef __COMPRESS_RANGECODER_OPT_H +#define __COMPRESS_RANGECODER_OPT_H + +#define RC_INIT_VAR \ + UInt32 range = rangeDecoder->Range; \ + UInt32 code = rangeDecoder->Code; + +#define RC_FLUSH_VAR \ + rangeDecoder->Range = range; \ + rangeDecoder->Code = code; + +#define RC_NORMALIZE \ + if (range < NCompress::NRangeCoder::kTopValue) \ + { code = (code << 8) | rangeDecoder->Stream.ReadByte(); range <<= 8; } + +#define RC_GETBIT2(numMoveBits, prob, mi, A0, A1) \ + { UInt32 bound = (range >> NCompress::NRangeCoder::kNumBitModelTotalBits) * prob; \ + if (code < bound) \ + { A0; range = bound; \ + prob += (NCompress::NRangeCoder::kBitModelTotal - prob) >> numMoveBits; \ + mi <<= 1; } \ + else \ + { A1; range -= bound; code -= bound; prob -= (prob) >> numMoveBits; \ + mi = (mi + mi) + 1; }} \ + RC_NORMALIZE + +#define RC_GETBIT(numMoveBits, prob, mi) RC_GETBIT2(numMoveBits, prob, mi, ; , ;) + +#endif diff --git a/lzma443/C/7zip/Compress/RangeCoder/StdAfx.h b/lzma443/C/7zip/Compress/RangeCoder/StdAfx.h new file mode 100755 index 0000000..b637fd4 --- /dev/null +++ b/lzma443/C/7zip/Compress/RangeCoder/StdAfx.h @@ -0,0 +1,6 @@ +// StdAfx.h + +#ifndef __STDAFX_H +#define __STDAFX_H + +#endif diff --git a/lzma443/C/7zip/ICoder.h b/lzma443/C/7zip/ICoder.h new file mode 100755 index 0000000..d84575d --- /dev/null +++ b/lzma443/C/7zip/ICoder.h @@ -0,0 +1,163 @@ +// ICoder.h + +#ifndef __ICODER_H +#define __ICODER_H + +#include "IStream.h" + +// "23170F69-40C1-278A-0000-000400xx0000" +#define CODER_INTERFACE(i, x) \ +DEFINE_GUID(IID_ ## i, \ +0x23170F69, 0x40C1, 0x278A, 0x00, 0x00, 0x00, 0x04, 0x00, x, 0x00, 0x00); \ +struct i: public IUnknown + +CODER_INTERFACE(ICompressProgressInfo, 0x04) +{ + STDMETHOD(SetRatioInfo)(const UInt64 *inSize, const UInt64 *outSize) PURE; +}; + +CODER_INTERFACE(ICompressCoder, 0x05) +{ + STDMETHOD(Code)(ISequentialInStream *inStream, + ISequentialOutStream *outStream, + const UInt64 *inSize, + const UInt64 *outSize, + ICompressProgressInfo *progress) PURE; +}; + +CODER_INTERFACE(ICompressCoder2, 0x18) +{ + STDMETHOD(Code)(ISequentialInStream **inStreams, + const UInt64 **inSizes, + UInt32 numInStreams, + ISequentialOutStream **outStreams, + const UInt64 **outSizes, + UInt32 numOutStreams, + ICompressProgressInfo *progress) PURE; +}; + +namespace NCoderPropID +{ + enum EEnum + { + kDictionarySize = 0x400, + kUsedMemorySize, + kOrder, + kPosStateBits = 0x440, + kLitContextBits, + kLitPosBits, + kNumFastBytes = 0x450, + kMatchFinder, + kMatchFinderCycles, + kNumPasses = 0x460, + kAlgorithm = 0x470, + kMultiThread = 0x480, + kNumThreads, + kEndMarker = 0x490 + }; +} + +CODER_INTERFACE(ICompressSetCoderProperties, 0x20) +{ + STDMETHOD(SetCoderProperties)(const PROPID *propIDs, + const PROPVARIANT *properties, UInt32 numProperties) PURE; +}; + +/* +CODER_INTERFACE(ICompressSetCoderProperties, 0x21) +{ + STDMETHOD(SetDecoderProperties)(ISequentialInStream *inStream) PURE; +}; +*/ + +CODER_INTERFACE(ICompressSetDecoderProperties2, 0x22) +{ + STDMETHOD(SetDecoderProperties2)(const Byte *data, UInt32 size) PURE; +}; + +CODER_INTERFACE(ICompressWriteCoderProperties, 0x23) +{ + STDMETHOD(WriteCoderProperties)(ISequentialOutStream *outStreams) PURE; +}; + +CODER_INTERFACE(ICompressGetInStreamProcessedSize, 0x24) +{ + STDMETHOD(GetInStreamProcessedSize)(UInt64 *value) PURE; +}; + +CODER_INTERFACE(ICompressSetCoderMt, 0x25) +{ + STDMETHOD(SetNumberOfThreads)(UInt32 numThreads) PURE; +}; + +CODER_INTERFACE(ICompressGetSubStreamSize, 0x30) +{ + STDMETHOD(GetSubStreamSize)(UInt64 subStream, UInt64 *value) PURE; +}; + +CODER_INTERFACE(ICompressSetInStream, 0x31) +{ + STDMETHOD(SetInStream)(ISequentialInStream *inStream) PURE; + STDMETHOD(ReleaseInStream)() PURE; +}; + +CODER_INTERFACE(ICompressSetOutStream, 0x32) +{ + STDMETHOD(SetOutStream)(ISequentialOutStream *outStream) PURE; + STDMETHOD(ReleaseOutStream)() PURE; +}; + +CODER_INTERFACE(ICompressSetInStreamSize, 0x33) +{ + STDMETHOD(SetInStreamSize)(const UInt64 *inSize) PURE; +}; + +CODER_INTERFACE(ICompressSetOutStreamSize, 0x34) +{ + STDMETHOD(SetOutStreamSize)(const UInt64 *outSize) PURE; +}; + +CODER_INTERFACE(ICompressFilter, 0x40) +{ + STDMETHOD(Init)() PURE; + STDMETHOD_(UInt32, Filter)(Byte *data, UInt32 size) PURE; + // Filter return outSize (UInt32) + // if (outSize <= size): Filter have converted outSize bytes + // if (outSize > size): Filter have not converted anything. + // and it needs at least outSize bytes to convert one block + // (it's for crypto block algorithms). +}; + +CODER_INTERFACE(ICryptoProperties, 0x80) +{ + STDMETHOD(SetKey)(const Byte *data, UInt32 size) PURE; + STDMETHOD(SetInitVector)(const Byte *data, UInt32 size) PURE; +}; + +CODER_INTERFACE(ICryptoSetPassword, 0x90) +{ + STDMETHOD(CryptoSetPassword)(const Byte *data, UInt32 size) PURE; +}; + +CODER_INTERFACE(ICryptoSetCRC, 0xA0) +{ + STDMETHOD(CryptoSetCRC)(UInt32 crc) PURE; +}; + +////////////////////// +// It's for DLL file +namespace NMethodPropID +{ + enum EEnum + { + kID, + kName, + kDecoder, + kEncoder, + kInStreams, + kOutStreams, + kDescription + }; +} + +#endif diff --git a/lzma443/C/7zip/IStream.h b/lzma443/C/7zip/IStream.h new file mode 100755 index 0000000..bba21a3 --- /dev/null +++ b/lzma443/C/7zip/IStream.h @@ -0,0 +1,62 @@ +// IStream.h + +#ifndef __ISTREAM_H +#define __ISTREAM_H + +#include "../Common/MyUnknown.h" +#include "../Common/Types.h" + +// "23170F69-40C1-278A-0000-000300xx0000" + +#define STREAM_INTERFACE_SUB(i, b, x) \ +DEFINE_GUID(IID_ ## i, \ +0x23170F69, 0x40C1, 0x278A, 0x00, 0x00, 0x00, 0x03, 0x00, x, 0x00, 0x00); \ +struct i: public b + +#define STREAM_INTERFACE(i, x) STREAM_INTERFACE_SUB(i, IUnknown, x) + +STREAM_INTERFACE(ISequentialInStream, 0x01) +{ + STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize) PURE; + /* + Out: if size != 0, return_value = S_OK and (*processedSize == 0), + then there are no more bytes in stream. + if (size > 0) && there are bytes in stream, + this function must read at least 1 byte. + This function is allowed to read less than number of remaining bytes in stream. + You must call Read function in loop, if you need exact amount of data + */ +}; + +STREAM_INTERFACE(ISequentialOutStream, 0x02) +{ + STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize) PURE; + /* + if (size > 0) this function must write at least 1 byte. + This function is allowed to write less than "size". + You must call Write function in loop, if you need to write exact amount of data + */ +}; + +STREAM_INTERFACE_SUB(IInStream, ISequentialInStream, 0x03) +{ + STDMETHOD(Seek)(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition) PURE; +}; + +STREAM_INTERFACE_SUB(IOutStream, ISequentialOutStream, 0x04) +{ + STDMETHOD(Seek)(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition) PURE; + STDMETHOD(SetSize)(Int64 newSize) PURE; +}; + +STREAM_INTERFACE(IStreamGetSize, 0x06) +{ + STDMETHOD(GetSize)(UInt64 *size) PURE; +}; + +STREAM_INTERFACE(IOutStreamFlush, 0x07) +{ + STDMETHOD(Flush)() PURE; +}; + +#endif diff --git a/lzma443/C/Common/Alloc.cpp b/lzma443/C/Common/Alloc.cpp new file mode 100755 index 0000000..1f1c68b --- /dev/null +++ b/lzma443/C/Common/Alloc.cpp @@ -0,0 +1,125 @@ +// Common/Alloc.cpp + +#include "StdAfx.h" + +#ifdef _WIN32 +#include "MyWindows.h" +#else +#include +#endif + +#include "Alloc.h" + +/* #define _SZ_ALLOC_DEBUG */ +/* use _SZ_ALLOC_DEBUG to debug alloc/free operations */ +#ifdef _SZ_ALLOC_DEBUG +#include +int g_allocCount = 0; +int g_allocCountMid = 0; +int g_allocCountBig = 0; +#endif + +void *MyAlloc(size_t size) throw() +{ + if (size == 0) + return 0; + #ifdef _SZ_ALLOC_DEBUG + fprintf(stderr, "\nAlloc %10d bytes; count = %10d", size, g_allocCount++); + #endif + return ::malloc(size); +} + +void MyFree(void *address) throw() +{ + #ifdef _SZ_ALLOC_DEBUG + if (address != 0) + fprintf(stderr, "\nFree; count = %10d", --g_allocCount); + #endif + + ::free(address); +} + +#ifdef __MINGW_H +void *MidAlloc(size_t size ) throw() { return MyAlloc(size);} +void MidFree(void *address) throw(){ return MyFree(address);} +void *BigAlloc(size_t size ) throw() { return MyAlloc(size);} +void BigFree(void *address) throw(){ return MyFree(address);} +#endif + +#if defined(_WIN32) && !defined(__MINGW_H) + +void *MidAlloc(size_t size) throw() +{ + if (size == 0) + return 0; + #ifdef _SZ_ALLOC_DEBUG + fprintf(stderr, "\nAlloc_Mid %10d bytes; count = %10d", size, g_allocCountMid++); + #endif + return ::VirtualAlloc(0, size, MEM_COMMIT, PAGE_READWRITE); +} + +void MidFree(void *address) throw() +{ + #ifdef _SZ_ALLOC_DEBUG + if (address != 0) + fprintf(stderr, "\nFree_Mid; count = %10d", --g_allocCountMid); + #endif + if (address == 0) + return; + ::VirtualFree(address, 0, MEM_RELEASE); +} + +static SIZE_T g_LargePageSize = + #ifdef _WIN64 + (1 << 21); + #else + (1 << 22); + #endif + +typedef SIZE_T (WINAPI *GetLargePageMinimumP)(); + +bool SetLargePageSize() +{ + GetLargePageMinimumP largePageMinimum = (GetLargePageMinimumP) + ::GetProcAddress(::GetModuleHandle(TEXT("kernel32.dll")), "GetLargePageMinimum"); + if (largePageMinimum == 0) + return false; + SIZE_T size = largePageMinimum(); + if (size == 0 || (size & (size - 1)) != 0) + return false; + g_LargePageSize = size; + return true; +} + + +void *BigAlloc(size_t size) throw() +{ + if (size == 0) + return 0; + #ifdef _SZ_ALLOC_DEBUG + fprintf(stderr, "\nAlloc_Big %10d bytes; count = %10d", size, g_allocCountBig++); + #endif + + if (size >= (1 << 18)) + { + void *res = ::VirtualAlloc(0, (size + g_LargePageSize - 1) & (~(g_LargePageSize - 1)), + MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE); + if (res != 0) + return res; + } + return ::VirtualAlloc(0, size, MEM_COMMIT, PAGE_READWRITE); +} + +void BigFree(void *address) throw() +{ + #ifdef _SZ_ALLOC_DEBUG + if (address != 0) + fprintf(stderr, "\nFree_Big; count = %10d", --g_allocCountBig); + #endif + + if (address == 0) + return; + ::VirtualFree(address, 0, MEM_RELEASE); +} + +#endif diff --git a/lzma443/C/Common/Alloc.h b/lzma443/C/Common/Alloc.h new file mode 100755 index 0000000..d444f63 --- /dev/null +++ b/lzma443/C/Common/Alloc.h @@ -0,0 +1,29 @@ +// Common/Alloc.h + +#ifndef __COMMON_ALLOC_H +#define __COMMON_ALLOC_H + +#include + +void *MyAlloc(size_t size) throw(); +void MyFree(void *address) throw(); + +#ifdef _WIN32 + +bool SetLargePageSize(); + +void *MidAlloc(size_t size) throw(); +void MidFree(void *address) throw(); +void *BigAlloc(size_t size) throw(); +void BigFree(void *address) throw(); + +#else + +#define MidAlloc(size) MyAlloc(size) +#define MidFree(address) MyFree(address) +#define BigAlloc(size) MyAlloc(size) +#define BigFree(address) MyFree(address) + +#endif + +#endif diff --git a/lzma443/C/Common/CRC.cpp b/lzma443/C/Common/CRC.cpp new file mode 100755 index 0000000..35e1a18 --- /dev/null +++ b/lzma443/C/Common/CRC.cpp @@ -0,0 +1,61 @@ +// Common/CRC.cpp + +#include "StdAfx.h" + +#include "CRC.h" + +static const UInt32 kCRCPoly = 0xEDB88320; + +UInt32 CCRC::Table[256]; + +void CCRC::InitTable() +{ + for (UInt32 i = 0; i < 256; i++) + { + UInt32 r = i; + for (int j = 0; j < 8; j++) + if (r & 1) + r = (r >> 1) ^ kCRCPoly; + else + r >>= 1; + CCRC::Table[i] = r; + } +} + +class CCRCTableInit +{ +public: + CCRCTableInit() { CCRC::InitTable(); } +} g_CRCTableInit; + +void CCRC::UpdateByte(Byte b) +{ + _value = Table[((Byte)(_value)) ^ b] ^ (_value >> 8); +} + +void CCRC::UpdateUInt16(UInt16 v) +{ + UpdateByte(Byte(v)); + UpdateByte(Byte(v >> 8)); +} + +void CCRC::UpdateUInt32(UInt32 v) +{ + for (int i = 0; i < 4; i++) + UpdateByte((Byte)(v >> (8 * i))); +} + +void CCRC::UpdateUInt64(UInt64 v) +{ + for (int i = 0; i < 8; i++) + UpdateByte((Byte)(v >> (8 * i))); +} + +void CCRC::Update(const void *data, size_t size) +{ + UInt32 v = _value; + const Byte *p = (const Byte *)data; + for (; size > 0 ; size--, p++) + v = Table[((Byte)(v)) ^ *p] ^ (v >> 8); + _value = v; +} diff --git a/lzma443/C/Common/CRC.h b/lzma443/C/Common/CRC.h new file mode 100755 index 0000000..6b4f1b7 --- /dev/null +++ b/lzma443/C/Common/CRC.h @@ -0,0 +1,36 @@ +// Common/CRC.h + +#ifndef __COMMON_CRC_H +#define __COMMON_CRC_H + +#include +#include "Types.h" + +class CCRC +{ + UInt32 _value; +public: + static UInt32 Table[256]; + static void InitTable(); + + CCRC(): _value(0xFFFFFFFF){}; + void Init() { _value = 0xFFFFFFFF; } + void UpdateByte(Byte v); + void UpdateUInt16(UInt16 v); + void UpdateUInt32(UInt32 v); + void UpdateUInt64(UInt64 v); + void Update(const void *data, size_t size); + UInt32 GetDigest() const { return _value ^ 0xFFFFFFFF; } + static UInt32 CalculateDigest(const void *data, size_t size) + { + CCRC crc; + crc.Update(data, size); + return crc.GetDigest(); + } + static bool VerifyDigest(UInt32 digest, const void *data, size_t size) + { + return (CalculateDigest(data, size) == digest); + } +}; + +#endif diff --git a/lzma443/C/Common/C_FileIO.cpp b/lzma443/C/Common/C_FileIO.cpp new file mode 100755 index 0000000..7d9e00d --- /dev/null +++ b/lzma443/C/Common/C_FileIO.cpp @@ -0,0 +1,78 @@ +// Common/C_FileIO.h + +#include "C_FileIO.h" + +#include +#include + +namespace NC { +namespace NFile { +namespace NIO { + +bool CFileBase::OpenBinary(const char *name, int flags) +{ + #ifdef O_BINARY + flags |= O_BINARY; + #endif + Close(); + _handle = ::open(name, flags, 0666); + return _handle != -1; +} + +bool CFileBase::Close() +{ + if(_handle == -1) + return true; + if (close(_handle) != 0) + return false; + _handle = -1; + return true; +} + +bool CFileBase::GetLength(UInt64 &length) const +{ + off_t curPos = Seek(0, SEEK_CUR); + off_t lengthTemp = Seek(0, SEEK_END); + Seek(curPos, SEEK_SET); + length = (UInt64)lengthTemp; + return true; +} + +off_t CFileBase::Seek(off_t distanceToMove, int moveMethod) const +{ + return ::lseek(_handle, distanceToMove, moveMethod); +} + +///////////////////////// +// CInFile + +bool CInFile::Open(const char *name) +{ + return CFileBase::OpenBinary(name, O_RDONLY); +} + +ssize_t CInFile::Read(void *data, size_t size) +{ + return read(_handle, data, size); +} + +///////////////////////// +// COutFile + +bool COutFile::Create(const char *name, bool createAlways) +{ + if (createAlways) + { + Close(); + _handle = ::creat(name, 0666); + return _handle != -1; + } + return OpenBinary(name, O_CREAT | O_EXCL | O_WRONLY); +} + +ssize_t COutFile::Write(const void *data, size_t size) +{ + return write(_handle, data, size); +} + +}}} diff --git a/lzma443/C/Common/C_FileIO.h b/lzma443/C/Common/C_FileIO.h new file mode 100755 index 0000000..2ad0716 --- /dev/null +++ b/lzma443/C/Common/C_FileIO.h @@ -0,0 +1,45 @@ +// Common/C_FileIO.h + +#ifndef __COMMON_C_FILEIO_H +#define __COMMON_C_FILEIO_H + +#include +#include + +#include "Types.h" +#include "MyWindows.h" + +namespace NC { +namespace NFile { +namespace NIO { + +class CFileBase +{ +protected: + int _handle; + bool OpenBinary(const char *name, int flags); +public: + CFileBase(): _handle(-1) {}; + ~CFileBase() { Close(); } + bool Close(); + bool GetLength(UInt64 &length) const; + off_t Seek(off_t distanceToMove, int moveMethod) const; +}; + +class CInFile: public CFileBase +{ +public: + bool Open(const char *name); + ssize_t Read(void *data, size_t size); +}; + +class COutFile: public CFileBase +{ +public: + bool Create(const char *name, bool createAlways); + ssize_t Write(const void *data, size_t size); +}; + +}}} + +#endif diff --git a/lzma443/C/Common/ComTry.h b/lzma443/C/Common/ComTry.h new file mode 100755 index 0000000..5153362 --- /dev/null +++ b/lzma443/C/Common/ComTry.h @@ -0,0 +1,17 @@ +// ComTry.h + +#ifndef __COM_TRY_H +#define __COM_TRY_H + +#include "MyWindows.h" +// #include "Exception.h" +// #include "NewHandler.h" + +#define COM_TRY_BEGIN try { +#define COM_TRY_END } catch(...) { return E_OUTOFMEMORY; } + + // catch(const CNewException &) { return E_OUTOFMEMORY; }\ + // catch(const CSystemException &e) { return e.ErrorCode; }\ + // catch(...) { return E_FAIL; } + +#endif diff --git a/lzma443/C/Common/CommandLineParser.cpp b/lzma443/C/Common/CommandLineParser.cpp new file mode 100755 index 0000000..69c4184 --- /dev/null +++ b/lzma443/C/Common/CommandLineParser.cpp @@ -0,0 +1,232 @@ +// CommandLineParser.cpp + +#include "StdAfx.h" + +#include "CommandLineParser.h" + +namespace NCommandLineParser { + +void SplitCommandLine(const UString &src, UString &dest1, UString &dest2) +{ + dest1.Empty(); + dest2.Empty(); + bool quoteMode = false; + int i; + for (i = 0; i < src.Length(); i++) + { + wchar_t c = src[i]; + if (c == L'\"') + quoteMode = !quoteMode; + else if (c == L' ' && !quoteMode) + { + i++; + break; + } + else + dest1 += c; + } + dest2 = src.Mid(i); +} + +void SplitCommandLine(const UString &s, UStringVector &parts) +{ + UString sTemp = s; + sTemp.Trim(); + parts.Clear(); + while (true) + { + UString s1, s2; + SplitCommandLine(sTemp, s1, s2); + // s1.Trim(); + // s2.Trim(); + if (!s1.IsEmpty()) + parts.Add(s1); + if (s2.IsEmpty()) + return; + sTemp = s2; + } +} + + +static const wchar_t kSwitchID1 = '-'; +// static const wchar_t kSwitchID2 = '/'; + +static const wchar_t kSwitchMinus = '-'; +static const wchar_t *kStopSwitchParsing = L"--"; + +static bool IsItSwitchChar(wchar_t c) +{ + return (c == kSwitchID1 /*|| c == kSwitchID2 */); +} + +CParser::CParser(int numSwitches): + _numSwitches(numSwitches) +{ + _switches = new CSwitchResult[_numSwitches]; +} + +CParser::~CParser() +{ + delete []_switches; +} + +void CParser::ParseStrings(const CSwitchForm *switchForms, + const UStringVector &commandStrings) +{ + int numCommandStrings = commandStrings.Size(); + bool stopSwitch = false; + for (int i = 0; i < numCommandStrings; i++) + { + const UString &s = commandStrings[i]; + if (stopSwitch) + NonSwitchStrings.Add(s); + else + if (s == kStopSwitchParsing) + stopSwitch = true; + else + if (!ParseString(s, switchForms)) + NonSwitchStrings.Add(s); + } +} + +// if string contains switch then function updates switch structures +// out: (string is a switch) +bool CParser::ParseString(const UString &s, const CSwitchForm *switchForms) +{ + int len = s.Length(); + if (len == 0) + return false; + int pos = 0; + if (!IsItSwitchChar(s[pos])) + return false; + while(pos < len) + { + if (IsItSwitchChar(s[pos])) + pos++; + const int kNoLen = -1; + int matchedSwitchIndex = 0; // GCC Warning + int maxLen = kNoLen; + for(int switchIndex = 0; switchIndex < _numSwitches; switchIndex++) + { + int switchLen = MyStringLen(switchForms[switchIndex].IDString); + if (switchLen <= maxLen || pos + switchLen > len) + continue; + + UString temp = s + pos; + temp = temp.Left(switchLen); + if(temp.CompareNoCase(switchForms[switchIndex].IDString) == 0) + // if(_strnicmp(switchForms[switchIndex].IDString, LPCSTR(s) + pos, switchLen) == 0) + { + matchedSwitchIndex = switchIndex; + maxLen = switchLen; + } + } + if (maxLen == kNoLen) + throw "maxLen == kNoLen"; + CSwitchResult &matchedSwitch = _switches[matchedSwitchIndex]; + const CSwitchForm &switchForm = switchForms[matchedSwitchIndex]; + if ((!switchForm.Multi) && matchedSwitch.ThereIs) + throw "switch must be single"; + matchedSwitch.ThereIs = true; + pos += maxLen; + int tailSize = len - pos; + NSwitchType::EEnum type = switchForm.Type; + switch(type) + { + case NSwitchType::kPostMinus: + { + if (tailSize == 0) + matchedSwitch.WithMinus = false; + else + { + matchedSwitch.WithMinus = (s[pos] == kSwitchMinus); + if (matchedSwitch.WithMinus) + pos++; + } + break; + } + case NSwitchType::kPostChar: + { + if (tailSize < switchForm.MinLen) + throw "switch is not full"; + UString set = switchForm.PostCharSet; + const int kEmptyCharValue = -1; + if (tailSize == 0) + matchedSwitch.PostCharIndex = kEmptyCharValue; + else + { + int index = set.Find(s[pos]); + if (index < 0) + matchedSwitch.PostCharIndex = kEmptyCharValue; + else + { + matchedSwitch.PostCharIndex = index; + pos++; + } + } + break; + } + case NSwitchType::kLimitedPostString: + case NSwitchType::kUnLimitedPostString: + { + int minLen = switchForm.MinLen; + if (tailSize < minLen) + throw "switch is not full"; + if (type == NSwitchType::kUnLimitedPostString) + { + matchedSwitch.PostStrings.Add(s.Mid(pos)); + return true; + } + int maxLen = switchForm.MaxLen; + UString stringSwitch = s.Mid(pos, minLen); + pos += minLen; + for(int i = minLen; i < maxLen && pos < len; i++, pos++) + { + wchar_t c = s[pos]; + if (IsItSwitchChar(c)) + break; + stringSwitch += c; + } + matchedSwitch.PostStrings.Add(stringSwitch); + break; + } + case NSwitchType::kSimple: + break; + } + } + return true; +} + +const CSwitchResult& CParser::operator[](size_t index) const +{ + return _switches[index]; +} + +///////////////////////////////// +// Command parsing procedures + +int ParseCommand(int numCommandForms, const CCommandForm *commandForms, + const UString &commandString, UString &postString) +{ + for(int i = 0; i < numCommandForms; i++) + { + const UString id = commandForms[i].IDString; + if (commandForms[i].PostStringMode) + { + if(commandString.Find(id) == 0) + { + postString = commandString.Mid(id.Length()); + return i; + } + } + else + if (commandString == id) + { + postString.Empty(); + return i; + } + } + return -1; +} + +} diff --git a/lzma443/C/Common/CommandLineParser.h b/lzma443/C/Common/CommandLineParser.h new file mode 100755 index 0000000..af698db --- /dev/null +++ b/lzma443/C/Common/CommandLineParser.h @@ -0,0 +1,72 @@ +// Common/CommandLineParser.h + +#ifndef __COMMON_COMMANDLINEPARSER_H +#define __COMMON_COMMANDLINEPARSER_H + +#include "Common/String.h" + +namespace NCommandLineParser { + +void SplitCommandLine(const UString &src, UString &dest1, UString &dest2); +void SplitCommandLine(const UString &s, UStringVector &parts); + +namespace NSwitchType { + enum EEnum + { + kSimple, + kPostMinus, + kLimitedPostString, + kUnLimitedPostString, + kPostChar + }; +} + +struct CSwitchForm +{ + const wchar_t *IDString; + NSwitchType::EEnum Type; + bool Multi; + int MinLen; + int MaxLen; + const wchar_t *PostCharSet; +}; + +struct CSwitchResult +{ + bool ThereIs; + bool WithMinus; + UStringVector PostStrings; + int PostCharIndex; + CSwitchResult(): ThereIs(false) {}; +}; + +class CParser +{ + int _numSwitches; + CSwitchResult *_switches; + bool ParseString(const UString &s, const CSwitchForm *switchForms); +public: + UStringVector NonSwitchStrings; + CParser(int numSwitches); + ~CParser(); + void ParseStrings(const CSwitchForm *switchForms, + const UStringVector &commandStrings); + const CSwitchResult& operator[](size_t index) const; +}; + +///////////////////////////////// +// Command parsing procedures + +struct CCommandForm +{ + wchar_t *IDString; + bool PostStringMode; +}; + +// Returns: Index of form and postString; -1, if there is no match +int ParseCommand(int numCommandForms, const CCommandForm *commandForms, + const UString &commandString, UString &postString); + +} + +#endif diff --git a/lzma443/C/Common/Defs.h b/lzma443/C/Common/Defs.h new file mode 100755 index 0000000..dad3ae8 --- /dev/null +++ b/lzma443/C/Common/Defs.h @@ -0,0 +1,20 @@ +// Common/Defs.h + +#ifndef __COMMON_DEFS_H +#define __COMMON_DEFS_H + +template inline T MyMin(T a, T b) + { return a < b ? a : b; } +template inline T MyMax(T a, T b) + { return a > b ? a : b; } + +template inline int MyCompare(T a, T b) + { return a < b ? -1 : (a == b ? 0 : 1); } + +inline int BoolToInt(bool value) + { return (value ? 1: 0); } + +inline bool IntToBool(int value) + { return (value != 0); } + +#endif diff --git a/lzma443/C/Common/MyCom.h b/lzma443/C/Common/MyCom.h new file mode 100755 index 0000000..e903493 --- /dev/null +++ b/lzma443/C/Common/MyCom.h @@ -0,0 +1,203 @@ +// MyCom.h + +#ifndef __MYCOM_H +#define __MYCOM_H + +#include "MyWindows.h" + +#define RINOK(x) { HRESULT __result_ = (x); if(__result_ != S_OK) return __result_; } + +template +class CMyComPtr +{ + T* _p; +public: + // typedef T _PtrClass; + CMyComPtr() { _p = NULL;} + CMyComPtr(T* p) {if ((_p = p) != NULL) p->AddRef(); } + CMyComPtr(const CMyComPtr& lp) + { + if ((_p = lp._p) != NULL) + _p->AddRef(); + } + ~CMyComPtr() { if (_p) _p->Release(); } + void Release() { if (_p) { _p->Release(); _p = NULL; } } + operator T*() const { return (T*)_p; } + // T& operator*() const { return *_p; } + T** operator&() { return &_p; } + T* operator->() const { return _p; } + T* operator=(T* p) + { + if (p != 0) + p->AddRef(); + if (_p) + _p->Release(); + _p = p; + return p; + } + T* operator=(const CMyComPtr& lp) { return (*this = lp._p); } + bool operator!() const { return (_p == NULL); } + // bool operator==(T* pT) const { return _p == pT; } + // Compare two objects for equivalence + void Attach(T* p2) + { + Release(); + _p = p2; + } + T* Detach() + { + T* pt = _p; + _p = NULL; + return pt; + } + #ifdef _WIN32 + HRESULT CoCreateInstance(REFCLSID rclsid, REFIID iid, LPUNKNOWN pUnkOuter = NULL, DWORD dwClsContext = CLSCTX_ALL) + { + return ::CoCreateInstance(rclsid, pUnkOuter, dwClsContext, iid, (void**)&_p); + } + #endif + /* + HRESULT CoCreateInstance(LPCOLESTR szProgID, LPUNKNOWN pUnkOuter = NULL, DWORD dwClsContext = CLSCTX_ALL) + { + CLSID clsid; + HRESULT hr = CLSIDFromProgID(szProgID, &clsid); + ATLASSERT(_p == NULL); + if (SUCCEEDED(hr)) + hr = ::CoCreateInstance(clsid, pUnkOuter, dwClsContext, __uuidof(T), (void**)&_p); + return hr; + } + */ + template + HRESULT QueryInterface(REFGUID iid, Q** pp) const + { + return _p->QueryInterface(iid, (void**)pp); + } +}; + +////////////////////////////////////////////////////////// + +class CMyComBSTR +{ +public: + BSTR m_str; + CMyComBSTR() { m_str = NULL; } + CMyComBSTR(LPCOLESTR pSrc) { m_str = ::SysAllocString(pSrc); } + // CMyComBSTR(int nSize) { m_str = ::SysAllocStringLen(NULL, nSize); } + // CMyComBSTR(int nSize, LPCOLESTR sz) { m_str = ::SysAllocStringLen(sz, nSize); } + CMyComBSTR(const CMyComBSTR& src) { m_str = src.MyCopy(); } + /* + CMyComBSTR(REFGUID src) + { + LPOLESTR szGuid; + StringFromCLSID(src, &szGuid); + m_str = ::SysAllocString(szGuid); + CoTaskMemFree(szGuid); + } + */ + ~CMyComBSTR() { ::SysFreeString(m_str); } + CMyComBSTR& operator=(const CMyComBSTR& src) + { + if (m_str != src.m_str) + { + if (m_str) + ::SysFreeString(m_str); + m_str = src.MyCopy(); + } + return *this; + } + CMyComBSTR& operator=(LPCOLESTR pSrc) + { + ::SysFreeString(m_str); + m_str = ::SysAllocString(pSrc); + return *this; + } + unsigned int Length() const { return ::SysStringLen(m_str); } + operator BSTR() const { return m_str; } + BSTR* operator&() { return &m_str; } + BSTR MyCopy() const + { + int byteLen = ::SysStringByteLen(m_str); + BSTR res = ::SysAllocStringByteLen(NULL, byteLen); + memmove(res, m_str, byteLen); + return res; + } + void Attach(BSTR src) { m_str = src; } + BSTR Detach() + { + BSTR s = m_str; + m_str = NULL; + return s; + } + void Empty() + { + ::SysFreeString(m_str); + m_str = NULL; + } + bool operator!() const { return (m_str == NULL); } +}; + + +////////////////////////////////////////////////////////// + +class CMyUnknownImp +{ +public: + ULONG __m_RefCount; + CMyUnknownImp(): __m_RefCount(0) {} +}; + +#define MY_QUERYINTERFACE_BEGIN STDMETHOD(QueryInterface) \ + (REFGUID iid, void **outObject) { + +#define MY_QUERYINTERFACE_ENTRY(i) if (iid == IID_ ## i) \ + { *outObject = (void *)(i *)this; AddRef(); return S_OK; } + +#define MY_QUERYINTERFACE_END return E_NOINTERFACE; } + +#define MY_ADDREF_RELEASE \ +STDMETHOD_(ULONG, AddRef)() { return ++__m_RefCount; } \ +STDMETHOD_(ULONG, Release)() { if (--__m_RefCount != 0) \ + return __m_RefCount; delete this; return 0; } + +#define MY_UNKNOWN_IMP_SPEC(i) \ + MY_QUERYINTERFACE_BEGIN \ + i \ + MY_QUERYINTERFACE_END \ + MY_ADDREF_RELEASE + + +#define MY_UNKNOWN_IMP STDMETHOD(QueryInterface)(REFGUID, void **) { \ + MY_QUERYINTERFACE_END \ + MY_ADDREF_RELEASE + +#define MY_UNKNOWN_IMP1(i) MY_UNKNOWN_IMP_SPEC( \ + MY_QUERYINTERFACE_ENTRY(i) \ + ) + +#define MY_UNKNOWN_IMP2(i1, i2) MY_UNKNOWN_IMP_SPEC( \ + MY_QUERYINTERFACE_ENTRY(i1) \ + MY_QUERYINTERFACE_ENTRY(i2) \ + ) + +#define MY_UNKNOWN_IMP3(i1, i2, i3) MY_UNKNOWN_IMP_SPEC( \ + MY_QUERYINTERFACE_ENTRY(i1) \ + MY_QUERYINTERFACE_ENTRY(i2) \ + MY_QUERYINTERFACE_ENTRY(i3) \ + ) + +#define MY_UNKNOWN_IMP4(i1, i2, i3, i4) MY_UNKNOWN_IMP_SPEC( \ + MY_QUERYINTERFACE_ENTRY(i1) \ + MY_QUERYINTERFACE_ENTRY(i2) \ + MY_QUERYINTERFACE_ENTRY(i3) \ + MY_QUERYINTERFACE_ENTRY(i4) \ + ) + +#define MY_UNKNOWN_IMP5(i1, i2, i3, i4, i5) MY_UNKNOWN_IMP_SPEC( \ + MY_QUERYINTERFACE_ENTRY(i1) \ + MY_QUERYINTERFACE_ENTRY(i2) \ + MY_QUERYINTERFACE_ENTRY(i3) \ + MY_QUERYINTERFACE_ENTRY(i4) \ + MY_QUERYINTERFACE_ENTRY(i5) \ + ) + +#endif diff --git a/lzma443/C/Common/MyGuidDef.h b/lzma443/C/Common/MyGuidDef.h new file mode 100755 index 0000000..2c954f8 --- /dev/null +++ b/lzma443/C/Common/MyGuidDef.h @@ -0,0 +1,54 @@ +// Common/MyGuidDef.h + +#ifndef GUID_DEFINED +#define GUID_DEFINED + +#include "Types.h" + +typedef struct { + UInt32 Data1; + UInt16 Data2; + UInt16 Data3; + unsigned char Data4[8]; +} GUID; + +#ifdef __cplusplus +#define REFGUID const GUID & +#else +#define REFGUID const GUID * +#endif + +#define REFCLSID REFGUID +#define REFIID REFGUID + +#ifdef __cplusplus +inline bool operator==(REFGUID g1, REFGUID g2) +{ + for (int i = 0; i < (int)sizeof(g1); i++) + if (((unsigned char *)&g1)[i] != ((unsigned char *)&g2)[i]) + return false; + return true; +} +inline bool operator!=(REFGUID g1, REFGUID g2) { return !(g1 == g2); } +#endif + +#ifdef __cplusplus + #define MY_EXTERN_C extern "C" +#else + #define MY_EXTERN_C extern +#endif + +#endif // GUID_DEFINED + + +#ifdef DEFINE_GUID +#undef DEFINE_GUID +#endif + +#ifdef INITGUID + #define DEFINE_GUID(name, l, w1, w2, b1, b2, b3, b4, b5, b6, b7, b8) \ + MY_EXTERN_C const GUID name = { l, w1, w2, { b1, b2, b3, b4, b5, b6, b7, b8 } } +#else + #define DEFINE_GUID(name, l, w1, w2, b1, b2, b3, b4, b5, b6, b7, b8) \ + MY_EXTERN_C const GUID name +#endif diff --git a/lzma443/C/Common/MyInitGuid.h b/lzma443/C/Common/MyInitGuid.h new file mode 100755 index 0000000..5bdfeed --- /dev/null +++ b/lzma443/C/Common/MyInitGuid.h @@ -0,0 +1,13 @@ +// Common/MyInitGuid.h + +#ifndef __COMMON_MYINITGUID_H +#define __COMMON_MYINITGUID_H + +#ifdef _WIN32 +#include +#else +#define INITGUID +#include "MyGuidDef.h" +#endif + +#endif diff --git a/lzma443/C/Common/MyUnknown.h b/lzma443/C/Common/MyUnknown.h new file mode 100755 index 0000000..d28d854 --- /dev/null +++ b/lzma443/C/Common/MyUnknown.h @@ -0,0 +1,24 @@ +// MyUnknown.h + +#ifndef __MYUNKNOWN_H +#define __MYUNKNOWN_H + +#ifdef _WIN32 + +#ifdef _WIN32_WCE +#if (_WIN32_WCE > 300) +#include +#else +#define MIDL_INTERFACE(x) struct +#endif +#else +#include +#endif + +#include + +#else +#include "MyWindows.h" +#endif + +#endif diff --git a/lzma443/C/Common/MyWindows.h b/lzma443/C/Common/MyWindows.h new file mode 100755 index 0000000..a93d750 --- /dev/null +++ b/lzma443/C/Common/MyWindows.h @@ -0,0 +1,203 @@ +// MyWindows.h + +#ifndef __MYWINDOWS_H +#define __MYWINDOWS_H + +#ifdef _WIN32 + +#include + +#define CHAR_PATH_SEPARATOR '\\' +#define WCHAR_PATH_SEPARATOR L'\\' +#define STRING_PATH_SEPARATOR "\\" +#define WSTRING_PATH_SEPARATOR L"\\" + +#else + +#define CHAR_PATH_SEPARATOR '/' +#define WCHAR_PATH_SEPARATOR L'/' +#define STRING_PATH_SEPARATOR "/" +#define WSTRING_PATH_SEPARATOR L"/" + +#include // for wchar_t +#include + +#include "MyGuidDef.h" + +typedef char CHAR; +typedef unsigned char UCHAR; + +#undef BYTE +typedef unsigned char BYTE; + +typedef short SHORT; +typedef unsigned short USHORT; + +#undef WORD +typedef unsigned short WORD; +typedef short VARIANT_BOOL; + +typedef int INT; +typedef Int32 INT32; +typedef unsigned int UINT; +typedef UInt32 UINT32; +typedef INT32 LONG; // LONG, ULONG and DWORD must be 32-bit +typedef UINT32 ULONG; + +#undef DWORD +typedef UINT32 DWORD; + +typedef Int64 LONGLONG; +typedef UInt64 ULONGLONG; + +typedef struct LARGE_INTEGER { LONGLONG QuadPart; }LARGE_INTEGER; +typedef struct _ULARGE_INTEGER { ULONGLONG QuadPart;} ULARGE_INTEGER; + +typedef const CHAR *LPCSTR; +typedef CHAR TCHAR; +typedef const TCHAR *LPCTSTR; +typedef wchar_t WCHAR; +typedef WCHAR OLECHAR; +typedef const WCHAR *LPCWSTR; +typedef OLECHAR *BSTR; +typedef const OLECHAR *LPCOLESTR; +typedef OLECHAR *LPOLESTR; + +typedef struct _FILETIME +{ + DWORD dwLowDateTime; + DWORD dwHighDateTime; +}FILETIME; + +#define HRESULT LONG +#define FAILED(Status) ((HRESULT)(Status)<0) +typedef ULONG PROPID; +typedef LONG SCODE; + +#define S_OK ((HRESULT)0x00000000L) +#define S_FALSE ((HRESULT)0x00000001L) +#define E_NOTIMPL ((HRESULT)0x80004001L) +#define E_NOINTERFACE ((HRESULT)0x80004002L) +#define E_ABORT ((HRESULT)0x80004004L) +#define E_FAIL ((HRESULT)0x80004005L) +#define STG_E_INVALIDFUNCTION ((HRESULT)0x80030001L) +#define E_OUTOFMEMORY ((HRESULT)0x8007000EL) +#define E_INVALIDARG ((HRESULT)0x80070057L) + +#ifdef _MSC_VER +#define STDMETHODCALLTYPE __stdcall +#else +#define STDMETHODCALLTYPE +#endif + +#define STDMETHOD_(t, f) virtual t STDMETHODCALLTYPE f +#define STDMETHOD(f) STDMETHOD_(HRESULT, f) +#define STDMETHODIMP_(type) type STDMETHODCALLTYPE +#define STDMETHODIMP STDMETHODIMP_(HRESULT) + +#define PURE = 0 + +#define MIDL_INTERFACE(x) struct + +struct IUnknown +{ + STDMETHOD(QueryInterface) (REFIID iid, void **outObject) PURE; + STDMETHOD_(ULONG, AddRef)() PURE; + STDMETHOD_(ULONG, Release)() PURE; + #ifndef _WIN32 + virtual ~IUnknown() {} + #endif +}; + +typedef IUnknown *LPUNKNOWN; + +#define VARIANT_TRUE ((VARIANT_BOOL)-1) +#define VARIANT_FALSE ((VARIANT_BOOL)0) + +enum VARENUM +{ + VT_EMPTY = 0, + VT_NULL = 1, + VT_I2 = 2, + VT_I4 = 3, + VT_R4 = 4, + VT_R8 = 5, + VT_CY = 6, + VT_DATE = 7, + VT_BSTR = 8, + VT_DISPATCH = 9, + VT_ERROR = 10, + VT_BOOL = 11, + VT_VARIANT = 12, + VT_UNKNOWN = 13, + VT_DECIMAL = 14, + VT_I1 = 16, + VT_UI1 = 17, + VT_UI2 = 18, + VT_UI4 = 19, + VT_I8 = 20, + VT_UI8 = 21, + VT_INT = 22, + VT_UINT = 23, + VT_VOID = 24, + VT_HRESULT = 25, + VT_FILETIME = 64 +}; + +typedef unsigned short VARTYPE; +typedef WORD PROPVAR_PAD1; +typedef WORD PROPVAR_PAD2; +typedef WORD PROPVAR_PAD3; + +typedef struct tagPROPVARIANT +{ + VARTYPE vt; + PROPVAR_PAD1 wReserved1; + PROPVAR_PAD2 wReserved2; + PROPVAR_PAD3 wReserved3; + union + { + CHAR cVal; + UCHAR bVal; + SHORT iVal; + USHORT uiVal; + LONG lVal; + ULONG ulVal; + INT intVal; + UINT uintVal; + LARGE_INTEGER hVal; + ULARGE_INTEGER uhVal; + VARIANT_BOOL boolVal; + SCODE scode; + FILETIME filetime; + BSTR bstrVal; + }; +} PROPVARIANT; + +typedef PROPVARIANT tagVARIANT; +typedef tagVARIANT VARIANT; +typedef VARIANT VARIANTARG; + +MY_EXTERN_C BSTR SysAllocStringByteLen(LPCSTR psz, UINT len); +MY_EXTERN_C BSTR SysAllocString(const OLECHAR *sz); +MY_EXTERN_C void SysFreeString(BSTR bstr); +MY_EXTERN_C UINT SysStringByteLen(BSTR bstr); +MY_EXTERN_C UINT SysStringLen(BSTR bstr); + +MY_EXTERN_C DWORD GetLastError(); +MY_EXTERN_C HRESULT VariantClear(VARIANTARG *prop); +MY_EXTERN_C HRESULT VariantCopy(VARIANTARG *dest, VARIANTARG *src); +MY_EXTERN_C LONG CompareFileTime(const FILETIME* ft1, const FILETIME* ft2); + +#define CP_ACP 0 +#define CP_OEMCP 1 + +typedef enum tagSTREAM_SEEK +{ + STREAM_SEEK_SET = 0, + STREAM_SEEK_CUR = 1, + STREAM_SEEK_END = 2 +} STREAM_SEEK; + +#endif +#endif diff --git a/lzma443/C/Common/NewHandler.cpp b/lzma443/C/Common/NewHandler.cpp new file mode 100755 index 0000000..094eb64 --- /dev/null +++ b/lzma443/C/Common/NewHandler.cpp @@ -0,0 +1,116 @@ +// NewHandler.cpp + +#include "StdAfx.h" + +#include + +#include "NewHandler.h" + +// #define DEBUG_MEMORY_LEAK + +#ifndef DEBUG_MEMORY_LEAK + +#ifdef _WIN32 +void * +#ifdef _MSC_VER +__cdecl +#endif +operator new(size_t size) +{ + // void *p = ::HeapAlloc(::GetProcessHeap(), 0, size); + void *p = ::malloc(size); + if (p == 0) + throw CNewException(); + return p; +} + +void +#ifdef _MSC_VER +__cdecl +#endif +operator delete(void *p) throw() +{ + /* + if (p == 0) + return; + ::HeapFree(::GetProcessHeap(), 0, p); + */ + ::free(p); +} +#endif + +#else + +#pragma init_seg(lib) +const int kDebugSize = 1000000; +static void *a[kDebugSize]; +static int index = 0; + +static int numAllocs = 0; +void * __cdecl operator new(size_t size) +{ + numAllocs++; + void *p = HeapAlloc(GetProcessHeap(), 0, size); + if (index == 40) + { + int t = 1; + } + if (index < kDebugSize) + { + a[index] = p; + index++; + } + if (p == 0) + throw CNewException(); + printf("Alloc %6d, size = %8d\n", numAllocs, size); + return p; +} + +class CC +{ +public: + CC() + { + for (int i = 0; i < kDebugSize; i++) + a[i] = 0; + } + ~CC() + { + for (int i = 0; i < kDebugSize; i++) + if (a[i] != 0) + return; + } +} g_CC; + + +void __cdecl operator delete(void *p) +{ + if (p == 0) + return; + /* + for (int i = 0; i < index; i++) + if (a[i] == p) + a[i] = 0; + */ + HeapFree(GetProcessHeap(), 0, p); + numAllocs--; + printf("Free %d\n", numAllocs); +} + +#endif + +/* +int MemErrorVC(size_t) +{ + throw CNewException(); + // return 1; +} +CNewHandlerSetter::CNewHandlerSetter() +{ + // MemErrorOldVCFunction = _set_new_handler(MemErrorVC); +} +CNewHandlerSetter::~CNewHandlerSetter() +{ + // _set_new_handler(MemErrorOldVCFunction); +} +*/ diff --git a/lzma443/C/Common/NewHandler.h b/lzma443/C/Common/NewHandler.h new file mode 100755 index 0000000..0619fc6 --- /dev/null +++ b/lzma443/C/Common/NewHandler.h @@ -0,0 +1,16 @@ +// Common/NewHandler.h + +#ifndef __COMMON_NEWHANDLER_H +#define __COMMON_NEWHANDLER_H + +class CNewException {}; + +#ifdef _WIN32 +void +#ifdef _MSC_VER +__cdecl +#endif +operator delete(void *p) throw(); +#endif + +#endif diff --git a/lzma443/C/Common/StdAfx.h b/lzma443/C/Common/StdAfx.h new file mode 100755 index 0000000..681ee93 --- /dev/null +++ b/lzma443/C/Common/StdAfx.h @@ -0,0 +1,9 @@ +// StdAfx.h + +#ifndef __STDAFX_H +#define __STDAFX_H + +// #include "MyWindows.h" +#include "NewHandler.h" + +#endif diff --git a/lzma443/C/Common/String.cpp b/lzma443/C/Common/String.cpp new file mode 100755 index 0000000..7921fe4 --- /dev/null +++ b/lzma443/C/Common/String.cpp @@ -0,0 +1,198 @@ +// Common/String.cpp + +#include "StdAfx.h" + +#ifdef _WIN32 +#include "StringConvert.h" +#else +#include +#endif + +#include "Common/String.h" + + +#ifdef _WIN32 + +#ifndef _UNICODE + +wchar_t MyCharUpper(wchar_t c) +{ + if (c == 0) + return 0; + wchar_t *res = CharUpperW((LPWSTR)(unsigned int)c); + if (res != 0 || ::GetLastError() != ERROR_CALL_NOT_IMPLEMENTED) + return (wchar_t)(unsigned int)res; + const int kBufferSize = 4; + char s[kBufferSize + 1]; + int numChars = ::WideCharToMultiByte(CP_ACP, 0, &c, 1, s, kBufferSize, 0, 0); + if (numChars == 0 || numChars > kBufferSize) + return c; + s[numChars] = 0; + ::CharUpperA(s); + ::MultiByteToWideChar(CP_ACP, 0, s, numChars, &c, 1); + return c; +} + +wchar_t MyCharLower(wchar_t c) +{ + if (c == 0) + return 0; + wchar_t *res = CharLowerW((LPWSTR)(unsigned int)c); + if (res != 0 || ::GetLastError() != ERROR_CALL_NOT_IMPLEMENTED) + return (wchar_t)(unsigned int)res; + const int kBufferSize = 4; + char s[kBufferSize + 1]; + int numChars = ::WideCharToMultiByte(CP_ACP, 0, &c, 1, s, kBufferSize, 0, 0); + if (numChars == 0 || numChars > kBufferSize) + return c; + s[numChars] = 0; + ::CharLowerA(s); + ::MultiByteToWideChar(CP_ACP, 0, s, numChars, &c, 1); + return c; +} + +wchar_t * MyStringUpper(wchar_t *s) +{ + if (s == 0) + return 0; + wchar_t *res = CharUpperW(s); + if (res != 0 || ::GetLastError() != ERROR_CALL_NOT_IMPLEMENTED) + return res; + AString a = UnicodeStringToMultiByte(s); + a.MakeUpper(); + return MyStringCopy(s, (const wchar_t *)MultiByteToUnicodeString(a)); +} + +wchar_t * MyStringLower(wchar_t *s) +{ + if (s == 0) + return 0; + wchar_t *res = CharLowerW(s); + if (res != 0 || ::GetLastError() != ERROR_CALL_NOT_IMPLEMENTED) + return res; + AString a = UnicodeStringToMultiByte(s); + a.MakeLower(); + return MyStringCopy(s, (const wchar_t *)MultiByteToUnicodeString(a)); +} + +#endif + +/* +inline int ConvertCompareResult(int r) { return r - 2; } + +int MyStringCollate(const wchar_t *s1, const wchar_t *s2) +{ + int res = CompareStringW( + LOCALE_USER_DEFAULT, SORT_STRINGSORT, s1, -1, s2, -1); + #ifdef _UNICODE + return ConvertCompareResult(res); + #else + if (res != 0 || ::GetLastError() != ERROR_CALL_NOT_IMPLEMENTED) + return ConvertCompareResult(res); + return MyStringCollate(UnicodeStringToMultiByte(s1), + UnicodeStringToMultiByte(s2)); + #endif +} + +#ifndef _WIN32_WCE +int MyStringCollate(const char *s1, const char *s2) +{ + return ConvertCompareResult(CompareStringA( + LOCALE_USER_DEFAULT, SORT_STRINGSORT, s1, -1, s2, -1)); +} + +int MyStringCollateNoCase(const char *s1, const char *s2) +{ + return ConvertCompareResult(CompareStringA( + LOCALE_USER_DEFAULT, NORM_IGNORECASE | SORT_STRINGSORT, s1, -1, s2, -1)); +} +#endif + +int MyStringCollateNoCase(const wchar_t *s1, const wchar_t *s2) +{ + int res = CompareStringW( + LOCALE_USER_DEFAULT, NORM_IGNORECASE | SORT_STRINGSORT, s1, -1, s2, -1); + #ifdef _UNICODE + return ConvertCompareResult(res); + #else + if (res != 0 || ::GetLastError() != ERROR_CALL_NOT_IMPLEMENTED) + return ConvertCompareResult(res); + return MyStringCollateNoCase(UnicodeStringToMultiByte(s1), + UnicodeStringToMultiByte(s2)); + #endif +} +*/ + +#else + +wchar_t MyCharUpper(wchar_t c) +{ + return toupper(c); +} + +/* +int MyStringCollateNoCase(const wchar_t *s1, const wchar_t *s2) +{ + while (true) + { + wchar_t c1 = *s1++; + wchar_t c2 = *s2++; + wchar_t u1 = MyCharUpper(c1); + wchar_t u2 = MyCharUpper(c2); + + if (u1 < u2) return -1; + if (u1 > u2) return 1; + if (u1 == 0) return 0; + } +} +*/ + +#endif + +int MyStringCompare(const char *s1, const char *s2) +{ + while (true) + { + unsigned char c1 = (unsigned char)*s1++; + unsigned char c2 = (unsigned char)*s2++; + if (c1 < c2) return -1; + if (c1 > c2) return 1; + if (c1 == 0) return 0; + } +} + +int MyStringCompare(const wchar_t *s1, const wchar_t *s2) +{ + while (true) + { + wchar_t c1 = *s1++; + wchar_t c2 = *s2++; + if (c1 < c2) return -1; + if (c1 > c2) return 1; + if (c1 == 0) return 0; + } +} + +int MyStringCompareNoCase(const wchar_t *s1, const wchar_t *s2) +{ + while (true) + { + wchar_t c1 = *s1++; + wchar_t c2 = *s2++; + if (c1 != c2) + { + wchar_t u1 = MyCharUpper(c1); + wchar_t u2 = MyCharUpper(c2); + if (u1 < u2) return -1; + if (u1 > u2) return 1; + } + if (c1 == 0) return 0; + } +} + +#ifdef _WIN32 +int MyStringCompareNoCase(const char *s1, const char *s2) +{ + return MyStringCompareNoCase(MultiByteToUnicodeString(s1), MultiByteToUnicodeString(s2)); +} +#endif diff --git a/lzma443/C/Common/String.h b/lzma443/C/Common/String.h new file mode 100755 index 0000000..c4277c1 --- /dev/null +++ b/lzma443/C/Common/String.h @@ -0,0 +1,631 @@ +// Common/String.h + +#ifndef __COMMON_STRING_H +#define __COMMON_STRING_H + +#include +// #include + +#include "Vector.h" + +#ifdef _WIN32 +#include "MyWindows.h" +#endif + +static const char *kTrimDefaultCharSet = " \n\t"; + +template +inline int MyStringLen(const T *s) +{ + int i; + for (i = 0; s[i] != '\0'; i++); + return i; +} + +template +inline T * MyStringCopy(T *dest, const T *src) +{ + T *destStart = dest; + while((*dest++ = *src++) != 0); + return destStart; +} + +inline wchar_t* MyStringGetNextCharPointer(wchar_t *p) + { return (p + 1); } +inline const wchar_t* MyStringGetNextCharPointer(const wchar_t *p) + { return (p + 1); } +inline wchar_t* MyStringGetPrevCharPointer(const wchar_t *, wchar_t *p) + { return (p - 1); } +inline const wchar_t* MyStringGetPrevCharPointer(const wchar_t *, const wchar_t *p) + { return (p - 1); } + +#ifdef _WIN32 + +inline char* MyStringGetNextCharPointer(char *p) + { return CharNextA(p); } +inline const char* MyStringGetNextCharPointer(const char *p) + { return CharNextA(p); } + +inline char* MyStringGetPrevCharPointer(char *base, char *p) + { return CharPrevA(base, p); } +inline const char* MyStringGetPrevCharPointer(const char *base, const char *p) + { return CharPrevA(base, p); } + +inline char MyCharUpper(char c) + { return (char)(unsigned int)CharUpperA((LPSTR)(unsigned int)(unsigned char)c); } +#ifdef _UNICODE +inline wchar_t MyCharUpper(wchar_t c) + { return (wchar_t)CharUpperW((LPWSTR)c); } +#else +wchar_t MyCharUpper(wchar_t c); +#endif + +inline char MyCharLower(char c) + { return (char)(unsigned int)CharLowerA((LPSTR)(unsigned int)(unsigned char)c); } +#ifdef _UNICODE +inline wchar_t MyCharLower(wchar_t c) + { return (wchar_t)CharLowerW((LPWSTR)c); } +#else +wchar_t MyCharLower(wchar_t c); +#endif + +inline char * MyStringUpper(char *s) { return CharUpperA(s); } +#ifdef _UNICODE +inline wchar_t * MyStringUpper(wchar_t *s) { return CharUpperW(s); } +#else +wchar_t * MyStringUpper(wchar_t *s); +#endif + +inline char * MyStringLower(char *s) { return CharLowerA(s); } +#ifdef _UNICODE +inline wchar_t * MyStringLower(wchar_t *s) { return CharLowerW(s); } +#else +wchar_t * MyStringLower(wchar_t *s); +#endif + +#else // Standard-C +wchar_t MyCharUpper(wchar_t c); +#endif + +////////////////////////////////////// +// Compare + +/* +#ifndef _WIN32_WCE +int MyStringCollate(const char *s1, const char *s2); +int MyStringCollateNoCase(const char *s1, const char *s2); +#endif +int MyStringCollate(const wchar_t *s1, const wchar_t *s2); +int MyStringCollateNoCase(const wchar_t *s1, const wchar_t *s2); +*/ + +int MyStringCompare(const char *s1, const char *s2); +int MyStringCompare(const wchar_t *s1, const wchar_t *s2); + +#ifdef _WIN32 +int MyStringCompareNoCase(const char *s1, const char *s2); +#endif + +int MyStringCompareNoCase(const wchar_t *s1, const wchar_t *s2); + +template +class CStringBase +{ + void TrimLeftWithCharSet(const CStringBase &charSet) + { + const T *p = _chars; + while (charSet.Find(*p) >= 0 && (*p != 0)) + p = GetNextCharPointer(p); + Delete(0, (int)(p - _chars)); + } + void TrimRightWithCharSet(const CStringBase &charSet) + { + const T *p = _chars; + const T *pLast = NULL; + while (*p != 0) + { + if (charSet.Find(*p) >= 0) + { + if (pLast == NULL) + pLast = p; + } + else + pLast = NULL; + p = GetNextCharPointer(p); + } + if(pLast != NULL) + { + int i = (int)(pLast - _chars); + Delete(i, _length - i); + } + + } + void MoveItems(int destIndex, int srcIndex) + { + memmove(_chars + destIndex, _chars + srcIndex, + sizeof(T) * (_length - srcIndex + 1)); + } + + void InsertSpace(int &index, int size) + { + CorrectIndex(index); + GrowLength(size); + MoveItems(index + size, index); + } + + static T *GetNextCharPointer(T *p) + { return MyStringGetNextCharPointer(p); } + static const T *GetNextCharPointer(const T *p) + { return MyStringGetNextCharPointer(p); } + static T *GetPrevCharPointer(T *base, T *p) + { return MyStringGetPrevCharPointer(base, p); } + static const T *GetPrevCharPointer(const T *base, const T *p) + { return MyStringGetPrevCharPointer(base, p); } +protected: + T *_chars; + int _length; + int _capacity; + + void SetCapacity(int newCapacity) + { + int realCapacity = newCapacity + 1; + if(realCapacity == _capacity) + return; + /* + const int kMaxStringSize = 0x20000000; + #ifndef _WIN32_WCE + if(newCapacity > kMaxStringSize || newCapacity < _length) + throw 1052337; + #endif + */ + T *newBuffer = new T[realCapacity]; + if(_capacity > 0) + { + for (int i = 0; i < (_length + 1); i++) + newBuffer[i] = _chars[i]; + delete []_chars; + _chars = newBuffer; + } + else + { + _chars = newBuffer; + _chars[0] = 0; + } + _capacity = realCapacity; + } + + void GrowLength(int n) + { + int freeSize = _capacity - _length - 1; + if (n <= freeSize) + return; + int delta; + if (_capacity > 64) + delta = _capacity / 2; + else if (_capacity > 8) + delta = 16; + else + delta = 4; + if (freeSize + delta < n) + delta = n - freeSize; + SetCapacity(_capacity + delta); + } + + void CorrectIndex(int &index) const + { + if (index > _length) + index = _length; + } + +public: + CStringBase(): _chars(0), _length(0), _capacity(0) + { SetCapacity(16 - 1); } + CStringBase(T c): _chars(0), _length(0), _capacity(0) + { + SetCapacity(1); + _chars[0] = c; + _chars[1] = 0; + _length = 1; + } + CStringBase(const T *chars): _chars(0), _length(0), _capacity(0) + { + int length = MyStringLen(chars); + SetCapacity(length); + MyStringCopy(_chars, chars); // can be optimized by memove() + _length = length; + } + CStringBase(const CStringBase &s): _chars(0), _length(0), _capacity(0) + { + SetCapacity(s._length); + MyStringCopy(_chars, s._chars); + _length = s._length; + } + ~CStringBase() { delete []_chars; } + + operator const T*() const { return _chars;} + + // The minimum size of the character buffer in characters. + // This value does not include space for a null terminator. + T* GetBuffer(int minBufLength) + { + if(minBufLength >= _capacity) + SetCapacity(minBufLength + 1); + return _chars; + } + void ReleaseBuffer() { ReleaseBuffer(MyStringLen(_chars)); } + void ReleaseBuffer(int newLength) + { + /* + #ifndef _WIN32_WCE + if(newLength >= _capacity) + throw 282217; + #endif + */ + _chars[newLength] = 0; + _length = newLength; + } + + CStringBase& operator=(T c) + { + Empty(); + SetCapacity(1); + _chars[0] = c; + _chars[1] = 0; + _length = 1; + return *this; + } + CStringBase& operator=(const T *chars) + { + Empty(); + int length = MyStringLen(chars); + SetCapacity(length); + MyStringCopy(_chars, chars); + _length = length; + return *this; + } + CStringBase& operator=(const CStringBase& s) + { + if(&s == this) + return *this; + Empty(); + SetCapacity(s._length); + MyStringCopy(_chars, s._chars); + _length = s._length; + return *this; + } + + CStringBase& operator+=(T c) + { + GrowLength(1); + _chars[_length] = c; + _chars[++_length] = 0; + return *this; + } + CStringBase& operator+=(const T *s) + { + int len = MyStringLen(s); + GrowLength(len); + MyStringCopy(_chars + _length, s); + _length += len; + return *this; + } + CStringBase& operator+=(const CStringBase &s) + { + GrowLength(s._length); + MyStringCopy(_chars + _length, s._chars); + _length += s._length; + return *this; + } + void Empty() + { + _length = 0; + _chars[0] = 0; + } + int Length() const { return _length; } + bool IsEmpty() const { return (_length == 0); } + + CStringBase Mid(int startIndex) const + { return Mid(startIndex, _length - startIndex); } + CStringBase Mid(int startIndex, int count ) const + { + if (startIndex + count > _length) + count = _length - startIndex; + + if (startIndex == 0 && startIndex + count == _length) + return *this; + + CStringBase result; + result.SetCapacity(count); + // MyStringNCopy(result._chars, _chars + startIndex, count); + for (int i = 0; i < count; i++) + result._chars[i] = _chars[startIndex + i]; + result._chars[count] = 0; + result._length = count; + return result; + } + CStringBase Left(int count) const + { return Mid(0, count); } + CStringBase Right(int count) const + { + if (count > _length) + count = _length; + return Mid(_length - count, count); + } + + void MakeUpper() + { MyStringUpper(_chars); } + void MakeLower() + { MyStringLower(_chars); } + + int Compare(const CStringBase& s) const + { return MyStringCompare(_chars, s._chars); } + + int CompareNoCase(const CStringBase& s) const + { return MyStringCompareNoCase(_chars, s._chars); } + /* + int Collate(const CStringBase& s) const + { return MyStringCollate(_chars, s._chars); } + int CollateNoCase(const CStringBase& s) const + { return MyStringCollateNoCase(_chars, s._chars); } + */ + + int Find(T c) const { return Find(c, 0); } + int Find(T c, int startIndex) const + { + T *p = _chars + startIndex; + while (true) + { + if (*p == c) + return (int)(p - _chars); + if (*p == 0) + return -1; + p = GetNextCharPointer(p); + } + } + int Find(const CStringBase &s) const { return Find(s, 0); } + int Find(const CStringBase &s, int startIndex) const + { + if (s.IsEmpty()) + return startIndex; + for (; startIndex < _length; startIndex++) + { + int j; + for (j = 0; j < s._length && startIndex + j < _length; j++) + if (_chars[startIndex+j] != s._chars[j]) + break; + if (j == s._length) + return startIndex; + } + return -1; + } + int ReverseFind(T c) const + { + if (_length == 0) + return -1; + T *p = _chars + _length - 1; + while (true) + { + if (*p == c) + return (int)(p - _chars); + if (p == _chars) + return -1; + p = GetPrevCharPointer(_chars, p); + } + } + int FindOneOf(const CStringBase &s) const + { + for(int i = 0; i < _length; i++) + if (s.Find(_chars[i]) >= 0) + return i; + return -1; + } + + void TrimLeft(T c) + { + const T *p = _chars; + while (c == *p) + p = GetNextCharPointer(p); + Delete(0, p - _chars); + } + private: + CStringBase GetTrimDefaultCharSet() + { + CStringBase charSet; + for(int i = 0; i < (int)(sizeof(kTrimDefaultCharSet) / + sizeof(kTrimDefaultCharSet[0])); i++) + charSet += (T)kTrimDefaultCharSet[i]; + return charSet; + } + public: + + void TrimLeft() + { + TrimLeftWithCharSet(GetTrimDefaultCharSet()); + } + void TrimRight() + { + TrimRightWithCharSet(GetTrimDefaultCharSet()); + } + void TrimRight(T c) + { + const T *p = _chars; + const T *pLast = NULL; + while (*p != 0) + { + if (*p == c) + { + if (pLast == NULL) + pLast = p; + } + else + pLast = NULL; + p = GetNextCharPointer(p); + } + if(pLast != NULL) + { + int i = pLast - _chars; + Delete(i, _length - i); + } + } + void Trim() + { + TrimRight(); + TrimLeft(); + } + + int Insert(int index, T c) + { + InsertSpace(index, 1); + _chars[index] = c; + _length++; + return _length; + } + int Insert(int index, const CStringBase &s) + { + CorrectIndex(index); + if (s.IsEmpty()) + return _length; + int numInsertChars = s.Length(); + InsertSpace(index, numInsertChars); + for(int i = 0; i < numInsertChars; i++) + _chars[index + i] = s[i]; + _length += numInsertChars; + return _length; + } + + // !!!!!!!!!!!!!!! test it if newChar = '\0' + int Replace(T oldChar, T newChar) + { + if (oldChar == newChar) + return 0; + int number = 0; + int pos = 0; + while (pos < Length()) + { + pos = Find(oldChar, pos); + if (pos < 0) + break; + _chars[pos] = newChar; + pos++; + number++; + } + return number; + } + int Replace(const CStringBase &oldString, const CStringBase &newString) + { + if (oldString.IsEmpty()) + return 0; + if (oldString == newString) + return 0; + int oldStringLength = oldString.Length(); + int newStringLength = newString.Length(); + int number = 0; + int pos = 0; + while (pos < _length) + { + pos = Find(oldString, pos); + if (pos < 0) + break; + Delete(pos, oldStringLength); + Insert(pos, newString); + pos += newStringLength; + number++; + } + return number; + } + int Delete(int index, int count = 1 ) + { + if (index + count > _length) + count = _length - index; + if (count > 0) + { + MoveItems(index, index + count); + _length -= count; + } + return _length; + } +}; + +template +CStringBase operator+(const CStringBase& s1, const CStringBase& s2) +{ + CStringBase result(s1); + result += s2; + return result; +} + +template +CStringBase operator+(const CStringBase& s, T c) +{ + CStringBase result(s); + result += c; + return result; +} + +template +CStringBase operator+(T c, const CStringBase& s) +{ + CStringBase result(c); + result += s; + return result; +} + +template +CStringBase operator+(const CStringBase& s, const T * chars) +{ + CStringBase result(s); + result += chars; + return result; +} + +template +CStringBase operator+(const T * chars, const CStringBase& s) +{ + CStringBase result(chars); + result += s; + return result; +} + +template +bool operator==(const CStringBase& s1, const CStringBase& s2) + { return (s1.Compare(s2) == 0); } + +template +bool operator<(const CStringBase& s1, const CStringBase& s2) + { return (s1.Compare(s2) < 0); } + +template +bool operator==(const T *s1, const CStringBase& s2) + { return (s2.Compare(s1) == 0); } + +template +bool operator==(const CStringBase& s1, const T *s2) + { return (s1.Compare(s2) == 0); } + +template +bool operator!=(const CStringBase& s1, const CStringBase& s2) + { return (s1.Compare(s2) != 0); } + +template +bool operator!=(const T *s1, const CStringBase& s2) + { return (s2.Compare(s1) != 0); } + +template +bool operator!=(const CStringBase& s1, const T *s2) + { return (s1.Compare(s2) != 0); } + +typedef CStringBase AString; +typedef CStringBase UString; + +typedef CObjectVector AStringVector; +typedef CObjectVector UStringVector; + +#ifdef _UNICODE + typedef UString CSysString; +#else + typedef AString CSysString; +#endif + +typedef CObjectVector CSysStringVector; + +#endif diff --git a/lzma443/C/Common/StringConvert.cpp b/lzma443/C/Common/StringConvert.cpp new file mode 100755 index 0000000..d6d05d6 --- /dev/null +++ b/lzma443/C/Common/StringConvert.cpp @@ -0,0 +1,93 @@ +// Common/StringConvert.cpp + +#include "StdAfx.h" + +#include "StringConvert.h" + +#ifndef _WIN32 +#include +#endif + +#ifdef _WIN32 +UString MultiByteToUnicodeString(const AString &srcString, UINT codePage) +{ + UString resultString; + if(!srcString.IsEmpty()) + { + int numChars = MultiByteToWideChar(codePage, 0, srcString, + srcString.Length(), resultString.GetBuffer(srcString.Length()), + srcString.Length() + 1); + #ifndef _WIN32_WCE + if(numChars == 0) + throw 282228; + #endif + resultString.ReleaseBuffer(numChars); + } + return resultString; +} + +AString UnicodeStringToMultiByte(const UString &srcString, UINT codePage) +{ + AString resultString; + if(!srcString.IsEmpty()) + { + int numRequiredBytes = srcString.Length() * 2; + int numChars = WideCharToMultiByte(codePage, 0, srcString, + srcString.Length(), resultString.GetBuffer(numRequiredBytes), + numRequiredBytes + 1, NULL, NULL); + #ifndef _WIN32_WCE + if(numChars == 0) + throw 282229; + #endif + resultString.ReleaseBuffer(numChars); + } + return resultString; +} + +#ifndef _WIN32_WCE +AString SystemStringToOemString(const CSysString &srcString) +{ + AString result; + CharToOem(srcString, result.GetBuffer(srcString.Length() * 2)); + result.ReleaseBuffer(); + return result; +} +#endif + +#else + +UString MultiByteToUnicodeString(const AString &srcString, UINT codePage) +{ + UString resultString; + for (int i = 0; i < srcString.Length(); i++) + resultString += wchar_t(srcString[i]); + /* + if(!srcString.IsEmpty()) + { + int numChars = mbstowcs(resultString.GetBuffer(srcString.Length()), srcString, srcString.Length() + 1); + if (numChars < 0) throw "Your environment does not support UNICODE"; + resultString.ReleaseBuffer(numChars); + } + */ + return resultString; +} + +AString UnicodeStringToMultiByte(const UString &srcString, UINT codePage) +{ + AString resultString; + for (int i = 0; i < srcString.Length(); i++) + resultString += char(srcString[i]); + /* + if(!srcString.IsEmpty()) + { + int numRequiredBytes = srcString.Length() * 6 + 1; + int numChars = wcstombs(resultString.GetBuffer(numRequiredBytes), srcString, numRequiredBytes); + if (numChars < 0) throw "Your environment does not support UNICODE"; + resultString.ReleaseBuffer(numChars); + } + */ + return resultString; +} + +#endif + diff --git a/lzma443/C/Common/StringConvert.h b/lzma443/C/Common/StringConvert.h new file mode 100755 index 0000000..648ca02 --- /dev/null +++ b/lzma443/C/Common/StringConvert.h @@ -0,0 +1,71 @@ +// Common/StringConvert.h + +#ifndef __COMMON_STRINGCONVERT_H +#define __COMMON_STRINGCONVERT_H + +#include "MyWindows.h" +#include "Common/String.h" +#include "Types.h" + +UString MultiByteToUnicodeString(const AString &srcString, UINT codePage = CP_ACP); +AString UnicodeStringToMultiByte(const UString &srcString, UINT codePage = CP_ACP); + +inline const wchar_t* GetUnicodeString(const wchar_t* unicodeString) + { return unicodeString; } +inline const UString& GetUnicodeString(const UString &unicodeString) + { return unicodeString; } +inline UString GetUnicodeString(const AString &ansiString) + { return MultiByteToUnicodeString(ansiString); } +inline UString GetUnicodeString(const AString &multiByteString, UINT codePage) + { return MultiByteToUnicodeString(multiByteString, codePage); } +inline const wchar_t* GetUnicodeString(const wchar_t* unicodeString, UINT) + { return unicodeString; } +inline const UString& GetUnicodeString(const UString &unicodeString, UINT) + { return unicodeString; } + +inline const char* GetAnsiString(const char* ansiString) + { return ansiString; } +inline const AString& GetAnsiString(const AString &ansiString) + { return ansiString; } +inline AString GetAnsiString(const UString &unicodeString) + { return UnicodeStringToMultiByte(unicodeString); } + +inline const char* GetOemString(const char* oemString) + { return oemString; } +inline const AString& GetOemString(const AString &oemString) + { return oemString; } +inline AString GetOemString(const UString &unicodeString) + { return UnicodeStringToMultiByte(unicodeString, CP_OEMCP); } + + +#ifdef _UNICODE + inline const wchar_t* GetSystemString(const wchar_t* unicodeString) + { return unicodeString;} + inline const UString& GetSystemString(const UString &unicodeString) + { return unicodeString;} + inline const wchar_t* GetSystemString(const wchar_t* unicodeString, UINT codePage) + { return unicodeString;} + inline const UString& GetSystemString(const UString &unicodeString, UINT codePage) + { return unicodeString;} + inline UString GetSystemString(const AString &multiByteString, UINT codePage) + { return MultiByteToUnicodeString(multiByteString, codePage);} + inline UString GetSystemString(const AString &multiByteString) + { return MultiByteToUnicodeString(multiByteString);} +#else + inline const char* GetSystemString(const char *ansiString) + { return ansiString; } + inline const AString& GetSystemString(const AString &multiByteString, UINT) + { return multiByteString; } + inline const char * GetSystemString(const char *multiByteString, UINT) + { return multiByteString; } + inline AString GetSystemString(const UString &unicodeString) + { return UnicodeStringToMultiByte(unicodeString); } + inline AString GetSystemString(const UString &unicodeString, UINT codePage) + { return UnicodeStringToMultiByte(unicodeString, codePage); } +#endif + +#ifndef _WIN32_WCE +AString SystemStringToOemString(const CSysString &srcString); +#endif + +#endif diff --git a/lzma443/C/Common/StringToInt.cpp b/lzma443/C/Common/StringToInt.cpp new file mode 100755 index 0000000..1fa8ef2 --- /dev/null +++ b/lzma443/C/Common/StringToInt.cpp @@ -0,0 +1,68 @@ +// Common/StringToInt.cpp + +#include "StdAfx.h" + +#include "StringToInt.h" + +UInt64 ConvertStringToUInt64(const char *s, const char **end) +{ + UInt64 result = 0; + while(true) + { + char c = *s; + if (c < '0' || c > '9') + { + if (end != NULL) + *end = s; + return result; + } + result *= 10; + result += (c - '0'); + s++; + } +} + +UInt64 ConvertOctStringToUInt64(const char *s, const char **end) +{ + UInt64 result = 0; + while(true) + { + char c = *s; + if (c < '0' || c > '7') + { + if (end != NULL) + *end = s; + return result; + } + result <<= 3; + result += (c - '0'); + s++; + } +} + + +UInt64 ConvertStringToUInt64(const wchar_t *s, const wchar_t **end) +{ + UInt64 result = 0; + while(true) + { + wchar_t c = *s; + if (c < '0' || c > '9') + { + if (end != NULL) + *end = s; + return result; + } + result *= 10; + result += (c - '0'); + s++; + } +} + + +Int64 ConvertStringToInt64(const char *s, const char **end) +{ + if (*s == '-') + return -(Int64)ConvertStringToUInt64(s + 1, end); + return ConvertStringToUInt64(s, end); +} diff --git a/lzma443/C/Common/StringToInt.h b/lzma443/C/Common/StringToInt.h new file mode 100755 index 0000000..bb971f6 --- /dev/null +++ b/lzma443/C/Common/StringToInt.h @@ -0,0 +1,17 @@ +// Common/StringToInt.h + +#ifndef __COMMON_STRINGTOINT_H +#define __COMMON_STRINGTOINT_H + +#include +#include "Types.h" + +UInt64 ConvertStringToUInt64(const char *s, const char **end); +UInt64 ConvertOctStringToUInt64(const char *s, const char **end); +UInt64 ConvertStringToUInt64(const wchar_t *s, const wchar_t **end); + +Int64 ConvertStringToInt64(const char *s, const char **end); + +#endif + + diff --git a/lzma443/C/Common/Types.h b/lzma443/C/Common/Types.h new file mode 100755 index 0000000..41d785e --- /dev/null +++ b/lzma443/C/Common/Types.h @@ -0,0 +1,57 @@ +// Common/Types.h + +#ifndef __COMMON_TYPES_H +#define __COMMON_TYPES_H + +#ifndef _7ZIP_BYTE_DEFINED +#define _7ZIP_BYTE_DEFINED +typedef unsigned char Byte; +#endif + +#ifndef _7ZIP_INT16_DEFINED +#define _7ZIP_INT16_DEFINED +typedef short Int16; +#endif + +#ifndef _7ZIP_UINT16_DEFINED +#define _7ZIP_UINT16_DEFINED +typedef unsigned short UInt16; +#endif + +#ifndef _7ZIP_INT32_DEFINED +#define _7ZIP_INT32_DEFINED +typedef int Int32; +#endif + +#ifndef _7ZIP_UINT32_DEFINED +#define _7ZIP_UINT32_DEFINED +typedef unsigned int UInt32; +#endif + +#ifdef _MSC_VER + +#ifndef _7ZIP_INT64_DEFINED +#define _7ZIP_INT64_DEFINED +typedef __int64 Int64; +#endif + +#ifndef _7ZIP_UINT64_DEFINED +#define _7ZIP_UINT64_DEFINED +typedef unsigned __int64 UInt64; +#endif + +#else + +#ifndef _7ZIP_INT64_DEFINED +#define _7ZIP_INT64_DEFINED +typedef long long int Int64; +#endif + +#ifndef _7ZIP_UINT64_DEFINED +#define _7ZIP_UINT64_DEFINED +typedef unsigned long long int UInt64; +#endif + +#endif + +#endif diff --git a/lzma443/C/Common/Vector.cpp b/lzma443/C/Common/Vector.cpp new file mode 100755 index 0000000..cb3d875 --- /dev/null +++ b/lzma443/C/Common/Vector.cpp @@ -0,0 +1,74 @@ +// Common/Vector.cpp + +#include "StdAfx.h" + +#include + +#include "Vector.h" + +CBaseRecordVector::~CBaseRecordVector() + { delete []((unsigned char *)_items); } +void CBaseRecordVector::Clear() + { DeleteFrom(0); } +void CBaseRecordVector::DeleteBack() + { Delete(_size - 1); } +void CBaseRecordVector::DeleteFrom(int index) + { Delete(index, _size - index); } + +void CBaseRecordVector::ReserveOnePosition() +{ + if(_size != _capacity) + return; + int delta; + if (_capacity > 64) + delta = _capacity / 2; + else if (_capacity > 8) + delta = 8; + else + delta = 4; + Reserve(_capacity + delta); +} + +void CBaseRecordVector::Reserve(int newCapacity) +{ + if(newCapacity <= _capacity) + return; + /* + #ifndef _DEBUG + static const unsigned int kMaxVectorSize = 0xF0000000; + if(newCapacity < _size || + ((unsigned int )newCapacity * (unsigned int )_itemSize) > kMaxVectorSize) + throw 1052354; + #endif + */ + unsigned char *p = new unsigned char[newCapacity * _itemSize]; + int numRecordsToMove = _capacity; + memmove(p, _items, _itemSize * numRecordsToMove); + delete [](unsigned char *)_items; + _items = p; + _capacity = newCapacity; +} + +void CBaseRecordVector::MoveItems(int destIndex, int srcIndex) +{ + memmove(((unsigned char *)_items) + destIndex * _itemSize, + ((unsigned char *)_items) + srcIndex * _itemSize, + _itemSize * (_size - srcIndex)); +} + +void CBaseRecordVector::InsertOneItem(int index) +{ + ReserveOnePosition(); + MoveItems(index + 1, index); + _size++; +} + +void CBaseRecordVector::Delete(int index, int num) +{ + TestIndexAndCorrectNum(index, num); + if (num > 0) + { + MoveItems(index, index + num); + _size -= num; + } +} diff --git a/lzma443/C/Common/Vector.h b/lzma443/C/Common/Vector.h new file mode 100755 index 0000000..210c385 --- /dev/null +++ b/lzma443/C/Common/Vector.h @@ -0,0 +1,228 @@ +// Common/Vector.h + +#ifndef __COMMON_VECTOR_H +#define __COMMON_VECTOR_H + +#include "Defs.h" + +class CBaseRecordVector +{ + void MoveItems(int destIndex, int srcIndex); +protected: + int _capacity; + int _size; + void *_items; + size_t _itemSize; + + void ReserveOnePosition(); + void InsertOneItem(int index); + void TestIndexAndCorrectNum(int index, int &num) const + { if (index + num > _size) num = _size - index; } +public: + CBaseRecordVector(size_t itemSize): + _capacity(0), _size(0), _items(0), _itemSize(itemSize) {} + virtual ~CBaseRecordVector(); + int Size() const { return _size; } + bool IsEmpty() const { return (_size == 0); } + void Reserve(int newCapacity); + virtual void Delete(int index, int num = 1); + void Clear(); + void DeleteFrom(int index); + void DeleteBack(); +}; + +template +class CRecordVector: public CBaseRecordVector +{ +public: + CRecordVector():CBaseRecordVector(sizeof(T)){}; + CRecordVector(const CRecordVector &v): + CBaseRecordVector(sizeof(T)) { *this = v;} + CRecordVector& operator=(const CRecordVector &v) + { + Clear(); + return (*this += v); + } + CRecordVector& operator+=(const CRecordVector &v) + { + int size = v.Size(); + Reserve(Size() + size); + for(int i = 0; i < size; i++) + Add(v[i]); + return *this; + } + int Add(T item) + { + ReserveOnePosition(); + ((T *)_items)[_size] = item; + return _size++; + } + void Insert(int index, T item) + { + InsertOneItem(index); + ((T *)_items)[index] = item; + } + // T* GetPointer() const { return (T*)_items; } + // operator const T *() const { return _items; }; + const T& operator[](int index) const { return ((T *)_items)[index]; } + T& operator[](int index) { return ((T *)_items)[index]; } + const T& Front() const { return operator[](0); } + T& Front() { return operator[](0); } + const T& Back() const { return operator[](_size - 1); } + T& Back() { return operator[](_size - 1); } + + void Swap(int i, int j) + { + T temp = operator[](i); + operator[](i) = operator[](j); + operator[](j) = temp; + } + + int FindInSorted(const T& item) const + { + int left = 0, right = Size(); + while (left != right) + { + int mid = (left + right) / 2; + const T& midValue = (*this)[mid]; + if (item == midValue) + return mid; + if (item < midValue) + right = mid; + else + left = mid + 1; + } + return -1; + } + + void Sort(int left, int right) + { + if (right - left < 2) + return; + Swap(left, (left + right) / 2); + int last = left; + for (int i = left; i < right; i++) + if (operator[](i) < operator[](left)) + Swap(++last, i); + Swap(left, last); + Sort(left, last); + Sort(last + 1, right); + } + void Sort() { Sort(0, Size()); } + void Sort(int left, int right, int (*compare)(const T*, const T*, void *), void *param) + { + if (right - left < 2) + return; + Swap(left, (left + right) / 2); + int last = left; + for (int i = left; i < right; i++) + if (compare(&operator[](i), &operator[](left), param) < 0) + Swap(++last, i); + Swap(left, last); + Sort(left, last, compare, param); + Sort(last + 1, right, compare, param); + } + + void Sort(int (*compare)(const T*, const T*, void *), void *param) + { + Sort(0, Size(), compare, param); + } +}; + +typedef CRecordVector CIntVector; +typedef CRecordVector CUIntVector; +typedef CRecordVector CBoolVector; +typedef CRecordVector CByteVector; +typedef CRecordVector CPointerVector; + +template +class CObjectVector: public CPointerVector +{ +public: + CObjectVector(){}; + ~CObjectVector() { Clear(); } + CObjectVector(const CObjectVector &objectVector) + { *this = objectVector; } + CObjectVector& operator=(const CObjectVector &objectVector) + { + Clear(); + return (*this += objectVector); + } + CObjectVector& operator+=(const CObjectVector &objectVector) + { + int size = objectVector.Size(); + Reserve(Size() + size); + for(int i = 0; i < size; i++) + Add(objectVector[i]); + return *this; + } + const T& operator[](int index) const { return *((T *)CPointerVector::operator[](index)); } + T& operator[](int index) { return *((T *)CPointerVector::operator[](index)); } + T& Front() { return operator[](0); } + const T& Front() const { return operator[](0); } + T& Back() { return operator[](_size - 1); } + const T& Back() const { return operator[](_size - 1); } + int Add(const T& item) + { return CPointerVector::Add(new T(item)); } + void Insert(int index, const T& item) + { CPointerVector::Insert(index, new T(item)); } + virtual void Delete(int index, int num = 1) + { + TestIndexAndCorrectNum(index, num); + for(int i = 0; i < num; i++) + delete (T *)(((void **)_items)[index + i]); + CPointerVector::Delete(index, num); + } + int Find(const T& item) const + { + for(int i = 0; i < Size(); i++) + if (item == (*this)[i]) + return i; + return -1; + } + int FindInSorted(const T& item) const + { + int left = 0, right = Size(); + while (left != right) + { + int mid = (left + right) / 2; + const T& midValue = (*this)[mid]; + if (item == midValue) + return mid; + if (item < midValue) + right = mid; + else + left = mid + 1; + } + return -1; + } + int AddToSorted(const T& item) + { + int left = 0, right = Size(); + while (left != right) + { + int mid = (left + right) / 2; + const T& midValue = (*this)[mid]; + if (item == midValue) + { + right = mid + 1; + break; + } + if (item < midValue) + right = mid; + else + left = mid + 1; + } + Insert(right, item); + return right; + } + + void Sort(int (*compare)(void *const *, void *const *, void *), void *param) + { CPointerVector::Sort(compare, param); } + + static int CompareObjectItems(void *const *a1, void *const *a2, void *param) + { return MyCompare(*(*((const T **)a1)), *(*((const T **)a2))); } + void Sort() { CPointerVector::Sort(CompareObjectItems, 0); } +}; + +#endif diff --git a/lzma443/C/Windows/Defs.h b/lzma443/C/Windows/Defs.h new file mode 100755 index 0000000..c0038d6 --- /dev/null +++ b/lzma443/C/Windows/Defs.h @@ -0,0 +1,18 @@ +// Windows/Defs.h + +#ifndef __WINDOWS_DEFS_H +#define __WINDOWS_DEFS_H + +inline bool BOOLToBool(BOOL value) + { return (value != FALSE); } + +inline BOOL BoolToBOOL(bool value) + { return (value ? TRUE: FALSE); } + +inline VARIANT_BOOL BoolToVARIANT_BOOL(bool value) + { return (value ? VARIANT_TRUE: VARIANT_FALSE); } + +inline bool VARIANT_BOOLToBool(VARIANT_BOOL value) + { return (value != VARIANT_FALSE); } + +#endif diff --git a/lzma443/C/Windows/FileIO.cpp b/lzma443/C/Windows/FileIO.cpp new file mode 100755 index 0000000..203de84 --- /dev/null +++ b/lzma443/C/Windows/FileIO.cpp @@ -0,0 +1,245 @@ +// Windows/FileIO.cpp + +#include "StdAfx.h" + +#include "FileIO.h" +#include "Defs.h" +#ifndef _UNICODE +#include "../Common/StringConvert.h" +#endif + +#ifndef _UNICODE +extern bool g_IsNT; +#endif + +namespace NWindows { +namespace NFile { +namespace NIO { + +CFileBase::~CFileBase() { Close(); } + +bool CFileBase::Create(LPCTSTR fileName, DWORD desiredAccess, + DWORD shareMode, DWORD creationDisposition, DWORD flagsAndAttributes) +{ + Close(); + _handle = ::CreateFile(fileName, desiredAccess, shareMode, + (LPSECURITY_ATTRIBUTES)NULL, creationDisposition, + flagsAndAttributes, (HANDLE) NULL); + return (_fileIsOpen = (_handle != INVALID_HANDLE_VALUE)); +} + +#ifndef _UNICODE +bool CFileBase::Create(LPCWSTR fileName, DWORD desiredAccess, + DWORD shareMode, DWORD creationDisposition, DWORD flagsAndAttributes) +{ + if (g_IsNT) + { + Close(); + _handle = ::CreateFileW(fileName, desiredAccess, shareMode, + (LPSECURITY_ATTRIBUTES)NULL, creationDisposition, + flagsAndAttributes, (HANDLE) NULL); + return (_fileIsOpen = (_handle != INVALID_HANDLE_VALUE)); + } + return Create(UnicodeStringToMultiByte(fileName, ::AreFileApisANSI() ? CP_ACP : CP_OEMCP), + desiredAccess, shareMode, creationDisposition, flagsAndAttributes); +} +#endif + +bool CFileBase::Close() +{ + if(!_fileIsOpen) + return true; + bool result = BOOLToBool(::CloseHandle(_handle)); + _fileIsOpen = !result; + return result; +} + +bool CFileBase::GetPosition(UInt64 &position) const +{ + return Seek(0, FILE_CURRENT, position); +} + +bool CFileBase::GetLength(UInt64 &length) const +{ + DWORD sizeHigh; + DWORD sizeLow = ::GetFileSize(_handle, &sizeHigh); + if(sizeLow == 0xFFFFFFFF) + if(::GetLastError() != NO_ERROR) + return false; + length = (((UInt64)sizeHigh) << 32) + sizeLow; + return true; +} + +bool CFileBase::Seek(Int64 distanceToMove, DWORD moveMethod, UInt64 &newPosition) const +{ + LARGE_INTEGER value; + value.QuadPart = distanceToMove; + value.LowPart = ::SetFilePointer(_handle, value.LowPart, &value.HighPart, moveMethod); + if (value.LowPart == 0xFFFFFFFF) + if(::GetLastError() != NO_ERROR) + return false; + newPosition = value.QuadPart; + return true; +} + +bool CFileBase::Seek(UInt64 position, UInt64 &newPosition) +{ + return Seek(position, FILE_BEGIN, newPosition); +} + +bool CFileBase::SeekToBegin() +{ + UInt64 newPosition; + return Seek(0, newPosition); +} + +bool CFileBase::SeekToEnd(UInt64 &newPosition) +{ + return Seek(0, FILE_END, newPosition); +} + +bool CFileBase::GetFileInformation(CByHandleFileInfo &fileInfo) const +{ + BY_HANDLE_FILE_INFORMATION winFileInfo; + if(!::GetFileInformationByHandle(_handle, &winFileInfo)) + return false; + fileInfo.Attributes = winFileInfo.dwFileAttributes; + fileInfo.CreationTime = winFileInfo.ftCreationTime; + fileInfo.LastAccessTime = winFileInfo.ftLastAccessTime; + fileInfo.LastWriteTime = winFileInfo.ftLastWriteTime; + fileInfo.VolumeSerialNumber = winFileInfo.dwFileAttributes; + fileInfo.Size = (((UInt64)winFileInfo.nFileSizeHigh) << 32) + winFileInfo.nFileSizeLow; + fileInfo.NumberOfLinks = winFileInfo.nNumberOfLinks; + fileInfo.FileIndex = (((UInt64)winFileInfo.nFileIndexHigh) << 32) + winFileInfo.nFileIndexLow; + return true; +} + +///////////////////////// +// CInFile + +bool CInFile::Open(LPCTSTR fileName, DWORD shareMode, DWORD creationDisposition, DWORD flagsAndAttributes) + { return Create(fileName, GENERIC_READ, shareMode, creationDisposition, flagsAndAttributes); } + +bool CInFile::Open(LPCTSTR fileName) + { return Open(fileName, FILE_SHARE_READ, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL); } + +#ifndef _UNICODE +bool CInFile::Open(LPCWSTR fileName, DWORD shareMode, DWORD creationDisposition, DWORD flagsAndAttributes) + { return Create(fileName, GENERIC_READ, shareMode, creationDisposition, flagsAndAttributes); } + +bool CInFile::Open(LPCWSTR fileName) + { return Open(fileName, FILE_SHARE_READ, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL); } +#endif + +// ReadFile and WriteFile functions in Windows have BUG: +// If you Read or Write 64MB or more (probably min_failure_size = 64MB - 32KB + 1) +// from/to Network file, it returns ERROR_NO_SYSTEM_RESOURCES +// (Insufficient system resources exist to complete the requested service). + +static UInt32 kChunkSizeMax = (1 << 24); + +bool CInFile::ReadPart(void *data, UInt32 size, UInt32 &processedSize) +{ + if (size > kChunkSizeMax) + size = kChunkSizeMax; + DWORD processedLoc = 0; + bool res = BOOLToBool(::ReadFile(_handle, data, size, &processedLoc, NULL)); + processedSize = (UInt32)processedLoc; + return res; +} + +bool CInFile::Read(void *data, UInt32 size, UInt32 &processedSize) +{ + processedSize = 0; + do + { + UInt32 processedLoc = 0; + bool res = ReadPart(data, size, processedLoc); + processedSize += processedLoc; + if (!res) + return false; + if (processedLoc == 0) + return true; + data = (void *)((unsigned char *)data + processedLoc); + size -= processedLoc; + } + while (size > 0); + return true; +} + +///////////////////////// +// COutFile + +bool COutFile::Open(LPCTSTR fileName, DWORD shareMode, DWORD creationDisposition, DWORD flagsAndAttributes) + { return CFileBase::Create(fileName, GENERIC_WRITE, shareMode, creationDisposition, flagsAndAttributes); } + +static inline DWORD GetCreationDisposition(bool createAlways) + { return createAlways? CREATE_ALWAYS: CREATE_NEW; } + +bool COutFile::Open(LPCTSTR fileName, DWORD creationDisposition) + { return Open(fileName, FILE_SHARE_READ, creationDisposition, FILE_ATTRIBUTE_NORMAL); } + +bool COutFile::Create(LPCTSTR fileName, bool createAlways) + { return Open(fileName, GetCreationDisposition(createAlways)); } + +#ifndef _UNICODE + +bool COutFile::Open(LPCWSTR fileName, DWORD shareMode, DWORD creationDisposition, DWORD flagsAndAttributes) + { return CFileBase::Create(fileName, GENERIC_WRITE, shareMode, creationDisposition, flagsAndAttributes); } + +bool COutFile::Open(LPCWSTR fileName, DWORD creationDisposition) + { return Open(fileName, FILE_SHARE_READ, creationDisposition, FILE_ATTRIBUTE_NORMAL); } + +bool COutFile::Create(LPCWSTR fileName, bool createAlways) + { return Open(fileName, GetCreationDisposition(createAlways)); } + +#endif + +bool COutFile::SetTime(const FILETIME *creationTime, const FILETIME *lastAccessTime, const FILETIME *lastWriteTime) + { return BOOLToBool(::SetFileTime(_handle, creationTime, lastAccessTime, lastWriteTime)); } + +bool COutFile::SetLastWriteTime(const FILETIME *lastWriteTime) + { return SetTime(NULL, NULL, lastWriteTime); } + +bool COutFile::WritePart(const void *data, UInt32 size, UInt32 &processedSize) +{ + if (size > kChunkSizeMax) + size = kChunkSizeMax; + DWORD processedLoc = 0; + bool res = BOOLToBool(::WriteFile(_handle, data, size, &processedLoc, NULL)); + processedSize = (UInt32)processedLoc; + return res; +} + +bool COutFile::Write(const void *data, UInt32 size, UInt32 &processedSize) +{ + processedSize = 0; + do + { + UInt32 processedLoc = 0; + bool res = WritePart(data, size, processedLoc); + processedSize += processedLoc; + if (!res) + return false; + if (processedLoc == 0) + return true; + data = (const void *)((const unsigned char *)data + processedLoc); + size -= processedLoc; + } + while (size > 0); + return true; +} + +bool COutFile::SetEndOfFile() { return BOOLToBool(::SetEndOfFile(_handle)); } + +bool COutFile::SetLength(UInt64 length) +{ + UInt64 newPosition; + if(!Seek(length, newPosition)) + return false; + if(newPosition != length) + return false; + return SetEndOfFile(); +} + +}}} diff --git a/lzma443/C/Windows/FileIO.h b/lzma443/C/Windows/FileIO.h new file mode 100755 index 0000000..1801484 --- /dev/null +++ b/lzma443/C/Windows/FileIO.h @@ -0,0 +1,98 @@ +// Windows/FileIO.h + +#ifndef __WINDOWS_FILEIO_H +#define __WINDOWS_FILEIO_H + +#include "../Common/Types.h" + +namespace NWindows { +namespace NFile { +namespace NIO { + +struct CByHandleFileInfo +{ + DWORD Attributes; + FILETIME CreationTime; + FILETIME LastAccessTime; + FILETIME LastWriteTime; + DWORD VolumeSerialNumber; + UInt64 Size; + DWORD NumberOfLinks; + UInt64 FileIndex; +}; + +class CFileBase +{ +protected: + bool _fileIsOpen; + HANDLE _handle; + bool Create(LPCTSTR fileName, DWORD desiredAccess, + DWORD shareMode, DWORD creationDisposition, DWORD flagsAndAttributes); + #ifndef _UNICODE + bool Create(LPCWSTR fileName, DWORD desiredAccess, + DWORD shareMode, DWORD creationDisposition, DWORD flagsAndAttributes); + #endif + +public: + CFileBase(): _fileIsOpen(false){}; + virtual ~CFileBase(); + + virtual bool Close(); + + bool GetPosition(UInt64 &position) const; + bool GetLength(UInt64 &length) const; + + bool Seek(Int64 distanceToMove, DWORD moveMethod, UInt64 &newPosition) const; + bool Seek(UInt64 position, UInt64 &newPosition); + bool SeekToBegin(); + bool SeekToEnd(UInt64 &newPosition); + + bool GetFileInformation(CByHandleFileInfo &fileInfo) const; +}; + +class CInFile: public CFileBase +{ +public: + bool Open(LPCTSTR fileName, DWORD shareMode, DWORD creationDisposition, DWORD flagsAndAttributes); + bool Open(LPCTSTR fileName); + #ifndef _UNICODE + bool Open(LPCWSTR fileName, DWORD shareMode, DWORD creationDisposition, DWORD flagsAndAttributes); + bool Open(LPCWSTR fileName); + #endif + bool ReadPart(void *data, UInt32 size, UInt32 &processedSize); + bool Read(void *data, UInt32 size, UInt32 &processedSize); +}; + +class COutFile: public CFileBase +{ + // DWORD m_CreationDisposition; +public: + // COutFile(): m_CreationDisposition(CREATE_NEW){}; + bool Open(LPCTSTR fileName, DWORD shareMode, DWORD creationDisposition, DWORD flagsAndAttributes); + bool Open(LPCTSTR fileName, DWORD creationDisposition); + bool Create(LPCTSTR fileName, bool createAlways); + + #ifndef _UNICODE + bool Open(LPCWSTR fileName, DWORD shareMode, DWORD creationDisposition, DWORD flagsAndAttributes); + bool Open(LPCWSTR fileName, DWORD creationDisposition); + bool Create(LPCWSTR fileName, bool createAlways); + #endif + + /* + void SetOpenCreationDisposition(DWORD creationDisposition) + { m_CreationDisposition = creationDisposition; } + void SetOpenCreationDispositionCreateAlways() + { m_CreationDisposition = CREATE_ALWAYS; } + */ + + bool SetTime(const FILETIME *creationTime, const FILETIME *lastAccessTime, const FILETIME *lastWriteTime); + bool SetLastWriteTime(const FILETIME *lastWriteTime); + bool WritePart(const void *data, UInt32 size, UInt32 &processedSize); + bool Write(const void *data, UInt32 size, UInt32 &processedSize); + bool SetEndOfFile(); + bool SetLength(UInt64 length); +}; + +}}} + +#endif diff --git a/lzma443/C/Windows/StdAfx.h b/lzma443/C/Windows/StdAfx.h new file mode 100755 index 0000000..e7924c8 --- /dev/null +++ b/lzma443/C/Windows/StdAfx.h @@ -0,0 +1,9 @@ +// StdAfx.h + +#ifndef __STDAFX_H +#define __STDAFX_H + +#include "../Common/MyWindows.h" +#include "../Common/NewHandler.h" + +#endif diff --git a/lzma443/CPL.html b/lzma443/CPL.html new file mode 100644 index 0000000..f1d3be1 --- /dev/null +++ b/lzma443/CPL.html @@ -0,0 +1,224 @@ + +Common Public License - v 1.0 + + +

Common Public License - v 1.0 +

+

THE ACCOMPANYING PROGRAM IS PROVIDED UNDER +THE TERMS OF THIS COMMON PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR +DISTRIBUTION OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS +AGREEMENT. +

+

1. DEFINITIONS +

"Contribution" means: +

    a) in the case of the initial Contributor, the initial code + and documentation distributed under this Agreement, and
    b) in + the case of each subsequent Contributor:
+
    i) changes to the Program, and
+
    ii) additions to the Program;
+
    where such changes and/or additions to the Program originate + from and are distributed by that particular Contributor. A + Contribution 'originates' from a Contributor if it was added to the Program by + such Contributor itself or anyone acting on such Contributor's behalf. + Contributions do not include additions to the Program + which: (i) are separate modules of software distributed in conjunction with + the Program under their own license agreement, and (ii) are not derivative + works of the Program.
+

+

"Contributor" means any person or entity that distributes the +Program. +

+

"Licensed Patents " mean patent claims licensable by a +Contributor which are necessarily infringed by the use or sale of its +Contribution alone or when combined with the Program. +

+

"Program" means the Contributions +distributed in accordance with this Agreement. +

+

"Recipient" means anyone who receives the Program under this +Agreement, including all Contributors. +

+

2. GRANT OF RIGHTS +

    a) Subject to the + terms of this Agreement, each Contributor hereby grants + Recipient a non-exclusive, worldwide, royalty-free copyright license + to reproduce, prepare + derivative works of, publicly display, publicly perform, distribute and + sublicense the Contribution of such Contributor, if any, and such derivative + works, in source code and object code form.
+
+
    b) Subject to the terms of this + Agreement, each Contributor hereby grants Recipient a + non-exclusive, worldwide, royalty-free patent license under Licensed Patents to make, use, sell, + offer to sell, import and otherwise transfer the Contribution of such + Contributor, if any, in source code and object code form. This patent license + shall apply to the combination of the Contribution and the Program if, at the + time the Contribution is added by the Contributor, such addition of the + Contribution causes such combination to be covered by the Licensed Patents. + The patent license shall not apply to any other combinations which include the + Contribution. No hardware per se is licensed hereunder.
+
+
    c) Recipient understands that although each Contributor + grants the licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the patent or + other intellectual property rights of any other entity. Each Contributor + disclaims any liability to Recipient for claims brought by any other entity + based on infringement of intellectual property rights or otherwise. As a + condition to exercising the rights and licenses granted hereunder, each + Recipient hereby assumes sole responsibility to secure any other intellectual + property rights needed, if any. For example, if a third party patent license + is required to allow Recipient to distribute the Program, it is Recipient's + responsibility to acquire that license before distributing the +Program.
+
+
    d) Each Contributor represents that to its knowledge it has + sufficient copyright rights in its Contribution, if any, to grant the + copyright license set forth in this Agreement.
+
+

3. REQUIREMENTS +

A Contributor may choose to distribute the Program in +object code form under its own license agreement, provided that: +

    a) it complies with the terms and conditions of this + Agreement; and
+
    b) its license agreement:
+
    i) effectively disclaims on behalf of all + Contributors all warranties and conditions, express and implied, including + warranties or conditions of title and non-infringement, and implied warranties + or conditions of merchantability and fitness for a particular purpose; +
+
    ii) effectively excludes on behalf of all Contributors all + liability for damages, including direct, indirect, special, incidental and + consequential damages, such as lost profits;
+
    iii) states that any provisions which + differ from this Agreement are offered by that Contributor alone and not by + any other party; and
+
    iv) states that source code for the Program is available from + such Contributor, and informs licensees how to obtain it in a reasonable + manner on or through a medium customarily used for software + exchange.
+
+

When the Program is made available in source code form: +

    a) it must be made available under this Agreement; and +
+
    b) a copy of this Agreement must be included with each copy + of the Program.
+

+

Contributors +may not remove or alter any copyright notices contained within the Program. + +

+

Each Contributor must identify itself as the originator of its +Contribution, if any, in a manner that reasonably allows subsequent Recipients +to identify the originator of the Contribution. +

+

4. COMMERCIAL DISTRIBUTION +

Commercial distributors of software may accept certain +responsibilities with respect to end users, business partners and the like. +While this license is intended to facilitate the commercial use of the Program, +the Contributor who includes the Program in a commercial product offering should +do so in a manner which does not create potential liability for other +Contributors. Therefore, if a Contributor includes the Program in a commercial +product offering, such Contributor ("Commercial Contributor") hereby agrees to +defend and indemnify every other Contributor ("Indemnified Contributor") against +any losses, damages and costs (collectively "Losses") arising from claims, +lawsuits and other legal actions brought by a third party against the +Indemnified Contributor to the extent caused by the acts or omissions of such +Commercial Contributor in connection with its distribution of the Program in a +commercial product offering. The obligations in this section do not apply to any +claims or Losses relating to any actual or alleged intellectual property +infringement. In order to qualify, an Indemnified Contributor must: a) promptly +notify the Commercial Contributor in writing of such claim, and b) allow the +Commercial Contributor to control, and cooperate with the Commercial Contributor +in, the defense and any related settlement negotiations. The Indemnified +Contributor may participate in any such claim at its own expense. +

+

For example, a Contributor might include the Program in a +commercial product offering, Product X. That Contributor is then a Commercial +Contributor. If that Commercial Contributor then makes performance claims, or +offers warranties related to Product X, those performance claims and warranties +are such Commercial Contributor's responsibility alone. Under this section, the +Commercial Contributor would have to defend claims against the other +Contributors related to those performance claims and warranties, and if a court +requires any other Contributor to pay any damages as a result, the Commercial +Contributor must pay those damages. +

+

5. NO WARRANTY +

EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS +PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR +CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A +PARTICULAR PURPOSE. Each Recipient is solely responsible for +determining the appropriateness of using and distributing the Program and assumes all risks associated with its +exercise of rights under this Agreement, including but not +limited to the risks and costs of program errors, compliance with applicable +laws, damage to or loss of data, programs or equipment, and +unavailability or interruption of operations. +

+

6. DISCLAIMER OF LIABILITY +

EXCEPT AS EXPRESSLY SET FORTH IN THIS +AGREEMENT, NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS +GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. +

+

7. GENERAL +

If any provision of this Agreement is +invalid or unenforceable under applicable law, it shall not affect the validity +or enforceability of the remainder of the terms of this Agreement, and without +further action by the parties hereto, such provision shall be reformed to the +minimum extent necessary to make such provision valid and enforceable. +

+

If Recipient institutes patent litigation against a Contributor +with respect to a patent applicable to software (including a cross-claim or +counterclaim in a lawsuit), then any patent licenses granted by that Contributor +to such Recipient under this Agreement shall terminate as of the date such +litigation is filed. In addition, if Recipient institutes patent litigation +against any entity (including a cross-claim or counterclaim in a lawsuit) +alleging that the Program itself (excluding combinations of the Program with +other software or hardware) infringes such Recipient's patent(s), then such +Recipient's rights granted under Section 2(b) shall terminate as of the date +such litigation is filed. +

+

All Recipient's rights under this Agreement shall terminate if +it fails to comply with any of the material terms or conditions of this +Agreement and does not cure such failure in a reasonable period of time after +becoming aware of such noncompliance. If all Recipient's rights under this +Agreement terminate, Recipient agrees to cease use and distribution of the +Program as soon as reasonably practicable. However, Recipient's obligations +under this Agreement and any licenses granted by Recipient relating to the +Program shall continue and survive. +

+

Everyone is permitted to copy and distribute +copies of this Agreement, but in order to avoid inconsistency the Agreement is +copyrighted and may only be modified in the following manner. The Agreement +Steward reserves the right to publish new versions +(including revisions) of this Agreement from time to time. +No one other than the Agreement Steward has the right to modify this Agreement. +IBM is the initial Agreement Steward. IBM may assign the responsibility to serve +as the Agreement Steward to a suitable separate entity. Each +new version of the Agreement will be given a distinguishing version number. The +Program (including Contributions) may always be distributed subject to the +version of the Agreement under which it was received. In addition, after a new +version of the Agreement is published, Contributor may elect to distribute the +Program (including its Contributions) under the new version. +Except as expressly stated in Sections 2(a) and 2(b) above, +Recipient receives no rights or licenses to the intellectual property of any +Contributor under this Agreement, whether expressly, by +implication, estoppel or otherwise. All +rights in the Program not expressly granted under this Agreement are +reserved. +

+

This Agreement is governed by the laws of the State of New York +and the intellectual property laws of the United States of America. No party to +this Agreement will bring a legal action under this Agreement more than one year +after the cause of action arose. Each party waives its rights to a jury trial in +any resulting litigation. +

+

diff --git a/lzma443/LGPL.txt b/lzma443/LGPL.txt new file mode 100644 index 0000000..4c38901 --- /dev/null +++ b/lzma443/LGPL.txt @@ -0,0 +1,504 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! + + diff --git a/lzma443/Makefile.am b/lzma443/Makefile.am new file mode 100644 index 0000000..d0018d7 --- /dev/null +++ b/lzma443/Makefile.am @@ -0,0 +1,157 @@ +# We don't actually build LZMA; we just +# want to bring along the source code to satisfy LGPL requirements. + +EXTRA_DIST = \ +7zC.txt \ +7zFormat.txt \ +C/7zip/Archive/7z_C/7zAlloc.c \ +C/7zip/Archive/7z_C/7zAlloc.h \ +C/7zip/Archive/7z_C/7zBuffer.c \ +C/7zip/Archive/7z_C/7zBuffer.h \ +C/7zip/Archive/7z_C/7zCrc.c \ +C/7zip/Archive/7z_C/7zCrc.h \ +C/7zip/Archive/7z_C/7zDecode.c \ +C/7zip/Archive/7z_C/7zDecode.h \ +C/7zip/Archive/7z_C/7zExtract.c \ +C/7zip/Archive/7z_C/7zExtract.h \ +C/7zip/Archive/7z_C/7zHeader.c \ +C/7zip/Archive/7z_C/7zHeader.h \ +C/7zip/Archive/7z_C/7zIn.c \ +C/7zip/Archive/7z_C/7zIn.h \ +C/7zip/Archive/7z_C/7zItem.c \ +C/7zip/Archive/7z_C/7zItem.h \ +C/7zip/Archive/7z_C/7zMain.c \ +C/7zip/Archive/7z_C/7zMethodID.c \ +C/7zip/Archive/7z_C/7zMethodID.h \ +C/7zip/Archive/7z_C/7zTypes.h \ +C/7zip/Archive/7z_C/7z_C.dsp \ +C/7zip/Archive/7z_C/7z_C.dsw \ +C/7zip/Archive/7z_C/makefile \ +C/7zip/Archive/7z_C/makefile.gcc \ +C/7zip/Common/FileStreams.cpp \ +C/7zip/Common/FileStreams.h \ +C/7zip/Common/InBuffer.cpp \ +C/7zip/Common/InBuffer.h \ +C/7zip/Common/OutBuffer.cpp \ +C/7zip/Common/OutBuffer.h \ +C/7zip/Common/StdAfx.h \ +C/7zip/Common/StreamUtils.cpp \ +C/7zip/Common/StreamUtils.h \ +C/7zip/Compress/Branch/ARM.cpp \ +C/7zip/Compress/Branch/ARM.h \ +C/7zip/Compress/Branch/ARMThumb.cpp \ +C/7zip/Compress/Branch/ARMThumb.h \ +C/7zip/Compress/Branch/BranchARM.c \ +C/7zip/Compress/Branch/BranchARM.h \ +C/7zip/Compress/Branch/BranchARMThumb.c \ +C/7zip/Compress/Branch/BranchARMThumb.h \ +C/7zip/Compress/Branch/BranchCoder.cpp \ +C/7zip/Compress/Branch/BranchCoder.h \ +C/7zip/Compress/Branch/BranchIA64.c \ +C/7zip/Compress/Branch/BranchIA64.h \ +C/7zip/Compress/Branch/BranchPPC.c \ +C/7zip/Compress/Branch/BranchPPC.h \ +C/7zip/Compress/Branch/BranchSPARC.c \ +C/7zip/Compress/Branch/BranchSPARC.h \ +C/7zip/Compress/Branch/BranchTypes.h \ +C/7zip/Compress/Branch/BranchX86.c \ +C/7zip/Compress/Branch/BranchX86.h \ +C/7zip/Compress/Branch/IA64.cpp \ +C/7zip/Compress/Branch/IA64.h \ +C/7zip/Compress/Branch/PPC.cpp \ +C/7zip/Compress/Branch/PPC.h \ +C/7zip/Compress/Branch/SPARC.cpp \ +C/7zip/Compress/Branch/SPARC.h \ +C/7zip/Compress/Branch/StdAfx.h \ +C/7zip/Compress/Branch/x86.cpp \ +C/7zip/Compress/Branch/x86.h \ +C/7zip/Compress/Branch/x86_2.cpp \ +C/7zip/Compress/Branch/x86_2.h \ +C/7zip/Compress/LZ/BinTree/BinTree.h \ +C/7zip/Compress/LZ/BinTree/BinTree2.h \ +C/7zip/Compress/LZ/BinTree/BinTree3.h \ +C/7zip/Compress/LZ/BinTree/BinTree3Z.h \ +C/7zip/Compress/LZ/BinTree/BinTree4.h \ +C/7zip/Compress/LZ/BinTree/BinTreeMain.h \ +C/7zip/Compress/LZ/HashChain/HC2.h \ +C/7zip/Compress/LZ/HashChain/HC3.h \ +C/7zip/Compress/LZ/HashChain/HC4.h \ +C/7zip/Compress/LZ/HashChain/HCMain.h \ +C/7zip/Compress/LZ/IMatchFinder.h \ +C/7zip/Compress/LZ/LZInWindow.cpp \ +C/7zip/Compress/LZ/LZInWindow.h \ +C/7zip/Compress/LZ/LZOutWindow.cpp \ +C/7zip/Compress/LZ/LZOutWindow.h \ +C/7zip/Compress/LZ/StdAfx.h \ +C/7zip/Compress/LZMA/LZMA.h \ +C/7zip/Compress/LZMA/LZMADecoder.cpp \ +C/7zip/Compress/LZMA/LZMADecoder.h \ +C/7zip/Compress/LZMA/LZMAEncoder.cpp \ +C/7zip/Compress/LZMA/LZMAEncoder.h \ +C/7zip/Compress/LZMA/StdAfx.h \ +C/7zip/Compress/LZMA_Alone/AloneLZMA.dsp \ +C/7zip/Compress/LZMA_Alone/AloneLZMA.dsw \ +C/7zip/Compress/LZMA_Alone/LzmaAlone.cpp \ +C/7zip/Compress/LZMA_Alone/LzmaBench.cpp \ +C/7zip/Compress/LZMA_Alone/LzmaBench.h \ +C/7zip/Compress/LZMA_Alone/LzmaRam.cpp \ +C/7zip/Compress/LZMA_Alone/LzmaRam.h \ +C/7zip/Compress/LZMA_Alone/LzmaRamDecode.c \ +C/7zip/Compress/LZMA_Alone/LzmaRamDecode.h \ +C/7zip/Compress/LZMA_Alone/StdAfx.cpp \ +C/7zip/Compress/LZMA_Alone/StdAfx.h \ +C/7zip/Compress/LZMA_Alone/makefile \ +C/7zip/Compress/LZMA_Alone/makefile.gcc \ +C/7zip/Compress/LZMA_C/LzmaDecode.c \ +C/7zip/Compress/LZMA_C/LzmaDecode.h \ +C/7zip/Compress/LZMA_C/LzmaDecodeSize.c \ +C/7zip/Compress/LZMA_C/LzmaStateDecode.c \ +C/7zip/Compress/LZMA_C/LzmaStateDecode.h \ +C/7zip/Compress/LZMA_C/LzmaStateTest.c \ +C/7zip/Compress/LZMA_C/LzmaTest.c \ +C/7zip/Compress/LZMA_C/LzmaTypes.h \ +C/7zip/Compress/LZMA_C/makefile \ +C/7zip/Compress/LZMA_C/makefile.gcc \ +C/7zip/Compress/RangeCoder/RangeCoder.h \ +C/7zip/Compress/RangeCoder/RangeCoderBit.cpp \ +C/7zip/Compress/RangeCoder/RangeCoderBit.h \ +C/7zip/Compress/RangeCoder/RangeCoderBitTree.h \ +C/7zip/Compress/RangeCoder/RangeCoderOpt.h \ +C/7zip/Compress/RangeCoder/StdAfx.h \ +C/7zip/ICoder.h \ +C/7zip/IStream.h \ +C/Common/Alloc.cpp \ +C/Common/Alloc.h \ +C/Common/CRC.cpp \ +C/Common/CRC.h \ +C/Common/C_FileIO.cpp \ +C/Common/C_FileIO.h \ +C/Common/ComTry.h \ +C/Common/CommandLineParser.cpp \ +C/Common/CommandLineParser.h \ +C/Common/Defs.h \ +C/Common/MyCom.h \ +C/Common/MyGuidDef.h \ +C/Common/MyInitGuid.h \ +C/Common/MyUnknown.h \ +C/Common/MyWindows.h \ +C/Common/NewHandler.cpp \ +C/Common/NewHandler.h \ +C/Common/StdAfx.h \ +C/Common/String.cpp \ +C/Common/String.h \ +C/Common/StringConvert.cpp \ +C/Common/StringConvert.h \ +C/Common/StringToInt.cpp \ +C/Common/StringToInt.h \ +C/Common/Types.h \ +C/Common/Vector.cpp \ +C/Common/Vector.h \ +C/Windows/Defs.h \ +C/Windows/FileIO.cpp \ +C/Windows/FileIO.h \ +C/Windows/StdAfx.h \ +LGPL.txt \ +Methods.txt \ +history.txt \ +lzma.txt diff --git a/lzma443/Methods.txt b/lzma443/Methods.txt new file mode 100644 index 0000000..393e1b0 --- /dev/null +++ b/lzma443/Methods.txt @@ -0,0 +1,138 @@ +Compression method IDs (4.38) +----------------------------- + +Each compression method in 7z has unique binary value (ID). +The length of ID in bytes is arbitrary but it can not exceed 15 bytes. + +If you want to add some new ID, you have two ways: +1) Write request for allocating IDs to 7-zip developers. +2) Use such random ID: + 03 E0 ZZ ... ZZ MM ... MM VV ... VV + + ZZ != 0, MM != 0, VV != 0 + + 03 E0 - Prefix for random IDs + ZZ ... ZZ - Developer ID. (length >= 4). Use real random bytes. + You can notify 7-Zip developers about your Developer ID. + MM ... MM - Method ID (length >= 1) + VV ... VV - Version (length >= 1) + + Note: Use new ID (MM ... MM VV .. VV) only if old codec can not decode + data encoded with new version. + + +List of defined IDs +------------------- + +00 - Copy +01 - Reserved +02 - Common + 03 Swap + - 2 Swap2 + - 4 Swap4 + 04 Delta (subject to change) + +03 - 7z + 01 - LZMA + 01 - Version + + 03 - Branch + 01 - x86 + 03 - BCJ + 1B - BCJ2 + 02 - PPC + 05 - BC_PPC_B (Big Endian) + 03 - Alpha + 01 - BC_Alpha + 04 - IA64 + 01 - BC_IA64 + 05 - ARM + 01 - BC_ARM + 06 - M68 + 05 - BC_M68_B (Big Endian) + 07 - ARM Thumb + 01 - BC_ARMThumb + 08 - SPARC + 05 - BC_SPARC + + 04 - PPMD + 01 - Version + + 80 - reserved for independent developers + + E0 - Random IDs + +04 - Misc + 00 - Reserved + 01 - Zip + 00 - Copy (not used). Use {00} instead + 01 - Shrink + 06 - Implode + 08 - Deflate + 09 - Deflate64 + 12 - BZip2 (not used). Use {04 02 02} instead + 02 - BZip + 02 - BZip2 + 03 - Rar + 01 - Rar15 + 02 - Rar20 + 03 - Rar29 + 04 - Arj + 01 - Arj (1,2,3) + 02 - Arj 4 + 05 - Z + 06 - Lzh + 07 - Reserved for 7z + 08 - Cab + 09 - NSIS + 01 - DeflateNSIS + 02 - BZip2NSIS + + +06 - Crypto + 00 - + 01 - AES + 0x - AES-128 + 4x - AES-192 + 8x - AES-256 + + x0 - ECB + x1 - CBC + x2 - CFB + x3 - OFB + + 07 - Reserved + 0F - Reserved + + F0 - Misc Ciphers (Real Ciphers without hashing algo) + + F1 - Misc Ciphers (Combine) + 01 - Zip + 01 - Main Zip crypto algo + 03 - RAR + 02 - + 03 - Rar29 AES-128 + (modified SHA-1) + 07 - 7z + 01 - AES-256 + SHA-256 + +07 - Hash (subject to change) + 00 - + 01 - CRC + 02 - SHA-1 + 03 - SHA-256 + 04 - SHA-384 + 05 - SHA-512 + + F0 - Misc Hash + + F1 - Misc + 03 - RAR + 03 - Rar29 Password Hashing (modified SHA1) + 07 - 7z + 01 - SHA-256 Password Hashing + + + + +--- +End of document diff --git a/lzma443/history.txt b/lzma443/history.txt new file mode 100644 index 0000000..bf2cf64 --- /dev/null +++ b/lzma443/history.txt @@ -0,0 +1,182 @@ +HISTORY of the LZMA SDK +----------------------- + + Version 4.43 2006-06-04 + -------------------------------------- + - Small changes for more compatibility with some C/C++ compilers. + + + Version 4.42 2006-05-15 + -------------------------------------- + - Small changes in .h files in ANSI-C version. + + + Version 4.39 beta 2006-04-14 + -------------------------------------- + - Bug in versions 4.33b:4.38b was fixed: + C++ version of LZMA encoder could not correctly compress + files larger than 2 GB with HC4 match finder (-mfhc4). + + + Version 4.37 beta 2005-04-06 + -------------------------------------- + - Fixes in C++ code: code could no be compiled if _NO_EXCEPTIONS was defined. + + + Version 4.35 beta 2005-03-02 + -------------------------------------- + - Bug was fixed in C++ version of LZMA Decoder: + If encoded stream was corrupted, decoder could access memory + outside of allocated range. + + + Version 4.34 beta 2006-02-27 + -------------------------------------- + - Compressing speed and memory requirements for compressing were increased + - LZMA now can use only these match finders: HC4, BT2, BT3, BT4 + + + Version 4.32 2005-12-09 + -------------------------------------- + - Java version of LZMA SDK was included + + + Version 4.30 2005-11-20 + -------------------------------------- + - Compression ratio was improved in -a2 mode + - Speed optimizations for compressing in -a2 mode + - -fb switch now supports values up to 273 + - Bug in 7z_C (7zIn.c) was fixed: + It used Alloc/Free functions from different memory pools. + So if program used two memory pools, it worked incorrectly. + - 7z_C: .7z format supporting was improved + - LZMA# SDK (C#.NET version) was included + + + Version 4.27 (Updated) 2005-09-21 + -------------------------------------- + - Some GUIDs/interfaces in C++ were changed. + IStream.h: + ISequentialInStream::Read now works as old ReadPart + ISequentialOutStream::Write now works as old WritePart + + + Version 4.27 2005-08-07 + -------------------------------------- + - Bug in LzmaDecodeSize.c was fixed: + if _LZMA_IN_CB and _LZMA_OUT_READ were defined, + decompressing worked incorrectly. + + + Version 4.26 2005-08-05 + -------------------------------------- + - Fixes in 7z_C code and LzmaTest.c: + previous versions could work incorrectly, + if malloc(0) returns 0 + + + Version 4.23 2005-06-29 + -------------------------------------- + - Small fixes in C++ code + + + Version 4.22 2005-06-10 + -------------------------------------- + - Small fixes + + + Version 4.21 2005-06-08 + -------------------------------------- + - Interfaces for ANSI-C LZMA Decoder (LzmaDecode.c) were changed + - New additional version of ANSI-C LZMA Decoder with zlib-like interface: + - LzmaStateDecode.h + - LzmaStateDecode.c + - LzmaStateTest.c + - ANSI-C LZMA Decoder now can decompress files larger than 4 GB + + + Version 4.17 2005-04-18 + -------------------------------------- + - New example for RAM->RAM compressing/decompressing: + LZMA + BCJ (filter for x86 code): + - LzmaRam.h + - LzmaRam.cpp + - LzmaRamDecode.h + - LzmaRamDecode.c + - -f86 switch for lzma.exe + + + Version 4.16 2005-03-29 + -------------------------------------- + - Bug was fixed in LzmaDecode.c (ANSI-C LZMA Decoder): + If _LZMA_OUT_READ was defined, and if encoded stream was corrupted, + decoder could access memory outside of allocated range. + - Speed optimization of ANSI-C LZMA Decoder (now it's about 20% faster). + Old version of LZMA Decoder now is in file LzmaDecodeSize.c. + LzmaDecodeSize.c can provide slightly smaller code than LzmaDecode.c + - Small speed optimization in LZMA C++ code + - filter for SPARC's code was added + - Simplified version of .7z ANSI-C Decoder was included + + + Version 4.06 2004-09-05 + -------------------------------------- + - Bug in v4.05 was fixed: + LZMA-Encoder didn't release output stream in some cases. + + + Version 4.05 2004-08-25 + -------------------------------------- + - Source code of filters for x86, IA-64, ARM, ARM-Thumb + and PowerPC code was included to SDK + - Some internal minor changes + + + Version 4.04 2004-07-28 + -------------------------------------- + - More compatibility with some C++ compilers + + + Version 4.03 2004-06-18 + -------------------------------------- + - "Benchmark" command was added. It measures compressing + and decompressing speed and shows rating values. + Also it checks hardware errors. + + + Version 4.02 2004-06-10 + -------------------------------------- + - C++ LZMA Encoder/Decoder code now is more portable + and it can be compiled by GCC on Linux. + + + Version 4.01 2004-02-15 + -------------------------------------- + - Some detection of data corruption was enabled. + LzmaDecode.c / RangeDecoderReadByte + ..... + { + rd->ExtraBytes = 1; + return 0xFF; + } + + + Version 4.00 2004-02-13 + -------------------------------------- + - Original version of LZMA SDK + + + +HISTORY of the LZMA +------------------- + 2001-2006: Improvements to LZMA compressing/decompressing code, + keeping compatibility with original LZMA format + 1996-2001: Development of LZMA compression format + + Some milestones: + + 2001-08-30: LZMA compression was added to 7-Zip + 1999-01-02: First version of 7-Zip was released + + +End of document diff --git a/lzma443/lzma.txt b/lzma443/lzma.txt new file mode 100644 index 0000000..a65fc1e --- /dev/null +++ b/lzma443/lzma.txt @@ -0,0 +1,630 @@ +LZMA SDK 4.43 +------------- + +LZMA SDK Copyright (C) 1999-2006 Igor Pavlov + +LZMA SDK provides the documentation, samples, header files, libraries, +and tools you need to develop applications that use LZMA compression. + +LZMA is default and general compression method of 7z format +in 7-Zip compression program (www.7-zip.org). LZMA provides high +compression ratio and very fast decompression. + +LZMA is an improved version of famous LZ77 compression algorithm. +It was improved in way of maximum increasing of compression ratio, +keeping high decompression speed and low memory requirements for +decompressing. + + + +LICENSE +------- + +LZMA SDK is available under any of the following licenses: + +1) GNU Lesser General Public License (GNU LGPL) +2) Common Public License (CPL) +3) Simplified license for unmodified code (read SPECIAL EXCEPTION) +4) Proprietary license + +It means that you can select one of these four options and follow rules of that license. + + +1,2) GNU LGPL and CPL licenses are pretty similar and both these +licenses are classified as + - "Free software licenses" at http://www.gnu.org/ + - "OSI-approved" at http://www.opensource.org/ + + +3) SPECIAL EXCEPTION + +Igor Pavlov, as the author of this code, expressly permits you +to statically or dynamically link your code (or bind by name) +to the files from LZMA SDK without subjecting your linked +code to the terms of the CPL or GNU LGPL. +Any modifications or additions to files from LZMA SDK, however, +are subject to the GNU LGPL or CPL terms. + +SPECIAL EXCEPTION allows you to use LZMA SDK in applications with closed code, +while you keep LZMA SDK code unmodified. + + +SPECIAL EXCEPTION #2: Igor Pavlov, as the author of this code, expressly permits +you to use this code under the same terms and conditions contained in the License +Agreement you have for any previous version of LZMA SDK developed by Igor Pavlov. + +SPECIAL EXCEPTION #2 allows owners of proprietary licenses to use latest version +of LZMA SDK as update for previous versions. + + +SPECIAL EXCEPTION #3: Igor Pavlov, as the author of this code, expressly permits +you to use code of the following files: +BranchTypes.h, LzmaTypes.h, LzmaTest.c, LzmaStateTest.c, LzmaAlone.cpp, +LzmaAlone.cs, LzmaAlone.java +as public domain code. + + +4) Proprietary license + +LZMA SDK also can be available under a proprietary license which +can include: + +1) Right to modify code without subjecting modified code to the +terms of the CPL or GNU LGPL +2) Technical support for code + +To request such proprietary license or any additional consultations, +send email message from that page: +http://www.7-zip.org/support.html + + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +You should have received a copy of the Common Public License +along with this library. + + +LZMA SDK Contents +----------------- + +LZMA SDK includes: + + - C++ source code of LZMA compressing and decompressing + - ANSI-C compatible source code for LZMA decompressing + - C# source code for LZMA compressing and decompressing + - Java source code for LZMA compressing and decompressing + - Compiled file->file LZMA compressing/decompressing program for Windows system + +ANSI-C LZMA decompression code was ported from original C++ sources to C. +Also it was simplified and optimized for code size. +But it is fully compatible with LZMA from 7-Zip. + + +UNIX/Linux version +------------------ +To compile C++ version of file->file LZMA, go to directory +C/7zip/Compress/LZMA_Alone +and type "make" or "make clean all" to recompile all. + +In some UNIX/Linux versions you must compile LZMA with static libraries. +To compile with static libraries, change string in makefile +LIB = -lm +to string +LIB = -lm -static + + +Files +--------------------- +C - C / CPP source code +CS - C# source code +Java - Java source code +lzma.txt - LZMA SDK description (this file) +7zFormat.txt - 7z Format description +7zC.txt - 7z ANSI-C Decoder description (this file) +methods.txt - Compression method IDs for .7z +LGPL.txt - GNU Lesser General Public License +CPL.html - Common Public License +lzma.exe - Compiled file->file LZMA encoder/decoder for Windows +history.txt - history of the LZMA SDK + + +Source code structure +--------------------- + +C - C / CPP files + Common - common files for C++ projects + Windows - common files for Windows related code + 7zip - files related to 7-Zip Project + Common - common files for 7-Zip + Compress - files related to compression/decompression + LZ - files related to LZ (Lempel-Ziv) compression algorithm + BinTree - Binary Tree Match Finder for LZ algorithm + HashChain - Hash Chain Match Finder for LZ algorithm + Patricia - Patricia Match Finder for LZ algorithm + RangeCoder - Range Coder (special code of compression/decompression) + LZMA - LZMA compression/decompression on C++ + LZMA_Alone - file->file LZMA compression/decompression + LZMA_C - ANSI-C compatible LZMA decompressor + LzmaDecode.h - interface for LZMA decoding on ANSI-C + LzmaDecode.c - LZMA decoding on ANSI-C (new fastest version) + LzmaDecodeSize.c - LZMA decoding on ANSI-C (old size-optimized version) + LzmaTest.c - test application that decodes LZMA encoded file + LzmaTypes.h - basic types for LZMA Decoder + LzmaStateDecode.h - interface for LZMA decoding (State version) + LzmaStateDecode.c - LZMA decoding on ANSI-C (State version) + LzmaStateTest.c - test application (State version) + Branch - Filters for x86, IA-64, ARM, ARM-Thumb, PowerPC and SPARC code + Archive - files related to archiving + 7z_C - 7z ANSI-C Decoder + +CS - C# files + 7zip + Common - some common files for 7-Zip + Compress - files related to compression/decompression + LZ - files related to LZ (Lempel-Ziv) compression algorithm + LZMA - LZMA compression/decompression + LzmaAlone - file->file LZMA compression/decompression + RangeCoder - Range Coder (special code of compression/decompression) + +Java - Java files + SevenZip + Compression - files related to compression/decompression + LZ - files related to LZ (Lempel-Ziv) compression algorithm + LZMA - LZMA compression/decompression + RangeCoder - Range Coder (special code of compression/decompression) + +C/C++ source code of LZMA SDK is part of 7-Zip project. + +You can find ANSI-C LZMA decompressing code at folder + C/7zip/Compress/LZMA_C +7-Zip doesn't use that ANSI-C LZMA code and that code was developed +specially for this SDK. And files from LZMA_C do not need files from +other directories of SDK for compiling. + +7-Zip source code can be downloaded from 7-Zip's SourceForge page: + + http://sourceforge.net/projects/sevenzip/ + + +LZMA features +------------- + - Variable dictionary size (up to 1 GB) + - Estimated compressing speed: about 1 MB/s on 1 GHz CPU + - Estimated decompressing speed: + - 8-12 MB/s on 1 GHz Intel Pentium 3 or AMD Athlon + - 500-1000 KB/s on 100 MHz ARM, MIPS, PowerPC or other simple RISC + - Small memory requirements for decompressing (8-32 KB + DictionarySize) + - Small code size for decompressing: 2-8 KB (depending from + speed optimizations) + +LZMA decoder uses only integer operations and can be +implemented in any modern 32-bit CPU (or on 16-bit CPU with some conditions). + +Some critical operations that affect to speed of LZMA decompression: + 1) 32*16 bit integer multiply + 2) Misspredicted branches (penalty mostly depends from pipeline length) + 3) 32-bit shift and arithmetic operations + +Speed of LZMA decompressing mostly depends from CPU speed. +Memory speed has no big meaning. But if your CPU has small data cache, +overall weight of memory speed will slightly increase. + + +How To Use +---------- + +Using LZMA encoder/decoder executable +-------------------------------------- + +Usage: LZMA inputFile outputFile [...] + + e: encode file + + d: decode file + + b: Benchmark. There are two tests: compressing and decompressing + with LZMA method. Benchmark shows rating in MIPS (million + instructions per second). Rating value is calculated from + measured speed and it is normalized with AMD Athlon 64 X2 CPU + results. Also Benchmark checks possible hardware errors (RAM + errors in most cases). Benchmark uses these settings: + (-a1, -d21, -fb32, -mfbt4). You can change only -d. Also you + can change number of iterations. Example for 30 iterations: + LZMA b 30 + Default number of iterations is 10. + + + + + -a{N}: set compression mode 0 = fast, 1 = normal + default: 1 (normal) + + d{N}: Sets Dictionary size - [0, 30], default: 23 (8MB) + The maximum value for dictionary size is 1 GB = 2^30 bytes. + Dictionary size is calculated as DictionarySize = 2^N bytes. + For decompressing file compressed by LZMA method with dictionary + size D = 2^N you need about D bytes of memory (RAM). + + -fb{N}: set number of fast bytes - [5, 273], default: 128 + Usually big number gives a little bit better compression ratio + and slower compression process. + + -lc{N}: set number of literal context bits - [0, 8], default: 3 + Sometimes lc=4 gives gain for big files. + + -lp{N}: set number of literal pos bits - [0, 4], default: 0 + lp switch is intended for periodical data when period is + equal 2^N. For example, for 32-bit (4 bytes) + periodical data you can use lp=2. Often it's better to set lc0, + if you change lp switch. + + -pb{N}: set number of pos bits - [0, 4], default: 2 + pb switch is intended for periodical data + when period is equal 2^N. + + -mf{MF_ID}: set Match Finder. Default: bt4. + Algorithms from hc* group doesn't provide good compression + ratio, but they often works pretty fast in combination with + fast mode (-a0). + + Memory requirements depend from dictionary size + (parameter "d" in table below). + + MF_ID Memory Description + + bt2 d * 9.5 + 4MB Binary Tree with 2 bytes hashing. + bt3 d * 11.5 + 4MB Binary Tree with 3 bytes hashing. + bt4 d * 11.5 + 4MB Binary Tree with 4 bytes hashing. + hc4 d * 7.5 + 4MB Hash Chain with 4 bytes hashing. + + -eos: write End Of Stream marker. By default LZMA doesn't write + eos marker, since LZMA decoder knows uncompressed size + stored in .lzma file header. + + -si: Read data from stdin (it will write End Of Stream marker). + -so: Write data to stdout + + +Examples: + +1) LZMA e file.bin file.lzma -d16 -lc0 + +compresses file.bin to file.lzma with 64 KB dictionary (2^16=64K) +and 0 literal context bits. -lc0 allows to reduce memory requirements +for decompression. + + +2) LZMA e file.bin file.lzma -lc0 -lp2 + +compresses file.bin to file.lzma with settings suitable +for 32-bit periodical data (for example, ARM or MIPS code). + +3) LZMA d file.lzma file.bin + +decompresses file.lzma to file.bin. + + +Compression ratio hints +----------------------- + +Recommendations +--------------- + +To increase compression ratio for LZMA compressing it's desirable +to have aligned data (if it's possible) and also it's desirable to locate +data in such order, where code is grouped in one place and data is +grouped in other place (it's better than such mixing: code, data, code, +data, ...). + + +Using Filters +------------- +You can increase compression ratio for some data types, using +special filters before compressing. For example, it's possible to +increase compression ratio on 5-10% for code for those CPU ISAs: +x86, IA-64, ARM, ARM-Thumb, PowerPC, SPARC. + +You can find C/C++ source code of such filters in folder "7zip/Compress/Branch" + +You can check compression ratio gain of these filters with such +7-Zip commands (example for ARM code): +No filter: + 7z a a1.7z a.bin -m0=lzma + +With filter for little-endian ARM code: + 7z a a2.7z a.bin -m0=bc_arm -m1=lzma + +With filter for big-endian ARM code (using additional Swap4 filter): + 7z a a3.7z a.bin -m0=swap4 -m1=bc_arm -m2=lzma + +It works in such manner: +Compressing = Filter_encoding + LZMA_encoding +Decompressing = LZMA_decoding + Filter_decoding + +Compressing and decompressing speed of such filters is very high, +so it will not increase decompressing time too much. +Moreover, it reduces decompression time for LZMA_decoding, +since compression ratio with filtering is higher. + +These filters convert CALL (calling procedure) instructions +from relative offsets to absolute addresses, so such data becomes more +compressible. Source code of these CALL filters is pretty simple +(about 20 lines of C++), so you can convert it from C++ version yourself. + +For some ISAs (for example, for MIPS) it's impossible to get gain from such filter. + + +LZMA compressed file format +--------------------------- +Offset Size Description + 0 1 Special LZMA properties for compressed data + 1 4 Dictionary size (little endian) + 5 8 Uncompressed size (little endian). -1 means unknown size + 13 Compressed data + + +ANSI-C LZMA Decoder +~~~~~~~~~~~~~~~~~~~ + +To compile ANSI-C LZMA Decoder you can use one of the following files sets: +1) LzmaDecode.h + LzmaDecode.c + LzmaTest.c (fastest version) +2) LzmaDecode.h + LzmaDecodeSize.c + LzmaTest.c (old size-optimized version) +3) LzmaStateDecode.h + LzmaStateDecode.c + LzmaStateTest.c (zlib-like interface) + + +Memory requirements for LZMA decoding +------------------------------------- + +LZMA decoder doesn't allocate memory itself, so you must +allocate memory and send it to LZMA. + +Stack usage of LZMA decoding function for local variables is not +larger than 200 bytes. + +How To decompress data +---------------------- + +LZMA Decoder (ANSI-C version) now supports 5 interfaces: +1) Single-call Decompressing +2) Single-call Decompressing with input stream callback +3) Multi-call Decompressing with output buffer +4) Multi-call Decompressing with input callback and output buffer +5) Multi-call State Decompressing (zlib-like interface) + +Variant-5 is similar to Variant-4, but Variant-5 doesn't use callback functions. + +Decompressing steps +------------------- + +1) read LZMA properties (5 bytes): + unsigned char properties[LZMA_PROPERTIES_SIZE]; + +2) read uncompressed size (8 bytes, little-endian) + +3) Decode properties: + + CLzmaDecoderState state; /* it's 24-140 bytes structure, if int is 32-bit */ + + if (LzmaDecodeProperties(&state.Properties, properties, LZMA_PROPERTIES_SIZE) != LZMA_RESULT_OK) + return PrintError(rs, "Incorrect stream properties"); + +4) Allocate memory block for internal Structures: + + state.Probs = (CProb *)malloc(LzmaGetNumProbs(&state.Properties) * sizeof(CProb)); + if (state.Probs == 0) + return PrintError(rs, kCantAllocateMessage); + + LZMA decoder uses array of CProb variables as internal structure. + By default, CProb is unsigned_short. But you can define _LZMA_PROB32 to make + it unsigned_int. It can increase speed on some 32-bit CPUs, but memory + usage will be doubled in that case. + + +5) Main Decompressing + +You must use one of the following interfaces: + +5.1 Single-call Decompressing +----------------------------- +When to use: RAM->RAM decompressing +Compile files: LzmaDecode.h, LzmaDecode.c +Compile defines: no defines +Memory Requirements: + - Input buffer: compressed size + - Output buffer: uncompressed size + - LZMA Internal Structures (~16 KB for default settings) + +Interface: + int res = LzmaDecode(&state, + inStream, compressedSize, &inProcessed, + outStream, outSize, &outProcessed); + + +5.2 Single-call Decompressing with input stream callback +-------------------------------------------------------- +When to use: File->RAM or Flash->RAM decompressing. +Compile files: LzmaDecode.h, LzmaDecode.c +Compile defines: _LZMA_IN_CB +Memory Requirements: + - Buffer for input stream: any size (for example, 16 KB) + - Output buffer: uncompressed size + - LZMA Internal Structures (~16 KB for default settings) + +Interface: + typedef struct _CBuffer + { + ILzmaInCallback InCallback; + FILE *File; + unsigned char Buffer[kInBufferSize]; + } CBuffer; + + int LzmaReadCompressed(void *object, const unsigned char **buffer, SizeT *size) + { + CBuffer *bo = (CBuffer *)object; + *buffer = bo->Buffer; + *size = MyReadFile(bo->File, bo->Buffer, kInBufferSize); + return LZMA_RESULT_OK; + } + + CBuffer g_InBuffer; + + g_InBuffer.File = inFile; + g_InBuffer.InCallback.Read = LzmaReadCompressed; + int res = LzmaDecode(&state, + &g_InBuffer.InCallback, + outStream, outSize, &outProcessed); + + +5.3 Multi-call decompressing with output buffer +----------------------------------------------- +When to use: RAM->File decompressing +Compile files: LzmaDecode.h, LzmaDecode.c +Compile defines: _LZMA_OUT_READ +Memory Requirements: + - Input buffer: compressed size + - Buffer for output stream: any size (for example, 16 KB) + - LZMA Internal Structures (~16 KB for default settings) + - LZMA dictionary (dictionary size is encoded in stream properties) + +Interface: + + state.Dictionary = (unsigned char *)malloc(state.Properties.DictionarySize); + + LzmaDecoderInit(&state); + do + { + LzmaDecode(&state, + inBuffer, inAvail, &inProcessed, + g_OutBuffer, outAvail, &outProcessed); + inAvail -= inProcessed; + inBuffer += inProcessed; + } + while you need more bytes + + see LzmaTest.c for more details. + + +5.4 Multi-call decompressing with input callback and output buffer +------------------------------------------------------------------ +When to use: File->File decompressing +Compile files: LzmaDecode.h, LzmaDecode.c +Compile defines: _LZMA_IN_CB, _LZMA_OUT_READ +Memory Requirements: + - Buffer for input stream: any size (for example, 16 KB) + - Buffer for output stream: any size (for example, 16 KB) + - LZMA Internal Structures (~16 KB for default settings) + - LZMA dictionary (dictionary size is encoded in stream properties) + +Interface: + + state.Dictionary = (unsigned char *)malloc(state.Properties.DictionarySize); + + LzmaDecoderInit(&state); + do + { + LzmaDecode(&state, + &bo.InCallback, + g_OutBuffer, outAvail, &outProcessed); + } + while you need more bytes + + see LzmaTest.c for more details: + + +5.5 Multi-call State Decompressing (zlib-like interface) +------------------------------------------------------------------ +When to use: file->file decompressing +Compile files: LzmaStateDecode.h, LzmaStateDecode.c +Compile defines: +Memory Requirements: + - Buffer for input stream: any size (for example, 16 KB) + - Buffer for output stream: any size (for example, 16 KB) + - LZMA Internal Structures (~16 KB for default settings) + - LZMA dictionary (dictionary size is encoded in stream properties) + +Interface: + + state.Dictionary = (unsigned char *)malloc(state.Properties.DictionarySize); + + + LzmaDecoderInit(&state); + do + { + res = LzmaDecode(&state, + inBuffer, inAvail, &inProcessed, + g_OutBuffer, outAvail, &outProcessed, + finishDecoding); + inAvail -= inProcessed; + inBuffer += inProcessed; + } + while you need more bytes + + see LzmaStateTest.c for more details: + + +6) Free all allocated blocks + + +Note +---- +LzmaDecodeSize.c is size-optimized version of LzmaDecode.c. +But compiled code of LzmaDecodeSize.c can be larger than +compiled code of LzmaDecode.c. So it's better to use +LzmaDecode.c in most cases. + + +EXIT codes +----------- + +LZMA decoder can return one of the following codes: + +#define LZMA_RESULT_OK 0 +#define LZMA_RESULT_DATA_ERROR 1 + +If you use callback function for input data and you return some +error code, LZMA Decoder also returns that code. + + + +LZMA Defines +------------ + +_LZMA_IN_CB - Use callback for input data + +_LZMA_OUT_READ - Use read function for output data + +_LZMA_LOC_OPT - Enable local speed optimizations inside code. + _LZMA_LOC_OPT is only for LzmaDecodeSize.c (size-optimized version). + _LZMA_LOC_OPT doesn't affect LzmaDecode.c (speed-optimized version) + and LzmaStateDecode.c + +_LZMA_PROB32 - It can increase speed on some 32-bit CPUs, + but memory usage will be doubled in that case + +_LZMA_UINT32_IS_ULONG - Define it if int is 16-bit on your compiler + and long is 32-bit. + +_LZMA_SYSTEM_SIZE_T - Define it if you want to use system's size_t. + You can use it to enable 64-bit sizes supporting + + + +C++ LZMA Encoder/Decoder +~~~~~~~~~~~~~~~~~~~~~~~~ +C++ LZMA code use COM-like interfaces. So if you want to use it, +you can study basics of COM/OLE. + +By default, LZMA Encoder contains all Match Finders. +But for compressing it's enough to have just one of them. +So for reducing size of compressing code you can define: + #define COMPRESS_MF_BT + #define COMPRESS_MF_BT4 +and it will use only bt4 match finder. + + +--- + +http://www.7-zip.org +http://www.7-zip.org/support.html diff --git a/make_mingw.sh b/make_mingw.sh new file mode 100644 index 0000000..cf96a5f --- /dev/null +++ b/make_mingw.sh @@ -0,0 +1,51 @@ +#!/bin/sh +# +# +# The easiest way to build AFFLIB for Win32 is to ues MinGW as either a cross-compiler +# or as a native compiler. In either case you will need to install OpenSSL and optionally +# libregex. +# +# This file is a work of a US government employee and as such is in the Public domain. +# Simson L. Garfinkel, March 12, 2012 +# +# This script will compile AFFLIB for Win32 on either Mac or Linux. + +# To install OpenSSL on On Linux, first download OpenSSL 1.0.0 and then: +# ./Configure mingw --prefix=/usr/i586-mingw32msvc/ +# sudo make install + +if test -r /opt/local/bin/i386-mingw32-gcc ; then + echo Compiling for mingw on a Mac installed with MacPorts + MBIN=/opt/local/bin/ + MPREFIX=/opt/local/i386-mingw32 + export CC=$MBIN/i386-mingw32-gcc + export CXX=$MBIN/i386-mingw32-g++ + export RANLIB=$MBIN/i386-mingw32-ranlib + export AR=$MBIN/i386-mingw32-ar + export PREFIX=/opt/local/i386-mingw32/ + export MINGWFLAGS="-mwin32 -mconsole -march=pentium4 " + export CFLAGS="$MINGWFLAGS" + export CXXFLAGS="$MINGWFLAGS" +fi +if test -r /usr/i586-mingw32msvc ; then + echo Compiling for mingw on Linux + export PREFIX=/usr/i586-mingw32msvc + export MINGW32PATH=/usr/i586-mingw32msvc + export CC=/usr/bin/i586-mingw32msvc-gcc + export CXX=/usr/bin/i586-mingw32msvc-g++ + export AR=${MINGW32PATH}/bin/ar + export RANLIB=${MINGW32PATH}/bin/ranlib + export STRIP=${MINGW32PATH}/bin/strip + export MINGWFLAGS="-mwin32 -mconsole -march=i586 " + export CFLAGS="$MINGWFLAGS" + export CXXFLAGS="$MINGWFLAGS" +fi + +autoreconf -f +./configure CC=$CC CXX=$CXX RANLIB=$RANLIB --target=i586-mingw32msvc --host=i586 --enable-winapi=yes --prefix=$PREFIX +make CC=$CC CXX=$CXX RANLIB=$RANLIB CFLAGS="$CFLAGS" CXXFLAGS="$CXXFLAGS" + +# Make a release of the executables + +zip afflib_windows.zip tools/*.exe + diff --git a/man/Makefile.am b/man/Makefile.am new file mode 100644 index 0000000..0fdbc35 --- /dev/null +++ b/man/Makefile.am @@ -0,0 +1,4 @@ +# This file is a work of a US government employee and as such is in the Public domain. +# Simson L. Garfinkel, March 12, 2012 + +dist_man_MANS = affcat.1 diff --git a/man/affcat.1 b/man/affcat.1 new file mode 100644 index 0000000..57425db --- /dev/null +++ b/man/affcat.1 @@ -0,0 +1,48 @@ +.\" Process this file with +.\" groff -man -Tascii foo.1 +.\" +.TH AFCAT 1 "OCT 2008" "User Manuals" +.SH NAME +afcat \- Output contents of an image file to stdout. +.SH SYNOPSIS +.B afcat [options] +.I image [images] +.SH DESCRIPTION +.B affcat +outputs the contents of an image file to stdout. Image files that are not raw but are recognized +by AFF will be output in raw format. Missing pages will not be padded, but the fact that they are missing +will be noted on STDERR. + +The options are as follows: +.IP "-s name" +Output the named segment, instead of the image data. This is a way to output metadata. +.IP "-p nnn" +Just output page number +.B nnn +.IP "-S nnn" +Just output data sector number +.B nnn. +Sector #0 is the first sector. +.IP "-q" +Quiet mode. Don't print to STDERR if a page is skipped because it is not present. +.IP "-n" +Noisy mode. Tell when pages are skipped. +.IP "-l" +List all the segment names, rather than outputing data. +.IP "-L" +List Long. Prints segment names, lengths, and args. +.IP "-d" +Debug mode. Print the page numbers to stderr as data goes to stdout. +.IP "-b" +Output BADFALG for bad blocks (default is to output NULLs). +.IP "-v" +Just print the version number and exit. +.IP "-r offset:count" +Seek to the given byte offset and output count characters in each file; may be repeated. +.SH HISTORY +.BR "afcat" " first appeared in " "AFFLIB" " v1.00." +.SH AUTHOR +Simson Garfinkel +.SH COPYRIGHT +This file is a work of a US government employee and as such is in the Public domain. + diff --git a/pyaff/Makefile.am b/pyaff/Makefile.am new file mode 100644 index 0000000..cd23e60 --- /dev/null +++ b/pyaff/Makefile.am @@ -0,0 +1,13 @@ +EXTRA_DIST = PKG-INFO README pyaff.c setup.py + +if HAVE_PYTHON +AM_CFLAGS = -I$(top_srcdir)/lib -include affconfig.h + +# This is for the pyaff python module +pyexec_LTLIBRARIES = pyaff.la + +pyaff_la_SOURCES = pyaff.c +pyaff_la_LIBADD = ../lib/libafflib.la +pyaff_la_CPPFLAGS = $(PYTHON_CPPFLAGS) +pyaff_la_LDFLAGS = -module -avoid-version $(PYTHON_LDFLAGS) +endif diff --git a/pyaff/PKG-INFO b/pyaff/PKG-INFO new file mode 100644 index 0000000..fc009ab --- /dev/null +++ b/pyaff/PKG-INFO @@ -0,0 +1,10 @@ +Metadata-Version: 1.0 +Name: PyAFF +Version: 0.1 +Summary: Python wrapper for AFFLIB +Home-page: www.pyflag.net +Author: David Collett +Author-email: david.collett@gmail.com +License: GPL +Description: UNKNOWN +Platform: UNKNOWN diff --git a/pyaff/README b/pyaff/README new file mode 100644 index 0000000..e6324ae --- /dev/null +++ b/pyaff/README @@ -0,0 +1,30 @@ +================================== +PyAFF - Python bindings for AFFLIB +================================== + +These bindings currently support a read-only file-like interface to AFFLIB and +basic metadata accessor functions. The binding is not currently complete. + +-------- +BUILDING +-------- + +python setup.py build +python setup.py install + +----- +USAGE +----- + +Basic usage example: + + #/usr/bin/python + import pyaff + + fd = pyaff("diskimage.aff") + data = fd.read(1000) + fd.seek(0, SEEK_SET) + + print fd.get_seg_names() + print fd.get_seg("afflib_version") + diff --git a/pyaff/pyaff.c b/pyaff/pyaff.c new file mode 100644 index 0000000..c40b5d0 --- /dev/null +++ b/pyaff/pyaff.c @@ -0,0 +1,279 @@ +/****************************************************** + * Copyright 2008: David Collett + + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * (LGPL) as published by the Free Software Foundation; either version + * 3 of the License as of 29 June 2007, or (at your option) any later + * version. + * + * See http://www.gnu.org/licenses/lgpl.txt + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02 + * -1307, USA. + ****************************************************/ + +#include "Python.h" +#include "lib/afflib.h" + +#include +#include +#include +#include +#include + +/****************************************************************** + * pyaff - afflib python binding + * ***************************************************************/ + +typedef struct { + PyObject_HEAD + AFFILE *af; + uint64_t size; +} affile; + +static void affile_dealloc(affile *self); +static int affile_init(affile *self, PyObject *args, PyObject *kwds); +static PyObject *affile_read(affile *self, PyObject *args, PyObject *kwds); +static PyObject *affile_seek(affile *self, PyObject *args, PyObject *kwds); +static PyObject *affile_get_seg(affile *self, PyObject *args, PyObject *kwds); +static PyObject *affile_get_seg_names(affile *self); +static PyObject *affile_tell(affile *self); +static PyObject *affile_close(affile *self); + +static PyMethodDef affile_methods[] = { + {"read", (PyCFunction)affile_read, METH_VARARGS|METH_KEYWORDS, + "Read data from file" }, + {"seek", (PyCFunction)affile_seek, METH_VARARGS|METH_KEYWORDS, + "Seek within a file" }, + {"get_seg", (PyCFunction)affile_get_seg, METH_VARARGS|METH_KEYWORDS, + "Retrieve an aff segment by name" }, + {"get_seg_names", (PyCFunction)affile_get_seg_names, METH_NOARGS, + "Retrieve a list of segments present" }, + {"tell", (PyCFunction)affile_tell, METH_NOARGS, + "Return possition within file" }, + {"close", (PyCFunction)affile_close, METH_NOARGS, + "Close the file" }, + {NULL} /* Sentinel */ +}; + +static PyTypeObject affileType = { + PyObject_HEAD_INIT(NULL) + 0, /* ob_size */ + "pyaff.affile", /* tp_name */ + sizeof(affile), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)affile_dealloc,/* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "afflib File Object", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + affile_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)affile_init, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; + +static void +affile_dealloc(affile *self) { + self->ob_type->tp_free((PyObject*)self); +} + +static int +affile_init(affile *self, PyObject *args, PyObject *kwds) { + char *filename; + static char *kwlist[] = {"filename", NULL}; + + self->size = 0; + + if(!PyArg_ParseTupleAndKeywords(args, kwds, "s", kwlist, &filename)) + return -1; + + self->af = af_open(filename, O_RDONLY, 0); + if(self->af == NULL) { + PyErr_Format(PyExc_IOError, "Failed to initialise afflib"); + return -1; + } + + self->size = af_get_imagesize(self->af); + return 0; +} + +static PyObject * +affile_read(affile *self, PyObject *args, PyObject *kwds) { + int written; + PyObject *retdata; + int readlen=-1; + + static char *kwlist[] = {"size", NULL}; + if(!PyArg_ParseTupleAndKeywords(args, kwds, "|i", kwlist, &readlen)) + return NULL; + + if(readlen < 0 || readlen > self->size) + readlen = self->size; + + retdata = PyString_FromStringAndSize(NULL, readlen); + written = af_read(self->af, (unsigned char *)PyString_AsString(retdata), readlen); + + if(readlen != written) { + return PyErr_Format(PyExc_IOError, "Failed to read all data: wanted %d, got %d", readlen, written); + } + + return retdata; +} + +static PyObject * +affile_seek(affile *self, PyObject *args, PyObject *kwds) { + int64_t offset=0; + int whence=0; + + static char *kwlist[] = {"offset", "whence", NULL}; + if(!PyArg_ParseTupleAndKeywords(args, kwds, "L|i", kwlist, + &offset, &whence)) + return NULL; + + if(af_seek(self->af, offset, whence) < 0) + return PyErr_Format(PyExc_IOError, "libaff_seek_offset failed"); + + Py_RETURN_NONE; +} + +static PyObject * +affile_tell(affile *self) { + return PyLong_FromLongLong(af_tell(self->af)); +} + +static PyObject * +affile_close(affile *self) { + af_close(self->af); + Py_RETURN_NONE; +} + +static PyObject *affile_get_seg(affile *self, PyObject *args, PyObject *kwds) { + PyObject *retdata; + char *buf; + size_t buflen=0; + char *segname=NULL; + static char *kwlist[] = {"segname", NULL}; + + if(!PyArg_ParseTupleAndKeywords(args, kwds, "s", kwlist, &segname)) + return NULL; + + // first get the size + if(af_get_seg(self->af, segname, 0, 0, &buflen) != 0) { + return PyErr_Format(PyExc_IOError, "error reading libaff segment\n"); + } + + // allocate a string to return data in + retdata = PyString_FromStringAndSize(NULL, buflen); + buf = PyString_AsString(retdata); + + if(af_get_seg(self->af, segname, 0, (unsigned char *)buf, &buflen) != 0) { + Py_DECREF(retdata); + return PyErr_Format(PyExc_IOError, "error reading libaff segment\n"); + } + + return retdata; +} + +static PyObject *affile_get_seg_names(affile *self) { + PyObject *headers, *tmp; + char segname[AF_MAX_NAME_LEN]; + + af_rewind_seg(self->af); + headers = PyList_New(0); + + while(af_get_next_seg(self->af, segname, sizeof(segname), 0, 0, 0) == 0){ + tmp = PyString_FromString(segname); + PyList_Append(headers, tmp); + Py_DECREF(tmp); + } + + return headers; +} + +static PyObject *pyaff_open(PyObject *self, PyObject *args, PyObject *kwds) { + int ret; + affile *file; + PyObject *files, *fileargs, *filekwds; + static char *kwlist[] = {"filename", NULL}; + + if(!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &files)) + return NULL; + + /* create an affile object and return it */ + fileargs = PyTuple_New(0); + filekwds = Py_BuildValue("{sO}", "filename", files); + if(!filekwds) return NULL; + + file = PyObject_New(affile, &affileType); + ret = affile_init(file, fileargs, filekwds); + Py_DECREF(fileargs); + Py_DECREF(filekwds); + + if(ret == -1) { + Py_DECREF(file); + return NULL; + } + return (PyObject *)file; +} + +/* these are the module methods */ +static PyMethodDef pyaff_methods[] = { + {"open", (PyCFunction)pyaff_open, METH_VARARGS|METH_KEYWORDS, + "Open afflib file (or set of files)" }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ +#define PyMODINIT_FUNC void +#endif +PyMODINIT_FUNC +initpyaff(void) +{ + PyObject* m; + + /* create module */ + m = Py_InitModule3("pyaff", pyaff_methods, "Python libaff module."); + + /* setup affile type */ + affileType.tp_new = PyType_GenericNew; + if (PyType_Ready(&affileType) < 0) + return; + + Py_INCREF(&affileType); + PyModule_AddObject(m, "affile", (PyObject *)&affileType); +} + diff --git a/pyaff/setup.py b/pyaff/setup.py new file mode 100644 index 0000000..a5ccb30 --- /dev/null +++ b/pyaff/setup.py @@ -0,0 +1,14 @@ +from distutils.core import setup, Extension + +pyaff = Extension('pyaff', + libraries = ['afflib'], + sources = ['pyaff.c']) + +setup (name = 'PyAFF', + version = '0.1', + description = 'Python wrapper for AFFLIB', + author = 'David Collett', + author_email = 'david.collett@gmail.com', + url = 'www.pyflag.net', + license = "GPL", + ext_modules = [pyaff]) diff --git a/stats/lzstats.py b/stats/lzstats.py new file mode 100644 index 0000000..4e7598f --- /dev/null +++ b/stats/lzstats.py @@ -0,0 +1,97 @@ +import sys,os; + +sys.path.append(os.getenv("HOME") + "/slg/src/python") + +from statbag import statbag + +drives = {} # hash table for the drives +gb = 1000*1000*1000 + +class dr: + pass + +def process_afcompare(fn): + import re + global null_drives,total_nochg,total_null,total_lzma,total_drives,drives_larger_1g + m = re.compile("\d\d\d\d[.]aff") + for line in open(fn,"r"): + r = m.search(line) + drive = r.group(0) + d = dr() + fields = line.split(" ") + while "" in fields: fields.remove("") + d.nochg = int(fields[4]) + d.nul = int(fields[6]) + d.lzma = int(fields[8]) + d.old_bytes = int(fields[10]) + d.new_bytes = int(fields[12]) + drives[drive] = d + +def process_afreport(fn): + import re + m = re.compile("\d\d\d\d[.]aff") + for line in open(fn,"r"): + r = m.search(line) + try: + drive = r.group(0) + if drive not in drives: continue + d = drives[drive] + line = line.replace("\t"," ") + fields = line.split(" ") + while "" in fields: fields.remove("") + d.imagesize = int(fields[1]) + d.compressed = int(fields[2]) + d.uncompressed = int(fields[3]) + print d.iamgesize,d.compressed,d.uncompressed + except AttributeError: + pass + + +if(__name__=="__main__"): + import glob + for fn in glob.glob("*afcompare*"): + process_afcompare(fn) + for fn in glob.glob("*-report*"): + process_afreport(fn) + + null_drives =0 + + big = {} + + for (fn,d) in drives.iteritems(): + if(d.nochg==0 and d.nul>=0 and d.lzma==0): + null_drives += 1 + continue + if(d.uncompressed b.uncompressed: return 1 + return 0 + + fns = big.keys() + fns.sort(myfunc) + avg_savings = statbag(); + for fn in fns: + d = big[fn] + savings = 100.0 - (100.0 * d.new_bytes/d.old_bytes) + if(d.old_bytes/1000000 < 10): continue + print "%8s %8d %8d %8d %5.2f%%" % ( + fn,d.uncompressed/1000000,d.old_bytes/1000000,d.new_bytes/1000000,savings) + avg_savings.addx(savings) + print "\nAverage savings: %5.2f%%" % avg_savings.average() diff --git a/stats/project1-report.txt b/stats/project1-report.txt new file mode 100644 index 0000000..daa269c --- /dev/null +++ b/stats/project1-report.txt @@ -0,0 +1,602 @@ +Name AF_IMAGESIZE Compressed Uncompressed Blank Bad +/access/p1/0001.aff 172922880 136730357 172922880 0 0 +/access/p1/0002.aff 730791936 176 730791936 0 0 +/access/p1/0003.aff 426131456 219766 426131456 0 0 +/access/p1/0004.aff 341299200 275920 341299200 0 0 +/access/p1/0005.aff 130351104 43711358 130351104 0 0 +/access/p1/0006.aff 131334144 42693709 131334144 0 0 +/access/p1/0007.aff 127533056 39498713 127533056 0 0 +/access/p1/0008.aff 170818560 27892775 170818560 0 0 +/access/p1/0009.aff 43130880 8519947 43130880 0 0 +/access/p1/0010.aff 127983616 42540127 127983616 0 0 +/access/p1/0011.aff 245426688 74462210 245426688 0 0 +/access/p1/0012.aff 540868608 77977486 540868608 0 0 +/access/p1/0013.aff 541900800 102128698 541900800 0 0 +/access/p1/0014.aff 426295296 201843315 426295296 0 0 +/access/p1/0015.aff 428067840 140915737 428067840 0 0 +/access/p1/0016.aff 524869632 172684857 524869632 0 0 +/access/p1/0017.aff 270663680 113085752 270663680 0 0 +/access/p1/0018.aff 212615168 70911529 212615168 0 0 +/access/p1/0019.aff 154650624 34938534 154650624 0 0 +/access/p1/0020.aff 541900800 1134831 541900800 0 0 +/access/p1/0021.aff 1281982464 95174080 1281982464 0 0 +/access/p1/0022.aff 1082253312 2237706 1082253312 0 0 +/access/p1/0023.aff 375017984 200490882 375017984 0 0 +/access/p1/0024.aff 343203840 108223065 343203840 0 0 +/access/p1/0025.aff 343203840 87932042 343203840 0 0 +/access/p1/0026.aff 121798656 24586983 121798656 0 0 +/access/p1/0027.aff 170139648 98547626 170139648 0 0 +/access/p1/0028.aff 1281982464 311593870 1281982464 0 0 +/access/p1/0029.aff 127983616 30592446 127983616 0 0 +/access/p1/0030.aff 212615168 26675133 212615168 0 0 +/access/p1/0031.aff 127233024 21343686 127233024 0 0 +/access/p1/0032.aff 155560960 39690470 155560960 0 0 +/access/p1/0033.aff 170818560 19432932 170818560 0 0 +/access/p1/0034.aff 41559040 12 41559040 0 0 +/access/p1/0035.aff 212615168 36335494 212615168 0 0 +/access/p1/0036.aff 156013056 13290732 156013056 0 0 +/access/p1/0037.aff 88656896 29604495 88656896 0 0 +/access/p1/0039.aff 114853376 33459460 114853376 0 0 +/access/p1/0040.aff 1083801600 425793233 1083801600 0 0 +/access/p1/0041.aff 1083801600 224919523 1083801600 0 0 +/access/p1/0042.aff 122247680 14916431 122247680 0 0 +/access/p1/0044.aff 251876352 155134664 251876352 0 0 +/access/p1/0045.aff 52293632 1166194 52293632 0 0 +/access/p1/0047.aff 1629831168 795603008 1629831168 0 0 +/access/p1/0048.aff 853622784 294973480 853622784 0 0 +/access/p1/0049.aff 541327360 145734353 541327360 0 0 +/access/p1/0050.aff 1285079040 43261421 1285079040 0 0 +/access/p1/0051.aff 2111864832 728137651 2111864832 0 0 +/access/p1/0052.aff 1281982464 600818154 1281982464 0 0 +/access/p1/0053.aff 223605760 60375296 223605760 0 0 +/access/p1/0054.aff 365395968 130057061 365395968 0 0 +/access/p1/0055.aff 528479232 165778351 528479232 0 0 +/access/p1/0056.aff 853622784 190850554 853622784 0 0 +/access/p1/0057.aff 542932992 196720942 542932992 0 0 +/access/p1/0058.aff 545513472 211576242 545513472 0 0 +/access/p1/0059.aff 426295296 64453491 426295296 0 0 +/access/p1/0060.aff 2559836160 435967550 2559836160 0 0 +/access/p1/0061.aff 223764992 58184006 223764992 0 0 +/access/p1/0062.aff 1281982464 311594834 1281982464 0 0 +/access/p1/0063.aff 636862464 178256299 636862464 0 0 +/access/p1/0064.aff 426295296 223235804 426295296 0 0 +/access/p1/0065.aff 428067840 33387339 428067840 0 0 +/access/p1/0066.aff 2111864832 1367081390 2111864832 0 0 +/access/p1/0067.aff 2111864832 223155106 2111864832 0 0 +/access/p1/0068.aff 544997376 168745841 544997376 0 0 +/access/p1/0069.aff 426295296 181892635 426295296 0 0 +/access/p1/0070.aff 541384704 104269515 541384704 0 0 +/access/p1/0072.aff 270663680 68 270663680 0 0 +/access/p1/0073.aff 2160377856 970873830 2160377856 0 0 +/access/p1/0074.aff 730791936 244692392 730791936 0 0 +/access/p1/0075.aff 730791936 238165184 730791936 0 0 +/access/p1/0076.aff 853622784 272756106 853622784 0 0 +/access/p1/0077.aff 540868608 189804181 540868608 0 0 +/access/p1/0078.aff 730791936 219137752 730791936 0 0 +/access/p1/0079.aff 1083801600 465819236 1083801600 0 0 +/access/p1/0080.aff 1083801600 411123666 1083801600 0 0 +/access/p1/0082.aff 170818560 44 170818560 0 0 +/access/p1/0083.aff 85038080 24 85038080 0 0 +/access/p1/0084.aff 211000320 52 211000320 0 0 +/access/p1/0085.aff 85038080 24 85038080 0 0 +/access/p1/0086.aff 85038080 24 85038080 0 0 +/access/p1/0087.aff 496052224 120 496052224 0 0 +/access/p1/0088.aff 85038080 24 85038080 0 0 +/access/p1/0089.aff 98076160 27016413 98076160 0 0 +/access/p1/0091.aff 17171968 8 17171968 0 0 +/access/p1/0092.aff 2111864832 221423 2111864832 0 0 +/access/p1/0094.aff 535824384 173799603 535824384 0 0 +/access/p1/0095.aff 1281982464 155792910 1281982464 0 0 +/access/p1/0096.aff 853622784 303566996 853622784 0 0 +/access/p1/0097.aff 540868608 199561786 540868608 0 0 +/access/p1/0098.aff 1706213376 821307583 1706213376 0 0 +/access/p1/0099.aff 642023424 204326233 642023424 0 0 +/access/p1/0100.aff 642023424 187842825 642023424 0 0 +/access/p1/0101.aff 638926848 890976 638926848 0 0 +/access/p1/0103.aff 642023424 207844840 642023424 0 0 +/access/p1/0104.aff 540868608 170471133 540868608 0 0 +/access/p1/0105.aff 642023424 106167352 642023424 0 0 +/access/p1/0106.aff 540868608 160571071 540868608 0 0 +/access/p1/0107.aff 730791936 258299554 730791936 0 0 +/access/p1/0109.aff 642023424 232203964 642023424 0 0 +/access/p1/0110.aff 639959040 313828550 639959040 0 0 +/access/p1/0111.aff 213909504 52 213909504 0 0 +/access/p1/0112.aff 730791936 253228790 730791936 0 0 +/access/p1/0113.aff 730791936 26441132 83886080 0 0 +/access/p1/0114.aff 545513472 260584626 545513472 0 0 +/access/p1/0115.aff 722018304 312324145 722018304 0 0 +/access/p1/0118.aff 170139648 61266792 170139648 0 0 +/access/p1/0121.aff 639959040 178345566 639959040 0 0 +/access/p1/0128.aff 2111864832 707252775 2111864832 0 0 +/access/p1/0133.aff 2111864832 433283155 2111864832 0 0 +/access/p1/0134.aff 545513472 43625429 545513472 0 0 +/access/p1/0138.aff 230309888 135727 230309888 0 0 +/access/p1/0139.aff 172922880 219831 172922880 0 0 +/access/p1/0140.aff 245571584 194214 245571584 0 0 +/access/p1/0141.aff 131334144 3329 131334144 0 0 +/access/p1/0142.aff 245571584 219573 245571584 0 0 +/access/p1/0143.aff 525271040 17758509 525271040 0 0 +/access/p1/0144.aff 245571584 194253 245571584 0 0 +/access/p1/0145.aff 245571584 3356 245571584 0 0 +/access/p1/0147.aff 172922880 2899 172922880 0 0 +/access/p1/0149.aff 212615168 3349 212615168 0 0 +/access/p1/0150.aff 172922880 3339 172922880 0 0 +/access/p1/0151.aff 212615168 3349 212615168 0 0 +/access/p1/0152.aff 245571584 3356 245571584 0 0 +/access/p1/0153.aff 245571584 2900 245571584 0 0 +/access/p1/0154.aff 212615168 3349 212615168 0 0 +/access/p1/0155.aff 172922880 3339 172922880 0 0 +/access/p1/0156.aff 131334144 3329 131334144 0 0 +/access/p1/0157.aff 172922880 3339 172922880 0 0 +/access/p1/0158.aff 125009920 3329 125009920 0 0 +/access/p1/0160.aff 4223729664 5960 4223729664 0 0 +/access/p1/0161.aff 576888832 334781779 576888832 0 0 +/access/p1/0166.aff 2111864832 215698 2111864832 0 0 +/access/p1/0168.aff 1080238080 225153053 1080238080 0 0 +/access/p1/0170.aff 431761408 85865656 431761408 0 0 +/access/p1/0171.aff 2104565760 168620799 2104565760 0 0 +/access/p1/0172.aff 2104565760 153775094 2104565760 0 0 +/access/p1/0176.aff 106522112 25398168 106522112 0 0 +/access/p1/0177.aff 2113413120 318288145 2113413120 0 0 +/access/p1/0178.aff 445057536 440290482 445057536 0 0 +/access/p1/0179.aff 2111864832 466335396 2111864832 0 0 +/access/p1/0181.aff 1974550528 1198653057 1974550528 0 0 +/access/p1/0182.aff 2111864832 650335095 2111864832 0 0 +/access/p1/0183.aff 1280950272 448794289 1280950272 0 0 +/access/p1/0184.aff 1280950272 230683374 1280950272 0 0 +/access/p1/0186.aff 1241972736 504348370 1241972736 0 0 +/access/p1/0187.aff 2111864832 1178133776 2111864832 0 0 +/access/p1/0190.aff 1280950272 236970703 1280950272 0 0 +/access/p1/0191.aff 170139648 66384379 170139648 0 0 +/access/p1/0193.aff 2559836160 625577173 2559836160 0 0 +/access/p1/0194.aff 544997376 256882298 544997376 0 0 +/access/p1/0195.aff 544997376 244445288 544997376 0 0 +/access/p1/0196.aff 100819456 33796260 100819456 0 0 +/access/p1/0197.aff 200678912 132656874 200678912 0 0 +/access/p1/0198.aff 1080735744 110441967 1080735744 0 0 +/access/p1/0199.aff 539836416 180258632 539836416 0 0 +/access/p1/0200.aff 541327360 37510214 541327360 0 0 +/access/p1/0201.aff 56041472 16510824 56041472 0 0 +/access/p1/0202.aff 1707761664 477182915 1707761664 0 0 +/access/p1/0204.aff 1281982464 390776314 1281982464 0 0 +/access/p1/0205.aff 1285079040 672027753 1285079040 0 0 +/access/p1/0206.aff 2111864832 984291295 2111864832 0 0 +/access/p1/0209.aff 2564997120 550202208 2564997120 0 0 +/access/p1/0210.aff 426295296 203689289 426295296 0 0 +/access/p1/0211.aff 213909504 96056124 213909504 0 0 +/access/p1/0212.aff 421956608 139051210 421956608 0 0 +/access/p1/0214.aff 270663680 49874747 270663680 0 0 +/access/p1/0215.aff 2559836160 4752 2559836160 0 0 +/access/p1/0216.aff 542932992 70941169 542932992 0 0 +/access/p1/0217.aff 2111864832 688370769 2111864832 0 0 +/access/p1/0219.aff 1080238080 59374722 1080238080 0 0 +/access/p1/0220.aff 1055287296 186303721 1055287296 0 0 +/access/p1/0221.aff 270663680 40041 270663680 0 0 +/access/p1/0222.aff 270663680 213627 270663680 0 0 +/access/p1/0224.aff 426295296 87554762 426295296 0 0 +/access/p1/0227.aff 213237760 159156686 213237760 0 0 +/access/p1/0231.aff 426295296 158877556 426295296 0 0 +/access/p1/0235.aff 112204800 33770953 112204800 0 0 +/access/p1/0237.aff 213909504 6012 33554432 0 6144 +/access/p1/0238.aff 213909504 6014 33554432 0 6144 +/access/p1/0241.aff 3145728 3096 3145728 0 6144 +/access/p1/0242.aff 853002240 8198177 50331648 2505 16384 +/access/p1/0245.aff 545513472 173306299 545513472 62622 0 +/access/p1/0246.aff 52293632 3229569 50331648 1342 6144 +/access/p1/0247.aff 4311465984 12250 436207616 813011 6656 +/access/p1/0248.aff 52293632 11534284 52293632 2773 0 +/access/p1/0249.aff 213417984 24003987 213417984 230576 0 +/access/p1/0250.aff 276368516096 6022 33554432 0 6144 +/access/p1/0251.aff 44728320 6014 33554432 0 6144 +/access/p1/0252.aff 4335206400 1497362059 4335206400 211715 0 +/access/p1/0256.aff 2431844352 3562 2431844352 4749696 0 +/access/p1/0258.aff 2431844352 395347595 2431844352 90738 0 +/access/p1/0259.aff 6448619520 12151 6448619520 12593385 6144 +/access/p1/0263.aff 2625005568 3611 2625005568 5126964 2048 +/access/p1/0264.aff 4311982080 4014 4311982080 8421840 0 +/access/p1/0265.aff 3240050688 6817541 3240050688 0 0 +/access/p1/0266.aff 2625005568 6622 2550136832 4933120 6144 +/access/p1/0267.aff 3239792640 6816712 3239792640 0 0 +/access/p1/0268.aff 1449197568 11578367 1449197568 2781782 0 +/access/p1/0269.aff 2167603200 62149 2167603200 4228834 0 +/access/p1/0270.aff 853622784 6019 33554432 0 6144 +/access/p1/0271.aff 853622784 6019 33554432 0 6144 +/access/p1/0272.aff 853622784 6012 33554432 0 6144 +/access/p1/0273.aff 50331648 8940 50331648 0 163840 +/access/p1/0275.aff 853622784 6019 33554432 0 6144 +/access/p1/0276.aff 853622784 6019 33554432 0 6144 +/access/p1/0277.aff 853622784 6019 33554432 0 6144 +/access/p1/0278.aff 853622784 6019 33554432 0 6144 +/access/p1/0279.aff 853622784 6017 33554432 0 6144 +/access/p1/0281.aff 853622784 6019 33554432 0 6144 +/access/p1/0283.aff 853622784 6019 33554432 0 6144 +/access/p1/0284.aff 853622784 6019 33554432 0 6144 +/access/p1/0286.aff 853622784 6019 33554432 0 6144 +/access/p1/0287.aff 853622784 6018 33554432 0 6144 +/access/p1/0289.aff 853622784 6021 33554432 0 6144 +/access/p1/0290.aff 4294817280 227725 4294817280 8387110 0 +/access/p1/0291.aff 4293632000 3455766128 4293632000 196299 0 +/access/p1/0292.aff 4293632000 2076061713 3472883712 335378 6144 +/access/p1/0293.aff 4311982080 4013 4311982080 8421840 0 +/access/p1/0295.aff 4311982080 4013 4311982080 8421840 0 +/access/p1/0300.aff 3249340416 1143326407 3249340416 0 0 +/access/p1/0301.aff 3166765056 708847162 3166765056 0 0 +/access/p1/0302.aff 3228696576 4398 3228696576 0 0 +/access/p1/0303.aff 3166765056 4383 3166765056 0 0 +/access/p1/0305.aff 20020396032 161757662 20020396032 0 0 +/access/p1/0308.aff 20416757760 990780197 20416757760 0 0 +/access/p1/0310.aff 900923392 432907738 900923392 0 0 +/access/p1/0312.aff 6716915712 1589668 6716915712 0 0 +/access/p1/0313.aff 20489961472 2930029996 20489961472 0 0 +/access/p1/0314.aff 20022951936 1850255643 20022951936 0 0 +/access/p1/0315.aff 20020396032 7569754325 20020396032 18539568 0 +/access/p1/0316.aff 6448619520 2047583831 6448619520 0 0 +/access/p1/0317.aff 6448619520 1576247368 6448619520 0 0 +/access/p1/0318.aff 106954752 11548994 100663296 1643 6144 +/access/p1/0319.aff 2111864832 14750279 83886080 2624 6144 +/access/p1/0320.aff 20547841536 934463630 20547841536 37288796 5514 +/access/p1/0321.aff 6448881664 1292080980 6448881664 0 512 +/access/p1/0322.aff 0 0 0 0 0 +/access/p1/0323.aff 6448619520 1097665864 6448619520 0 0 +/access/p1/0324.aff 6448619520 2289893443 6448619520 502717 0 +/access/p1/0325.aff 6448619520 1342108561 6448619520 292774 0 +/access/p1/0326.aff 6448619520 1054634212 6448619520 0 0 +/access/p1/0327.aff 4311982080 304267566 1056964608 0 6144 +/access/p1/0328.aff 6448619520 713742978 2432696320 1009961 6144 +/access/p1/0329.aff 1183318016 316620019 1183318016 0 7168 +/access/p1/0330.aff 126976 21279 126976 0 96 +/access/p1/0331.aff 2626682880 292540595 2626682880 0 7168 +/access/p1/0332.aff 2143027200 120329733 2143027200 0 6656 +/access/p1/0333.aff 6448619520 1201100251 6448619520 4617138 32768 +/access/p1/0334.aff 0 0 0 0 0 +/access/p1/0335.aff 0 243264931 0 0 0 +/access/p1/0338.aff 13578485760 2223725191 13578485760 4018957 0 +/access/p1/0339.aff 9056904704 976174308 9056904704 13017701 0 +/access/p1/0340.aff 9056904704 975167294 9056904704 13020648 0 +/access/p1/0341.aff 9056904704 3532052 9056904704 5138 0 +/access/p1/0342.aff 9056904704 977936233 9056904704 13020024 0 +/access/p1/0343.aff 9056904704 976144576 9056904704 13018955 0 +/access/p1/0344.aff 9056904704 3532564 9056904704 674 0 +/access/p1/0345.aff 9056904704 975798215 9056904704 13021381 0 +/access/p1/0346.aff 9056904704 977138339 9056904704 13016093 0 +/access/p1/0347.aff 9056904704 21086472 9056904704 56763 0 +/access/p1/0348.aff 9056904704 974590752 9056904704 13011314 6144 +/access/p1/0349.aff 9056904704 976585573 9056904704 13025199 0 +/access/p1/0350.aff 9056904704 974957061 9056904704 13022374 0 +/access/p1/0351.aff 9056904704 976308645 9056904704 13022215 0 +/access/p1/0354.aff 9056904704 977028433 9056904704 13019785 0 +/access/p1/0355.aff 9056904704 928435878 8455716864 12027246 6144 +/access/p1/0356.aff 9056904704 976016354 9056904704 13021868 0 +/access/p1/0358.aff 540868608 6017 33554432 0 6144 +/access/p1/0359.aff 270434304 6015 33554432 0 6144 +/access/p1/0360.aff 50331648 118902 50331648 26 163840 +/access/p1/0362.aff 1052090368 493417587 553648128 21678 9216 +/access/p1/0363.aff 426295296 222449331 426295296 53925 0 +/access/p1/0364.aff 426295296 47123088 426295296 581057 6144 +/access/p1/0365.aff 528482304 6015 33554432 0 6144 +/access/p1/0366.aff 1278400512 549195077 1278400512 290289 0 +/access/p1/0367.aff 730791936 6913 33554432 0 6144 +/access/p1/0370.aff 540868608 6016 33554432 0 6144 +/access/p1/0371.aff 561176576 251493231 561176576 47250 6144 +/access/p1/0372.aff 561176576 179823046 561176576 119214 6144 +/access/p1/0373.aff 6448619520 607352774 1811939328 821049 6144 +/access/p1/0375.aff 2149130240 1936962 2149130240 0 0 +/access/p1/0376.aff 2149130240 1936793 2149130240 0 0 +/access/p1/0377.aff 2149130240 1936963 2149130240 0 0 +/access/p1/0378.aff 2149130240 1936936 2149130240 0 0 +/access/p1/0380.aff 1216954368 1041792 1216954368 2366483 0 +/access/p1/0381.aff 722534400 648822174 722534400 7990 0 +/access/p1/0382.aff 344332800 81609324 301989888 12919 6144 +/access/p1/0388.aff 4293632000 566803624 4293632000 734684 0 +/access/p1/0389.aff 4293632000 10911 4293632000 8385457 0 +/access/p1/0390.aff 0 0 0 0 0 +/access/p1/0391.aff 85038080 40518080 85038080 0 0 +/access/p1/0392.aff 4293632000 10909 4293632000 8385457 0 +/access/p1/0394.aff 2431844352 5816074 2431844352 0 0 +/access/p1/0397.aff 3145728 3084 3145728 0 6144 +/access/p1/0399.aff 4223729664 1598458048 4223729664 0 0 +/access/p1/0400.aff 11010048 3279 11010048 0 0 +/access/p1/0401.aff 10242892800 886862473 10242892800 0 0 +/access/p1/0402.aff 10242892800 250924568 10242892800 0 0 +/access/p1/0403.aff 10242892800 251025737 10242892800 0 0 +/access/p1/0404.aff 10242892800 526153857 10242892800 0 0 +/access/p1/0405.aff 3145728 3085 3145728 0 6144 +/access/p1/0406.aff 10242949120 251219515 10242949120 0 0 +/access/p1/0407.aff 1082523648 246305696 1082523648 0 0 +/access/p1/0409.aff 0 0 0 0 0 +/access/p1/0410.aff 8455258112 2713062231 8455258112 0 0 +/access/p1/0411.aff 65536 10627 65536 0 0 +/access/p1/0412.aff 2559836160 579996854 2559836160 0 0 +/access/p1/0413.aff 2111897600 2053373038 2111897600 0 0 +/access/p1/0414.aff 850010112 27225393 850010112 0 0 +/access/p1/0415.aff 0 812136334 0 0 0 +/access/p1/0416.aff 4335206400 2164387019 4335206400 0 0 +/access/p1/0417.aff 84672512 27032273 84672512 0 6144 +/access/p1/0418.aff 4871301120 654884167 1258291200 58734 6144 +/access/p1/0419.aff 2167603200 1022727102 1862270976 116949 6144 +/access/p1/0420.aff 234881024 70212516 234881024 8455 163840 +/access/p1/0421.aff 176160768 27706094 176160768 4056 393216 +/access/p1/0423.aff 4311982080 1821089919 4311982080 0 0 +/access/p1/0424.aff 6448619520 2694144281 6448619520 0 0 +/access/p1/0425.aff 4311982080 256588 4311982080 0 0 +/access/p1/0426.aff 0 0 0 0 0 +/access/p1/0427.aff 4311982080 1896888200 4311982080 0 0 +/access/p1/0428.aff 1358991360 681435938 1358991360 102555 0 +/access/p1/0429.aff 4311982080 1980822197 4311982080 0 0 +/access/p1/0430.aff 4311982080 1838435887 4311982080 0 0 +/access/p1/0432.aff 1358991360 557772635 1358991360 112300 0 +/access/p1/0438.aff 6448619520 1133391 6448619520 12575843 0 +/access/p1/0439.aff 6448619520 1132719 6448619520 12575843 0 +/access/p1/0440.aff 1207664640 422456863 1207664640 218394 277376 +/access/p1/0441.aff 1275068416 504623189 1275068416 80314 163840 +/access/p1/0445.aff 816084480 181881265 816084480 52520 0 +/access/p1/0446.aff 528482304 217071 528482304 1030501 0 +/access/p1/0452.aff 4294817280 829150 4294817280 2894080 0 +/access/p1/0453.aff 4294816768 414659 4294816768 2908353 0 +/access/p1/0456.aff 4294816768 621578 4294816768 2910104 0 +/access/p1/0457.aff 4294816768 621581 4294816768 2910167 0 +/access/p1/0458.aff 4265238016 156236546 4265238016 60324 0 +/access/p1/0459.aff 4265238016 114907210 4265238016 64281 0 +/access/p1/0461.aff 4265238016 530770221 4265238016 827663 0 +/access/p1/0462.aff 4265238016 203612448 4265238016 480410 0 +/access/p1/0469.aff 20491337728 9608477115 20491337728 0 512 +/access/p1/0470.aff 2564997120 1126934798 2564997120 0 0 +/access/p1/0471.aff 4197711872 2029842231 4197711872 0 9216 +/access/p1/0472.aff 0 0 0 0 0 +/access/p1/0473.aff 0 0 0 0 0 +/access/p1/0474.aff 4791992320 3510683585 4791992320 0 6144 +/access/p1/0475.aff 4310433792 3025796732 4310433792 0 0 +/access/p1/0476.aff 7683637248 2456071076 7683637248 0 0 +/access/p1/0478.aff 4311982080 675013977 4311982080 0 0 +/access/p1/0479.aff 3249340416 1268213372 3249340416 0 0 +/access/p1/0481.aff 5249664000 1381308136 5249664000 0 0 +/access/p1/0482.aff 3249340416 1588776073 3249340416 0 0 +/access/p1/0485.aff 2564481024 849168610 2564481024 0 0 +/access/p1/0486.aff 3249340416 400012961 3249340416 0 0 +/access/p1/0488.aff 4000776192 3937 4000776192 0 0 +/access/p1/0489.aff 4226023424 7087 4226023424 0 8192 +/access/p1/0490.aff 4000776192 3938 4000776192 0 0 +/access/p1/0491.aff 4311982080 4014 4311982080 0 0 +/access/p1/0493.aff 4311982080 6461 4311982080 0 0 +/access/p1/0494.aff 4000776192 6004 100663296 164864 3072 +/access/p1/0496.aff 4311982080 13466 4311982080 0 0 +/access/p1/0497.aff 4311982080 406217846 4311982080 126055 0 +/access/p1/0498.aff 4223729664 317568 4223729664 0 0 +/access/p1/0499.aff 4303272960 812149331 4303272960 0 0 +/access/p1/0511.aff 2111864832 7113 2111864832 0 0 +/access/p1/0512.aff 13893632 4263 13893632 0 6144 +/access/p1/0513.aff 2559836160 7219 2559836160 0 0 +/access/p1/0514.aff 2559836160 958637642 2559836160 0 0 +/access/p1/0517.aff 2111864832 1080895633 2111864832 0 0 +/access/p1/0518.aff 2079588352 10205 2079588352 0 6144 +/access/p1/0519.aff 2111864832 7113 2111864832 0 0 +/access/p1/0520.aff 2559836160 7219 2559836160 0 0 +/access/p1/0521.aff 8455200768 6994616531 8455200768 149304 0 +/access/p1/0522.aff 3209084928 767354767 3209084928 135985 0 +/access/p1/0523.aff 1083801600 46397386 83886080 4229 6144 +/access/p1/0524.aff 3227148288 1902048082 3227148288 130324 0 +/access/p1/0525.aff 6448619520 1150732 6448619520 12576788 0 +/access/p1/0526.aff 6448619520 1828782 6448619520 12590204 0 +/access/p1/0527.aff 6448619520 1133609 6448619520 12575843 0 +/access/p1/0528.aff 1441972224 6015 1441972224 2816346 0 +/access/p1/0529.aff 4286182680 6011 33554432 0 6144 +/access/p1/0530.aff 4286182680 6012 33554432 0 6144 +/access/p1/0531.aff 4286182680 6012 33554432 0 6144 +/access/p1/0532.aff 18113808896 1093864163 18113808896 25207089 0 +/access/p1/0533.aff 2008860032 6012 33554432 0 6144 +/access/p1/0534.aff 2164083200 145658257 318767104 24777 6144 +/access/p1/0535.aff 9173114880 5144289 9173114880 17633152 0 +/access/p1/0536.aff 2164083200 314879 2164083200 72560 0 +/access/p1/0538.aff 2164083200 3498 2164083200 4226725 0 +/access/p1/0539.aff 9173114880 195796746 9173114880 16034837 0 +/access/p1/0540.aff 130072576 6017 33554432 0 6144 +/access/p1/0541.aff 209797120 6013 33554432 0 0 +/access/p1/0542.aff 341961728 110078179 341961728 34741 6144 +/access/p1/0543.aff 1094545576960 289084346 385875968 15397 6144 +/access/p1/0544.aff 209797120 6012 33554432 0 6144 +/access/p1/0545.aff 130072576 6017 33554432 0 6144 +/access/p1/0546.aff 1350136832 11209 1350136832 2635776 0 +/access/p1/0548.aff 2104565760 232652497 2104565760 45482 0 +/access/p1/0549.aff 4569600000 198441270 4569600000 6791436 0 +/access/p1/0550.aff 2255098368 116682673 2255098368 3661780 0 +/access/p1/0551.aff 4290600960 101851008 4290600960 296525 0 +/access/p1/0552.aff 2255098368 98666759 2255098368 3344446 0 +/access/p1/0553.aff 4551129088 528903968 4551129088 81074 0 +/access/p1/0554.aff 4551129088 171723650 469762048 28060 6144 +/access/p1/0556.aff 4569600000 487033497 4569600000 6574182 0 +/access/p1/0558.aff 12072517632 9830 33554432 0 6144 +/access/p1/0562.aff 2428174336 680152034 2428174336 0 0 +/access/p1/0563.aff 6505758720 4299819106 6505758720 0 0 +/access/p1/0567.aff 1281982464 6015 33554432 0 6144 +/access/p1/0568.aff 10249382912 6019 33554432 0 6144 +/access/p1/0569.aff 33820284928 324455140 1174405120 138294 6144 +/access/p1/0572.aff 3253469184 12477 33554432 0 6144 +/access/p1/0574.aff 15096840192 6014 33554432 0 6144 +/access/p1/0576.aff 3253469184 116495 33554432 9472 6144 +/access/p1/0577.aff 1707761664 390812601 1707761664 60068 0 +/access/p1/0578.aff 1707245568 333522274 1707245568 17630 0 +/access/p1/0579.aff 541384704 171360477 541384704 8194 4096 +/access/p1/0580.aff 541499392 149643958 541499392 11313 6656 +/access/p1/0581.aff 541384704 161098757 541384704 8235 1536 +/access/p1/0582.aff 541384704 172145700 541384704 9727 2048 +/access/p1/0583.aff 541384704 189454170 541384704 19211 512 +/access/p1/0584.aff 9100032000 154573769 671088640 13089 6144 +/access/p1/0585.aff 9100032000 1542375757 9100032000 1470373 0 +/access/p1/0586.aff 9100032000 1544356054 9100032000 1470646 0 +/access/p1/0587.aff 9100032000 1542857477 9100032000 1470316 0 +/access/p1/0588.aff 9100032000 1542366681 9100032000 1471408 0 +/access/p1/0589.aff 8455200768 12648 8455200768 16513018 0 +/access/p1/0590.aff 4303756800 10924 4303756800 8405231 0 +/access/p1/0591.aff 2564481024 7724 2564481024 5008417 0 +/access/p1/0592.aff 2111864832 7536 2111864832 4124457 0 +/access/p1/0593.aff 2111864832 13517 2030043136 3931881 6656 +/access/p1/0594.aff 812851200 5986 812851200 1587599 0 +/access/p1/0595.aff 812851200 335635612 812851200 42915 0 +/access/p1/0596.aff 811302912 54377025 811302912 8937 0 +/access/p1/0597.aff 528482304 247732814 528482304 29324 0 +/access/p1/0598.aff 4099866624 664755220 4099866624 82640 0 +/access/p1/0599.aff 2167603200 815252050 2167603200 265402 0 +/access/p1/0601.aff 4099866624 35783331 4099866624 838269 0 +/access/p1/0603.aff 2564481024 2310450309 2564481024 76001 0 +/access/p1/0606.aff 4223729664 514171044 2197815296 338684 6144 +/access/p1/0607.aff 213909504 40917973 213909504 10439 0 +/access/p1/0612.aff 12072517632 6702654389 12072517632 571957 6144 +/access/p1/0613.aff 4293632000 4006 4293632000 8386000 0 +/access/p1/0614.aff 4293632000 3857424248 4293632000 45275 0 +/access/p1/0615.aff 853622784 6020 33554432 0 6144 +/access/p1/0616.aff 0 0 0 0 0 +/access/p1/0617.aff 541384704 34160967 536870912 1647 6656 +/access/p1/0618.aff 169795584 69194339 169795584 23621 0 +/access/p1/0619.aff 122247680 30906975 122247680 9227 0 +/access/p1/0620.aff 184287232 29583178 184287232 0 0 +/access/p1/0621.aff 0 0 0 0 0 +/access/p1/0622.aff 1624670208 484497885 1624670208 170849 0 +/access/p1/0623.aff 3860398080 1711906258 3860398080 329053 0 +/access/p1/0624.aff 3228696576 1799569452 3228696576 284816 0 +/access/p1/0625.aff 3228696576 1319463664 3228696576 492099 0 +/access/p1/0626.aff 3228696576 3754 3228696576 6306048 0 +/access/p1/0627.aff 3228696576 1493933419 3228696576 797146 0 +/access/p1/0628.aff 1707245568 250641 1707245568 0 0 +/access/p1/0629.aff 1624670208 141412 956301312 0 6144 +/access/p1/0630.aff 1089994752 159916 1089994752 0 0 +/access/p1/0631.aff 1614348288 482242203 1614348288 1194500 0 +/access/p1/0632.aff 2111864832 309489 2111864832 0 0 +/access/p1/0633.aff 2625005568 386613 2625005568 213 7168 +/access/p1/0634.aff 2625005568 385504 2625005568 213 0 +/access/p1/0635.aff 2160377856 6030 33554432 0 6144 +/access/p1/0636.aff 3166765056 463963 3166765056 0 0 +/access/p1/0637.aff 3249340416 656656278 3137339392 194377 6656 +/access/p1/0638.aff 3249340416 733027085 3154116608 164759 7168 +/access/p1/0639.aff 3249340416 655617337 3249340416 653312 0 +/access/p1/0640.aff 3249340416 710586293 3249340416 176368 0 +/access/p1/0641.aff 3249340416 1281122068 3249340416 225488 0 +/access/p1/0643.aff 1281982464 6016 33554432 0 6144 +/access/p1/0644.aff 1281982464 6014 33554432 0 6144 +/access/p1/0645.aff 2111864832 3484 2111864832 4124736 0 +/access/p1/0646.aff 2111864832 3484 2111864832 4124736 512 +/access/p1/0647.aff 2111864832 3485 2111864832 4124736 0 +/access/p1/0648.aff 3243663360 537482864 3120562176 3554797 7168 +/access/p1/0649.aff 2147357696 847463337 2147357696 81163 0 +/access/p1/0650.aff 3243663360 531218457 3154116608 3334331 8704 +/access/p1/0651.aff 3501793280 1038183050 3501793280 127055 0 +/access/p1/0652.aff 2147357184 433656675 2147357184 68459 0 +/access/p1/0653.aff 3209084928 710615594 3209084928 2266931 0 +/access/p1/0654.aff 4294816768 4010 4294816768 8388261 0 +/access/p1/0655.aff 6448619520 1465638097 6448619520 396326 0 +/access/p1/0656.aff 6448619520 535695838 6448619520 183836 0 +/access/p1/0657.aff 6448619520 71047534 352321536 53519 6144 +/access/p1/0658.aff 3249340416 1040022361 2785017856 282241 8192 +/access/p1/0659.aff 6448619520 1539830999 5788139520 295449 6144 +/access/p1/0660.aff 6448619520 2138990387 6448619520 534129 6144 +/access/p1/0661.aff 6448619520 2105158855 6448619520 569734 6144 +/access/p1/0662.aff 6448619520 2035171898 6448619520 259664 0 +/access/p1/0663.aff 6448619520 2061969768 6448619520 342874 0 +/access/p1/0664.aff 6448619520 1850500410 6448619520 429151 0 +/access/p1/0665.aff 6448619520 515838018 5519704064 190624 6144 +/access/p1/0667.aff 6448619520 551320021 6448619520 173919 0 +/access/p1/0668.aff 6448619520 1272411264 6448619520 398711 0 +/access/p1/0669.aff 6448619520 1590513581 5972688896 328010 6144 +/access/p1/0670.aff 6448619520 2091314941 6448619520 492887 0 +/access/p1/0671.aff 6448619520 1935855618 6448619520 468311 0 +/access/p1/0672.aff 6448619520 1559602719 6174015488 280164 6656 +/access/p1/0673.aff 10005037056 3520650970 10005037056 351937 0 +/access/p1/0674.aff 6448619520 1741203371 6448619520 354121 0 +/access/p1/0676.aff 6448619520 2470238112 6448619520 450255 0 +/access/p1/0677.aff 3228696576 1035266302 3228696576 353250 0 +/access/p1/0678.aff 6448619520 2002519199 6448619520 488008 0 +/access/p1/0679.aff 6448619520 1214640411 6448619520 345512 0 +/access/p1/0680.aff 6448619520 1453109058 6448619520 234715 0 +/access/p1/0681.aff 6448619520 1786025040 6448619520 362909 0 +/access/p1/0682.aff 6448619520 987585396 4043309056 1469460 6144 +/access/p1/0683.aff 6448619520 1488797932 5922357248 1200458 8704 +/access/p1/0684.aff 6448619520 12191 352321536 657916 6144 +/access/p1/0685.aff 6448619520 113587953 5200936960 9457150 7168 +/access/p1/0686.aff 6448619520 6014 33554432 0 6144 +/access/p1/0687.aff 9747587072 789858317 2868903936 277074 6144 +/access/p1/0689.aff 6448619520 463214775 1610612736 459155 11264 +/access/p1/0690.aff 6448619520 1247598170 6448619520 4237161 0 +/access/p1/0691.aff 6448619520 992592993 6448619520 523536 0 +/access/p1/0693.aff 6448619520 2054847767 6448619520 339457 0 +/access/p1/0694.aff 6448619520 1240638123 6448619520 272795 0 +/access/p1/0695.aff 6448619520 1378181748 6448619520 391757 0 +/access/p1/0696.aff 6448619520 1702512757 6448619520 364502 0 +/access/p1/0697.aff 6448619520 2037012433 6448619520 610440 0 +/access/p1/0698.aff 6448619520 1000025606 6448619520 4884714 0 +/access/p1/0699.aff 6448619520 586942410 1828716544 167545 6144 +/access/p1/0700.aff 6448619520 2040153367 6341787648 418104 6144 +/access/p1/0701.aff 6448619520 1242271448 6448619520 326069 0 +/access/p1/0702.aff 3200311296 1133762 3200311296 6247485 0 +/access/p1/0703.aff 3243663360 1845243 3243663360 6325260 0 +/access/p1/0704.aff 3243663360 1127109 3243663360 6332228 0 +/access/p1/0705.aff 3243663360 1937491 3243663360 6324315 0 +/access/p1/0706.aff 3860398080 6016 33554432 0 6144 +/access/p1/0707.aff 3243663360 1127208 3243663360 6332228 0 +/access/p1/0708.aff 3243663360 1126983 3243663360 6332228 0 +/access/p1/0709.aff 3249340416 1127248 3249340416 6343316 0 +/access/p1/0710.aff 3249340416 1142528 3249340416 6335252 0 +/access/p1/0711.aff 3239792640 1136144 3239792640 6317108 0 +/access/p1/0713.aff 3249340416 1893478620 3249340416 69407 0 +/access/p1/0715.aff 9173114880 3352922609 9173114880 770704 0 +/access/p1/0716.aff 9173114880 2828530055 9173114880 622815 0 +/access/p1/0717.aff 9173114880 3009634575 9173114880 413161 0 +/access/p1/0718.aff 9173114880 2644801229 9173114880 673072 0 +/access/p1/0725.aff 541384704 43873289 150994944 22244 6144 +/access/p1/0726.aff 631185408 293507351 620756992 55329 6144 +/access/p1/0728.aff 528482304 145752774 528482304 29743 0 +/access/p1/0730.aff 2147837440 364869 2147837440 1611 0 +/access/p1/0731.aff 2147837440 282157396 2147837440 14752 0 +/access/p1/0732.aff 2147837440 364875 2147837440 1598 0 +/access/p1/0733.aff 2147837440 364805 2147837440 1604 0 +/access/p1/0734.aff 2147837440 364794 2147837440 1596 0 +/access/p1/0736.aff 2147837440 282265589 2147837440 16247 0 +/access/p1/0737.aff 2147837440 282227696 2147837440 15088 0 +/access/p1/0738.aff 2147837440 281840145 2147837440 13705 0 +/access/p1/0739.aff 2559836160 2108668061 2559836160 45476 0 +/access/p1/0740.aff 2111864832 1112530767 2111864832 257382 0 +/access/p1/0741.aff 2559836160 830190290 2559836160 428311 0 +/access/p1/0742.aff 2564352000 1671326812 2564352000 618368 0 +/access/p1/0743.aff 2559836160 1003742811 2559836160 404472 512 +/access/p1/0744.aff 6449587200 1943585546 6449587200 3770822 0 +/access/p1/0745.aff 6449587200 4521 6449587200 12596849 0 +/access/p1/0746.aff 6448619520 4521 6448619520 12594959 0 +/access/p1/0747.aff 6447537664 4522 6447537664 12592846 0 +/access/p1/0748.aff 6448619520 4522 6448619520 12594959 0 +/access/p1/0749.aff 8455200768 1140693 8455200768 16431443 0 +/access/p1/0750.aff 4223729664 1127504 4223729664 8246332 0 +/access/p1/0751.aff 4223729664 1127390 4223729664 8246332 0 +/access/p1/0752.aff 8455200768 12900131 8455200768 16437945 0 +/access/p1/0753.aff 8455200768 5327509 8455200768 16495201 0 +/access/p1/0754.aff 4310433792 1127460 4310433792 8415768 2560 +/access/p1/0755.aff 4223729664 2176484 4223729664 8239052 0 +/access/p1/0756.aff 4310433792 4831002 4310433792 8400220 0 +/access/p1/0757.aff 4310433792 1127575 4310433792 8415763 0 +/access/p1/0758.aff 2160431104 6014 33554432 11 6144 +/access/p1/0759.aff 8622931968 4239199 8622931968 16826590 0 +/access/p1/0760.aff 4303272960 1130631 3506438144 6794783 6144 +/access/p1/0761.aff 4310433792 1127566 4310433792 8415763 0 +/access/p1/0762.aff 4311982080 1195352 4311982080 8398943 0 +/access/p1/0763.aff 4311982080 5020215 4311982080 8399276 0 +/access/p1/0764.aff 1282830336 506734704 1282830336 169104 0 +/access/p1/0765.aff 1082460160 220004471 838860800 76366 6144 +/access/p1/0766.aff 1707761664 372199021 973078528 76745 6144 +/access/p1/0767.aff 4321787904 842079576 4321787904 5386004 0 +/access/p1/0768.aff 2111864832 591214331 2111864832 575665 0 +/access/p1/0770.aff 13020069888 4805821517 7381975040 265744 6144 +/access/p1/0771.aff 4311982080 2143920930 4311982080 320691 0 +/access/p1/0773.aff 2113413120 603560861 1996488704 136855 6144 +/access/p1/0775.aff 3249340416 6012 33554432 11 6144 +/access/p1/0776.aff 2111864832 438565271 2111864832 76233 0 +/access/p1/0777.aff 852266496 122576175 167772160 2192 8704 +/access/p1/0778.aff 329638748160 8985 50331648 11 6144 +/access/p1/0779.aff 4551129088 992931296 4551129088 1286332 0 +/access/p1/0780.aff 40981118976 4128855998 40981118976 529993 0 +/access/p1/0782.aff 329638748160 105606317 352321536 34346 15360 +/access/p1/0783.aff 1281982464 6015 33554432 11 6144 +/access/p1/0784.aff 1055932416 9449 838860800 1603323 7680 +/access/p1/0790.aff 540868608 6017 33554432 11 6144 +/access/p1/0791.aff 329638748160 84222806 318767104 27359 8704 +/access/p1/0792.aff 11525455872 7672 33554432 0 6144 +/access/p1/0793.aff 730791936 174719198 730791936 85175 0 +/access/p1/0794.aff 1281982464 514614577 1281982464 158556 0 +/access/p1/0797.aff 6448619520 25512 33554432 85 6144 +/access/p1/0799.aff 11525455872 2741274563 11039408128 10574180 6144 diff --git a/stats/project1.afcompare.preen.txt b/stats/project1.afcompare.preen.txt new file mode 100644 index 0000000..eace873 --- /dev/null +++ b/stats/project1.afcompare.preen.txt @@ -0,0 +1,601 @@ +/Volumes/TB/project1/affs/0001.aff -> /Volumes/s.eecs.harvard.edu/p1/0001.aff Nochg: 0 NUL: 0 LZMA: 11 old: 149752486 new: 136730357 LZred: 8.70% +/Volumes/TB/project1/affs/0002.aff -> /Volumes/s.eecs.harvard.edu/p1/0002.aff Nochg: 0 NUL: 44 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project1/affs/0003.aff -> /Volumes/s.eecs.harvard.edu/p1/0003.aff Nochg: 0 NUL: 25 LZMA: 1 old: 252752 new: 219666 LZred: 13.09% +/Volumes/TB/project1/affs/0004.aff -> /Volumes/s.eecs.harvard.edu/p1/0004.aff Nochg: 0 NUL: 0 LZMA: 21 old: 1734123 new: 275920 LZred: 84.09% +/Volumes/TB/project1/affs/0005.aff -> /Volumes/s.eecs.harvard.edu/p1/0005.aff Nochg: 0 NUL: 0 LZMA: 8 old: 58923795 new: 43711358 LZred: 25.82% +/Volumes/TB/project1/affs/0006.aff -> /Volumes/s.eecs.harvard.edu/p1/0006.aff Nochg: 0 NUL: 0 LZMA: 8 old: 59953614 new: 42693709 LZred: 28.79% +/Volumes/TB/project1/affs/0007.aff -> /Volumes/s.eecs.harvard.edu/p1/0007.aff Nochg: 0 NUL: 0 LZMA: 8 old: 51790520 new: 39498713 LZred: 23.73% +/Volumes/TB/project1/affs/0008.aff -> /Volumes/s.eecs.harvard.edu/p1/0008.aff Nochg: 0 NUL: 4 LZMA: 7 old: 35308860 new: 27892759 LZred: 21.00% +/Volumes/TB/project1/affs/0009.aff -> /Volumes/s.eecs.harvard.edu/p1/0009.aff Nochg: 0 NUL: 1 LZMA: 2 old: 10702712 new: 8519943 LZred: 20.39% +/Volumes/TB/project1/affs/0010.aff -> /Volumes/s.eecs.harvard.edu/p1/0010.aff Nochg: 0 NUL: 0 LZMA: 8 old: 57824872 new: 42540127 LZred: 26.43% +/Volumes/TB/project1/affs/0011.aff -> /Volumes/s.eecs.harvard.edu/p1/0011.aff Nochg: 0 NUL: 0 LZMA: 15 old: 100930552 new: 74462210 LZred: 26.22% +/Volumes/TB/project1/affs/0012.aff -> /Volumes/s.eecs.harvard.edu/p1/0012.aff Nochg: 0 NUL: 0 LZMA: 33 old: 403175079 new: 77977486 LZred: 80.66% +/Volumes/TB/project1/affs/0013.aff -> /Volumes/s.eecs.harvard.edu/p1/0013.aff Nochg: 0 NUL: 9 LZMA: 24 old: 145921545 new: 102128662 LZred: 30.01% +/Volumes/TB/project1/affs/0014.aff -> /Volumes/s.eecs.harvard.edu/p1/0014.aff Nochg: 0 NUL: 0 LZMA: 24 old: 211899752 new: 168288883 LZred: 20.58% +/Volumes/TB/project1/affs/0015.aff -> /Volumes/s.eecs.harvard.edu/p1/0015.aff Nochg: 0 NUL: 0 LZMA: 26 old: 178489530 new: 140915737 LZred: 21.05% +/Volumes/TB/project1/affs/0016.aff -> /Volumes/s.eecs.harvard.edu/p1/0016.aff Nochg: 0 NUL: 0 LZMA: 32 old: 227923927 new: 172684857 LZred: 24.24% +/Volumes/TB/project1/affs/0017.aff -> /Volumes/s.eecs.harvard.edu/p1/0017.aff Nochg: 0 NUL: 0 LZMA: 16 old: 122320362 new: 96308536 LZred: 21.27% +/Volumes/TB/project1/affs/0018.aff -> /Volumes/s.eecs.harvard.edu/p1/0018.aff Nochg: 0 NUL: 0 LZMA: 13 old: 93698328 new: 70911529 LZred: 24.32% +/Volumes/TB/project1/affs/0019.aff -> /Volumes/s.eecs.harvard.edu/p1/0019.aff Nochg: 0 NUL: 0 LZMA: 10 old: 52052536 new: 34938534 LZred: 32.88% +/Volumes/TB/project1/affs/0020.aff -> /Volumes/s.eecs.harvard.edu/p1/0020.aff Nochg: 0 NUL: 0 LZMA: 33 old: 533334395 new: 1134831 LZred: 99.79% +/Volumes/TB/project1/affs/0021.aff -> /Volumes/s.eecs.harvard.edu/p1/0021.aff Nochg: 0 NUL: 0 LZMA: 77 old: 713562186 new: 95174080 LZred: 86.66% +/Volumes/TB/project1/affs/0022.aff -> /Volumes/s.eecs.harvard.edu/p1/0022.aff Nochg: 0 NUL: 0 LZMA: 65 old: 1063226077 new: 2237706 LZred: 99.79% +/Volumes/TB/project1/affs/0023.aff -> /Volumes/s.eecs.harvard.edu/p1/0023.aff Nochg: 0 NUL: 0 LZMA: 23 old: 229389074 new: 200490882 LZred: 12.60% +/Volumes/TB/project1/affs/0024.aff -> /Volumes/s.eecs.harvard.edu/p1/0024.aff Nochg: 0 NUL: 0 LZMA: 21 old: 141130117 new: 108223065 LZred: 23.32% +/Volumes/TB/project1/affs/0025.aff -> /Volumes/s.eecs.harvard.edu/p1/0025.aff Nochg: 0 NUL: 3 LZMA: 18 old: 117029109 new: 87932030 LZred: 24.86% +/Volumes/TB/project1/affs/0026.aff -> /Volumes/s.eecs.harvard.edu/p1/0026.aff Nochg: 0 NUL: 0 LZMA: 8 old: 45370212 new: 24586983 LZred: 45.81% +/Volumes/TB/project1/affs/0027.aff -> /Volumes/s.eecs.harvard.edu/p1/0027.aff Nochg: 0 NUL: 0 LZMA: 11 old: 112754955 new: 98547626 LZred: 12.60% +/Volumes/TB/project1/affs/0028.aff -> /Volumes/s.eecs.harvard.edu/p1/0028.aff Nochg: 0 NUL: 0 LZMA: 77 old: 751337343 new: 311593870 LZred: 58.53% +/Volumes/TB/project1/affs/0029.aff -> /Volumes/s.eecs.harvard.edu/p1/0029.aff Nochg: 0 NUL: 0 LZMA: 8 old: 43164097 new: 30592446 LZred: 29.13% +/Volumes/TB/project1/affs/0030.aff -> /Volumes/s.eecs.harvard.edu/p1/0030.aff Nochg: 0 NUL: 0 LZMA: 13 old: 41729731 new: 26675133 LZred: 36.08% +/Volumes/TB/project1/affs/0031.aff -> /Volumes/s.eecs.harvard.edu/p1/0031.aff Nochg: 0 NUL: 0 LZMA: 8 old: 29167018 new: 21343686 LZred: 26.82% +/Volumes/TB/project1/affs/0032.aff -> /Volumes/s.eecs.harvard.edu/p1/0032.aff Nochg: 0 NUL: 0 LZMA: 10 old: 87950796 new: 39690470 LZred: 54.87% +/Volumes/TB/project1/affs/0033.aff -> /Volumes/s.eecs.harvard.edu/p1/0033.aff Nochg: 0 NUL: 0 LZMA: 11 old: 31888724 new: 19432932 LZred: 39.06% +/Volumes/TB/project1/affs/0034.aff -> /Volumes/s.eecs.harvard.edu/p1/0034.aff Nochg: 0 NUL: 3 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project1/affs/0035.aff -> /Volumes/s.eecs.harvard.edu/p1/0035.aff Nochg: 0 NUL: 5 LZMA: 8 old: 51511809 new: 36335474 LZred: 29.46% +/Volumes/TB/project1/affs/0036.aff -> /Volumes/s.eecs.harvard.edu/p1/0036.aff Nochg: 0 NUL: 1 LZMA: 9 old: 19444839 new: 13290728 LZred: 31.65% +/Volumes/TB/project1/affs/0037.aff -> /Volumes/s.eecs.harvard.edu/p1/0037.aff Nochg: 0 NUL: 0 LZMA: 6 old: 54352003 new: 29604495 LZred: 45.53% +/Volumes/TB/project1/affs/0039.aff -> /Volumes/s.eecs.harvard.edu/p1/0039.aff Nochg: 0 NUL: 0 LZMA: 7 old: 42440249 new: 33459460 LZred: 21.16% +/Volumes/TB/project1/affs/0040.aff -> /Volumes/s.eecs.harvard.edu/p1/0040.aff Nochg: 0 NUL: 0 LZMA: 65 old: 523543300 new: 425793233 LZred: 18.67% +/Volumes/TB/project1/affs/0041.aff -> /Volumes/s.eecs.harvard.edu/p1/0041.aff Nochg: 0 NUL: 0 LZMA: 65 old: 337546786 new: 224919523 LZred: 33.37% +/Volumes/TB/project1/affs/0042.aff -> /Volumes/s.eecs.harvard.edu/p1/0042.aff Nochg: 0 NUL: 0 LZMA: 8 old: 25600749 new: 14916431 LZred: 41.73% +/Volumes/TB/project1/affs/0044.aff -> /Volumes/s.eecs.harvard.edu/p1/0044.aff Nochg: 3 NUL: 1 LZMA: 12 old: 118131977 new: 104803012 LZred: 11.28% +/Volumes/TB/project1/affs/0045.aff -> /Volumes/s.eecs.harvard.edu/p1/0045.aff Nochg: 0 NUL: 0 LZMA: 4 old: 12121524 new: 1166194 LZred: 90.38% +/Volumes/TB/project1/affs/0047.aff -> /Volumes/s.eecs.harvard.edu/p1/0047.aff Nochg: 0 NUL: 0 LZMA: 87 old: 796388165 new: 611053632 LZred: 23.27% +/Volumes/TB/project1/affs/0048.aff -> /Volumes/s.eecs.harvard.edu/p1/0048.aff Nochg: 0 NUL: 0 LZMA: 51 old: 378704095 new: 294973480 LZred: 22.11% +/Volumes/TB/project1/affs/0049.aff -> /Volumes/s.eecs.harvard.edu/p1/0049.aff Nochg: 0 NUL: 0 LZMA: 33 old: 311708585 new: 145734353 LZred: 53.25% +/Volumes/TB/project1/affs/0050.aff -> /Volumes/s.eecs.harvard.edu/p1/0050.aff Nochg: 0 NUL: 56 LZMA: 21 old: 69788656 new: 43261197 LZred: 38.01% +/Volumes/TB/project1/affs/0051.aff -> /Volumes/s.eecs.harvard.edu/p1/0051.aff Nochg: 0 NUL: 0 LZMA: 126 old: 955929647 new: 728137651 LZred: 23.83% +/Volumes/TB/project1/affs/0052.aff -> /Volumes/s.eecs.harvard.edu/p1/0052.aff Nochg: 0 NUL: 0 LZMA: 73 old: 642851003 new: 533709290 LZred: 16.98% +/Volumes/TB/project1/affs/0053.aff -> /Volumes/s.eecs.harvard.edu/p1/0053.aff Nochg: 0 NUL: 0 LZMA: 14 old: 103608488 new: 60375296 LZred: 41.73% +/Volumes/TB/project1/affs/0054.aff -> /Volumes/s.eecs.harvard.edu/p1/0054.aff Nochg: 0 NUL: 0 LZMA: 22 old: 165749875 new: 130057061 LZred: 21.53% +/Volumes/TB/project1/affs/0055.aff -> /Volumes/s.eecs.harvard.edu/p1/0055.aff Nochg: 0 NUL: 0 LZMA: 32 old: 227286034 new: 165778351 LZred: 27.06% +/Volumes/TB/project1/affs/0056.aff -> /Volumes/s.eecs.harvard.edu/p1/0056.aff Nochg: 0 NUL: 0 LZMA: 51 old: 286183203 new: 190850554 LZred: 33.31% +/Volumes/TB/project1/affs/0057.aff -> /Volumes/s.eecs.harvard.edu/p1/0057.aff Nochg: 0 NUL: 0 LZMA: 33 old: 246598936 new: 196720942 LZred: 20.23% +/Volumes/TB/project1/affs/0058.aff -> /Volumes/s.eecs.harvard.edu/p1/0058.aff Nochg: 0 NUL: 2 LZMA: 31 old: 258740580 new: 211576234 LZred: 18.23% +/Volumes/TB/project1/affs/0059.aff -> /Volumes/s.eecs.harvard.edu/p1/0059.aff Nochg: 0 NUL: 13 LZMA: 13 old: 85065445 new: 64453439 LZred: 24.23% +/Volumes/TB/project1/affs/0060.aff -> /Volumes/s.eecs.harvard.edu/p1/0060.aff Nochg: 0 NUL: 0 LZMA: 150 old: 1140628908 new: 385635902 LZred: 66.19% +/Volumes/TB/project1/affs/0061.aff -> /Volumes/s.eecs.harvard.edu/p1/0061.aff Nochg: 0 NUL: 0 LZMA: 14 old: 80299325 new: 58184006 LZred: 27.54% +/Volumes/TB/project1/affs/0062.aff -> /Volumes/s.eecs.harvard.edu/p1/0062.aff Nochg: 0 NUL: 0 LZMA: 77 old: 751337206 new: 311594834 LZred: 58.53% +/Volumes/TB/project1/affs/0063.aff -> /Volumes/s.eecs.harvard.edu/p1/0063.aff Nochg: 0 NUL: 0 LZMA: 38 old: 355084098 new: 178256299 LZred: 49.80% +/Volumes/TB/project1/affs/0064.aff -> /Volumes/s.eecs.harvard.edu/p1/0064.aff Nochg: 0 NUL: 0 LZMA: 25 old: 239477614 new: 206458588 LZred: 13.79% +/Volumes/TB/project1/affs/0065.aff -> /Volumes/s.eecs.harvard.edu/p1/0065.aff Nochg: 0 NUL: 18 LZMA: 8 old: 46403931 new: 33387267 LZred: 28.05% +/Volumes/TB/project1/affs/0066.aff -> /Volumes/s.eecs.harvard.edu/p1/0066.aff Nochg: 8 NUL: 0 LZMA: 110 old: 1240860066 new: 1098645934 LZred: 11.46% +/Volumes/TB/project1/affs/0067.aff -> /Volumes/s.eecs.harvard.edu/p1/0067.aff Nochg: 0 NUL: 0 LZMA: 126 old: 288972603 new: 223155106 LZred: 22.78% +/Volumes/TB/project1/affs/0068.aff -> /Volumes/s.eecs.harvard.edu/p1/0068.aff Nochg: 0 NUL: 0 LZMA: 33 old: 233052008 new: 168745841 LZred: 27.59% +/Volumes/TB/project1/affs/0069.aff -> /Volumes/s.eecs.harvard.edu/p1/0069.aff Nochg: 0 NUL: 0 LZMA: 26 old: 224113710 new: 181892635 LZred: 18.84% +/Volumes/TB/project1/affs/0070.aff -> /Volumes/s.eecs.harvard.edu/p1/0070.aff Nochg: 0 NUL: 0 LZMA: 33 old: 144829999 new: 104269515 LZred: 28.01% +/Volumes/TB/project1/affs/0072.aff -> /Volumes/s.eecs.harvard.edu/p1/0072.aff Nochg: 0 NUL: 17 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project1/affs/0073.aff -> /Volumes/s.eecs.harvard.edu/p1/0073.aff Nochg: 0 NUL: 1 LZMA: 126 old: 1136216448 new: 937319394 LZred: 17.51% +/Volumes/TB/project1/affs/0074.aff -> /Volumes/s.eecs.harvard.edu/p1/0074.aff Nochg: 0 NUL: 0 LZMA: 44 old: 327617381 new: 244692392 LZred: 25.31% +/Volumes/TB/project1/affs/0075.aff -> /Volumes/s.eecs.harvard.edu/p1/0075.aff Nochg: 0 NUL: 0 LZMA: 44 old: 306024584 new: 238165184 LZred: 22.17% +/Volumes/TB/project1/affs/0076.aff -> /Volumes/s.eecs.harvard.edu/p1/0076.aff Nochg: 0 NUL: 0 LZMA: 51 old: 345934569 new: 272756106 LZred: 21.15% +/Volumes/TB/project1/affs/0077.aff -> /Volumes/s.eecs.harvard.edu/p1/0077.aff Nochg: 0 NUL: 0 LZMA: 33 old: 245811324 new: 189804181 LZred: 22.78% +/Volumes/TB/project1/affs/0078.aff -> /Volumes/s.eecs.harvard.edu/p1/0078.aff Nochg: 0 NUL: 0 LZMA: 42 old: 246098506 new: 185583320 LZred: 24.59% +/Volumes/TB/project1/affs/0079.aff -> /Volumes/s.eecs.harvard.edu/p1/0079.aff Nochg: 0 NUL: 0 LZMA: 62 old: 539880317 new: 415487588 LZred: 23.04% +/Volumes/TB/project1/affs/0080.aff -> /Volumes/s.eecs.harvard.edu/p1/0080.aff Nochg: 0 NUL: 0 LZMA: 64 old: 482868282 new: 394346450 LZred: 18.33% +/Volumes/TB/project1/affs/0082.aff -> /Volumes/s.eecs.harvard.edu/p1/0082.aff Nochg: 0 NUL: 11 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project1/affs/0083.aff -> /Volumes/s.eecs.harvard.edu/p1/0083.aff Nochg: 0 NUL: 6 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project1/affs/0084.aff -> /Volumes/s.eecs.harvard.edu/p1/0084.aff Nochg: 0 NUL: 13 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project1/affs/0085.aff -> /Volumes/s.eecs.harvard.edu/p1/0085.aff Nochg: 0 NUL: 6 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project1/affs/0086.aff -> /Volumes/s.eecs.harvard.edu/p1/0086.aff Nochg: 0 NUL: 6 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project1/affs/0087.aff -> /Volumes/s.eecs.harvard.edu/p1/0087.aff Nochg: 0 NUL: 30 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project1/affs/0088.aff -> /Volumes/s.eecs.harvard.edu/p1/0088.aff Nochg: 0 NUL: 6 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project1/affs/0089.aff -> /Volumes/s.eecs.harvard.edu/p1/0089.aff Nochg: 0 NUL: 0 LZMA: 6 old: 37454969 new: 27016413 LZred: 27.87% +/Volumes/TB/project1/affs/0091.aff -> /Volumes/s.eecs.harvard.edu/p1/0091.aff Nochg: 0 NUL: 2 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project1/affs/0092.aff -> /Volumes/s.eecs.harvard.edu/p1/0092.aff Nochg: 0 NUL: 125 LZMA: 1 old: 255314 new: 220923 LZred: 13.47% +/Volumes/TB/project1/affs/0094.aff -> /Volumes/s.eecs.harvard.edu/p1/0094.aff Nochg: 0 NUL: 0 LZMA: 32 old: 234010292 new: 173799603 LZred: 25.73% +/Volumes/TB/project1/affs/0095.aff -> /Volumes/s.eecs.harvard.edu/p1/0095.aff Nochg: 0 NUL: 0 LZMA: 77 old: 986305784 new: 155792910 LZred: 84.20% +/Volumes/TB/project1/affs/0096.aff -> /Volumes/s.eecs.harvard.edu/p1/0096.aff Nochg: 0 NUL: 0 LZMA: 50 old: 360703648 new: 286789780 LZred: 20.49% +/Volumes/TB/project1/affs/0097.aff -> /Volumes/s.eecs.harvard.edu/p1/0097.aff Nochg: 0 NUL: 0 LZMA: 32 old: 235812235 new: 182784570 LZred: 22.49% +/Volumes/TB/project1/affs/0098.aff -> /Volumes/s.eecs.harvard.edu/p1/0098.aff Nochg: 0 NUL: 0 LZMA: 102 old: 979020838 new: 821307583 LZred: 16.11% +/Volumes/TB/project1/affs/0099.aff -> /Volumes/s.eecs.harvard.edu/p1/0099.aff Nochg: 0 NUL: 0 LZMA: 38 old: 305849453 new: 187549017 LZred: 38.68% +/Volumes/TB/project1/affs/0100.aff -> /Volumes/s.eecs.harvard.edu/p1/0100.aff Nochg: 0 NUL: 0 LZMA: 38 old: 343401439 new: 171065609 LZred: 50.18% +/Volumes/TB/project1/affs/0101.aff -> /Volumes/s.eecs.harvard.edu/p1/0101.aff Nochg: 0 NUL: 0 LZMA: 39 old: 1585044 new: 890976 LZred: 43.79% +/Volumes/TB/project1/affs/0103.aff -> /Volumes/s.eecs.harvard.edu/p1/0103.aff Nochg: 0 NUL: 0 LZMA: 39 old: 297938985 new: 207844840 LZred: 30.24% +/Volumes/TB/project1/affs/0104.aff -> /Volumes/s.eecs.harvard.edu/p1/0104.aff Nochg: 0 NUL: 0 LZMA: 33 old: 240483032 new: 170471133 LZred: 29.11% +/Volumes/TB/project1/affs/0105.aff -> /Volumes/s.eecs.harvard.edu/p1/0105.aff Nochg: 0 NUL: 0 LZMA: 39 old: 407188919 new: 106167352 LZred: 73.93% +/Volumes/TB/project1/affs/0106.aff -> /Volumes/s.eecs.harvard.edu/p1/0106.aff Nochg: 0 NUL: 0 LZMA: 33 old: 237261879 new: 160571071 LZred: 32.32% +/Volumes/TB/project1/affs/0107.aff -> /Volumes/s.eecs.harvard.edu/p1/0107.aff Nochg: 0 NUL: 0 LZMA: 44 old: 346516770 new: 258299554 LZred: 25.46% +/Volumes/TB/project1/affs/0109.aff -> /Volumes/s.eecs.harvard.edu/p1/0109.aff Nochg: 0 NUL: 0 LZMA: 39 old: 293837612 new: 232203964 LZred: 20.98% +/Volumes/TB/project1/affs/0110.aff -> /Volumes/s.eecs.harvard.edu/p1/0110.aff Nochg: 0 NUL: 0 LZMA: 39 old: 361669632 new: 313828550 LZred: 13.23% +/Volumes/TB/project1/affs/0111.aff -> /Volumes/s.eecs.harvard.edu/p1/0111.aff Nochg: 0 NUL: 13 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project1/affs/0112.aff -> /Volumes/s.eecs.harvard.edu/p1/0112.aff Nochg: 0 NUL: 0 LZMA: 44 old: 323096019 new: 253228790 LZred: 21.62% +/Volumes/TB/project1/affs/0113.aff -> /Volumes/s.eecs.harvard.edu/p1/0113.aff Nochg: 0 NUL: 0 LZMA: 5 old: 35688238 new: 26441132 LZred: 25.91% +/Volumes/TB/project1/affs/0114.aff -> /Volumes/s.eecs.harvard.edu/p1/0114.aff Nochg: 0 NUL: 0 LZMA: 33 old: 309052067 new: 260584626 LZred: 15.68% +/Volumes/TB/project1/affs/0115.aff -> /Volumes/s.eecs.harvard.edu/p1/0115.aff Nochg: 0 NUL: 0 LZMA: 42 old: 340825762 new: 278769713 LZred: 18.21% +/Volumes/TB/project1/affs/0118.aff -> /Volumes/s.eecs.harvard.edu/p1/0118.aff Nochg: 0 NUL: 0 LZMA: 11 old: 81731418 new: 61266792 LZred: 25.04% +/Volumes/TB/project1/affs/0121.aff -> /Volumes/s.eecs.harvard.edu/p1/0121.aff Nochg: 0 NUL: 0 LZMA: 39 old: 317062099 new: 178345566 LZred: 43.75% +/Volumes/TB/project1/affs/0128.aff -> /Volumes/s.eecs.harvard.edu/p1/0128.aff Nochg: 0 NUL: 0 LZMA: 125 old: 917940936 new: 690475559 LZred: 24.78% +/Volumes/TB/project1/affs/0133.aff -> /Volumes/s.eecs.harvard.edu/p1/0133.aff Nochg: 0 NUL: 0 LZMA: 126 old: 564049209 new: 433283155 LZred: 23.18% +/Volumes/TB/project1/affs/0134.aff -> /Volumes/s.eecs.harvard.edu/p1/0134.aff Nochg: 0 NUL: 0 LZMA: 33 old: 59669375 new: 43625429 LZred: 26.89% +/Volumes/TB/project1/affs/0138.aff -> /Volumes/s.eecs.harvard.edu/p1/0138.aff Nochg: 0 NUL: 13 LZMA: 1 old: 159744 new: 135675 LZred: 15.07% +/Volumes/TB/project1/affs/0139.aff -> /Volumes/s.eecs.harvard.edu/p1/0139.aff Nochg: 0 NUL: 10 LZMA: 1 old: 252607 new: 219791 LZred: 12.99% +/Volumes/TB/project1/affs/0140.aff -> /Volumes/s.eecs.harvard.edu/p1/0140.aff Nochg: 0 NUL: 14 LZMA: 1 old: 223719 new: 194158 LZred: 13.21% +/Volumes/TB/project1/affs/0141.aff -> /Volumes/s.eecs.harvard.edu/p1/0141.aff Nochg: 0 NUL: 7 LZMA: 1 old: 17423 new: 3301 LZred: 81.05% +/Volumes/TB/project1/affs/0142.aff -> /Volumes/s.eecs.harvard.edu/p1/0142.aff Nochg: 0 NUL: 14 LZMA: 1 old: 252679 new: 219517 LZred: 13.12% +/Volumes/TB/project1/affs/0143.aff -> /Volumes/s.eecs.harvard.edu/p1/0143.aff Nochg: 0 NUL: 0 LZMA: 32 old: 27463847 new: 17758509 LZred: 35.34% +/Volumes/TB/project1/affs/0144.aff -> /Volumes/s.eecs.harvard.edu/p1/0144.aff Nochg: 0 NUL: 14 LZMA: 1 old: 223697 new: 194197 LZred: 13.19% +/Volumes/TB/project1/affs/0145.aff -> /Volumes/s.eecs.harvard.edu/p1/0145.aff Nochg: 0 NUL: 14 LZMA: 1 old: 17527 new: 3300 LZred: 81.17% +/Volumes/TB/project1/affs/0147.aff -> /Volumes/s.eecs.harvard.edu/p1/0147.aff Nochg: 0 NUL: 10 LZMA: 1 old: 16867 new: 2859 LZred: 83.05% +/Volumes/TB/project1/affs/0149.aff -> /Volumes/s.eecs.harvard.edu/p1/0149.aff Nochg: 0 NUL: 12 LZMA: 1 old: 17492 new: 3301 LZred: 81.13% +/Volumes/TB/project1/affs/0150.aff -> /Volumes/s.eecs.harvard.edu/p1/0150.aff Nochg: 0 NUL: 10 LZMA: 1 old: 17540 new: 3299 LZred: 81.19% +/Volumes/TB/project1/affs/0151.aff -> /Volumes/s.eecs.harvard.edu/p1/0151.aff Nochg: 0 NUL: 12 LZMA: 1 old: 17492 new: 3301 LZred: 81.13% +/Volumes/TB/project1/affs/0152.aff -> /Volumes/s.eecs.harvard.edu/p1/0152.aff Nochg: 0 NUL: 14 LZMA: 1 old: 17527 new: 3300 LZred: 81.17% +/Volumes/TB/project1/affs/0153.aff -> /Volumes/s.eecs.harvard.edu/p1/0153.aff Nochg: 0 NUL: 14 LZMA: 1 old: 16995 new: 2844 LZred: 83.27% +/Volumes/TB/project1/affs/0154.aff -> /Volumes/s.eecs.harvard.edu/p1/0154.aff Nochg: 0 NUL: 12 LZMA: 1 old: 17492 new: 3301 LZred: 81.13% +/Volumes/TB/project1/affs/0155.aff -> /Volumes/s.eecs.harvard.edu/p1/0155.aff Nochg: 0 NUL: 10 LZMA: 1 old: 17541 new: 3299 LZred: 81.19% +/Volumes/TB/project1/affs/0156.aff -> /Volumes/s.eecs.harvard.edu/p1/0156.aff Nochg: 0 NUL: 7 LZMA: 1 old: 17424 new: 3301 LZred: 81.05% +/Volumes/TB/project1/affs/0157.aff -> /Volumes/s.eecs.harvard.edu/p1/0157.aff Nochg: 0 NUL: 10 LZMA: 1 old: 17540 new: 3299 LZred: 81.19% +/Volumes/TB/project1/affs/0158.aff -> /Volumes/s.eecs.harvard.edu/p1/0158.aff Nochg: 0 NUL: 7 LZMA: 1 old: 17416 new: 3301 LZred: 81.05% +/Volumes/TB/project1/affs/0160.aff -> /Volumes/s.eecs.harvard.edu/p1/0160.aff Nochg: 0 NUL: 250 LZMA: 2 old: 32752 new: 4960 LZred: 84.86% +/Volumes/TB/project1/affs/0161.aff -> /Volumes/s.eecs.harvard.edu/p1/0161.aff Nochg: 0 NUL: 0 LZMA: 33 old: 340615170 new: 301227347 LZred: 11.56% +/Volumes/TB/project1/affs/0166.aff -> /Volumes/s.eecs.harvard.edu/p1/0166.aff Nochg: 0 NUL: 38 LZMA: 88 old: 1435495 new: 215546 LZred: 84.98% +/Volumes/TB/project1/affs/0168.aff -> /Volumes/s.eecs.harvard.edu/p1/0168.aff Nochg: 0 NUL: 0 LZMA: 65 old: 301401539 new: 225153053 LZred: 25.30% +/Volumes/TB/project1/affs/0170.aff -> /Volumes/s.eecs.harvard.edu/p1/0170.aff Nochg: 0 NUL: 0 LZMA: 26 old: 134546700 new: 85865656 LZred: 36.18% +/Volumes/TB/project1/affs/0171.aff -> /Volumes/s.eecs.harvard.edu/p1/0171.aff Nochg: 0 NUL: 0 LZMA: 126 old: 294741236 new: 168620799 LZred: 42.79% +/Volumes/TB/project1/affs/0172.aff -> /Volumes/s.eecs.harvard.edu/p1/0172.aff Nochg: 0 NUL: 0 LZMA: 126 old: 245909619 new: 153775094 LZred: 37.47% +/Volumes/TB/project1/affs/0176.aff -> /Volumes/s.eecs.harvard.edu/p1/0176.aff Nochg: 0 NUL: 0 LZMA: 7 old: 33539778 new: 25398168 LZred: 24.27% +/Volumes/TB/project1/affs/0177.aff -> /Volumes/s.eecs.harvard.edu/p1/0177.aff Nochg: 0 NUL: 0 LZMA: 126 old: 408039660 new: 318288145 LZred: 22.00% +/Volumes/TB/project1/affs/0178.aff -> /Volumes/s.eecs.harvard.edu/p1/0178.aff Nochg: 4 NUL: 0 LZMA: 7 old: 105728943 new: 104746162 LZred: 0.93% +/Volumes/TB/project1/affs/0179.aff -> /Volumes/s.eecs.harvard.edu/p1/0179.aff Nochg: 0 NUL: 0 LZMA: 124 old: 575228611 new: 432780964 LZred: 24.76% +/Volumes/TB/project1/affs/0181.aff -> /Volumes/s.eecs.harvard.edu/p1/0181.aff Nochg: 3 NUL: 0 LZMA: 101 old: 1041825330 new: 913440385 LZred: 12.32% +/Volumes/TB/project1/affs/0182.aff -> /Volumes/s.eecs.harvard.edu/p1/0182.aff Nochg: 0 NUL: 0 LZMA: 124 old: 812103428 new: 616780663 LZred: 24.05% +/Volumes/TB/project1/affs/0183.aff -> /Volumes/s.eecs.harvard.edu/p1/0183.aff Nochg: 0 NUL: 0 LZMA: 76 old: 546932549 new: 432017073 LZred: 21.01% +/Volumes/TB/project1/affs/0184.aff -> /Volumes/s.eecs.harvard.edu/p1/0184.aff Nochg: 0 NUL: 0 LZMA: 75 old: 833963752 new: 197128942 LZred: 76.36% +/Volumes/TB/project1/affs/0186.aff -> /Volumes/s.eecs.harvard.edu/p1/0186.aff Nochg: 0 NUL: 0 LZMA: 73 old: 580717099 new: 470793938 LZred: 18.93% +/Volumes/TB/project1/affs/0187.aff -> /Volumes/s.eecs.harvard.edu/p1/0187.aff Nochg: 2 NUL: 0 LZMA: 120 old: 1217665236 new: 1077470480 LZred: 11.51% +/Volumes/TB/project1/affs/0190.aff -> /Volumes/s.eecs.harvard.edu/p1/0190.aff Nochg: 0 NUL: 0 LZMA: 77 old: 307368870 new: 236970703 LZred: 22.90% +/Volumes/TB/project1/affs/0191.aff -> /Volumes/s.eecs.harvard.edu/p1/0191.aff Nochg: 0 NUL: 0 LZMA: 11 old: 84443102 new: 66384379 LZred: 21.39% +/Volumes/TB/project1/affs/0193.aff -> /Volumes/s.eecs.harvard.edu/p1/0193.aff Nochg: 0 NUL: 0 LZMA: 150 old: 877676821 new: 575245525 LZred: 34.46% +/Volumes/TB/project1/affs/0194.aff -> /Volumes/s.eecs.harvard.edu/p1/0194.aff Nochg: 0 NUL: 0 LZMA: 33 old: 301829918 new: 256882298 LZred: 14.89% +/Volumes/TB/project1/affs/0195.aff -> /Volumes/s.eecs.harvard.edu/p1/0195.aff Nochg: 0 NUL: 0 LZMA: 32 old: 270633955 new: 227668072 LZred: 15.88% +/Volumes/TB/project1/affs/0196.aff -> /Volumes/s.eecs.harvard.edu/p1/0196.aff Nochg: 0 NUL: 1 LZMA: 6 old: 44476758 new: 33796256 LZred: 24.01% +/Volumes/TB/project1/affs/0197.aff -> /Volumes/s.eecs.harvard.edu/p1/0197.aff Nochg: 0 NUL: 0 LZMA: 10 old: 110215569 new: 99102442 LZred: 10.08% +/Volumes/TB/project1/affs/0198.aff -> /Volumes/s.eecs.harvard.edu/p1/0198.aff Nochg: 0 NUL: 0 LZMA: 65 old: 262198425 new: 110441967 LZred: 57.88% +/Volumes/TB/project1/affs/0199.aff -> /Volumes/s.eecs.harvard.edu/p1/0199.aff Nochg: 0 NUL: 0 LZMA: 31 old: 177955341 new: 146704200 LZred: 17.56% +/Volumes/TB/project1/affs/0200.aff -> /Volumes/s.eecs.harvard.edu/p1/0200.aff Nochg: 0 NUL: 0 LZMA: 33 old: 397771128 new: 37510214 LZred: 90.57% +/Volumes/TB/project1/affs/0201.aff -> /Volumes/s.eecs.harvard.edu/p1/0201.aff Nochg: 0 NUL: 0 LZMA: 4 old: 23061583 new: 16510824 LZred: 28.41% +/Volumes/TB/project1/affs/0202.aff -> /Volumes/s.eecs.harvard.edu/p1/0202.aff Nochg: 0 NUL: 0 LZMA: 102 old: 651169386 new: 477182915 LZred: 26.72% +/Volumes/TB/project1/affs/0204.aff -> /Volumes/s.eecs.harvard.edu/p1/0204.aff Nochg: 0 NUL: 0 LZMA: 77 old: 520533599 new: 390776314 LZred: 24.93% +/Volumes/TB/project1/affs/0205.aff -> /Volumes/s.eecs.harvard.edu/p1/0205.aff Nochg: 0 NUL: 0 LZMA: 76 old: 736654187 new: 655250537 LZred: 11.05% +/Volumes/TB/project1/affs/0206.aff -> /Volumes/s.eecs.harvard.edu/p1/0206.aff Nochg: 0 NUL: 0 LZMA: 124 old: 1119807399 new: 950736863 LZred: 15.10% +/Volumes/TB/project1/affs/0209.aff -> /Volumes/s.eecs.harvard.edu/p1/0209.aff Nochg: 0 NUL: 73 LZMA: 78 old: 616095051 new: 516647484 LZred: 16.14% +/Volumes/TB/project1/affs/0210.aff -> /Volumes/s.eecs.harvard.edu/p1/0210.aff Nochg: 0 NUL: 0 LZMA: 24 old: 207114149 new: 170134857 LZred: 17.85% +/Volumes/TB/project1/affs/0211.aff -> /Volumes/s.eecs.harvard.edu/p1/0211.aff Nochg: 0 NUL: 0 LZMA: 13 old: 122103817 new: 96056124 LZred: 21.33% +/Volumes/TB/project1/affs/0212.aff -> /Volumes/s.eecs.harvard.edu/p1/0212.aff Nochg: 0 NUL: 0 LZMA: 26 old: 178998948 new: 139051210 LZred: 22.32% +/Volumes/TB/project1/affs/0214.aff -> /Volumes/s.eecs.harvard.edu/p1/0214.aff Nochg: 0 NUL: 0 LZMA: 17 old: 166045572 new: 49874747 LZred: 69.96% +/Volumes/TB/project1/affs/0215.aff -> /Volumes/s.eecs.harvard.edu/p1/0215.aff Nochg: 0 NUL: 152 LZMA: 1 old: 19689 new: 4144 LZred: 78.95% +/Volumes/TB/project1/affs/0216.aff -> /Volumes/s.eecs.harvard.edu/p1/0216.aff Nochg: 0 NUL: 0 LZMA: 33 old: 100070480 new: 70941169 LZred: 29.11% +/Volumes/TB/project1/affs/0217.aff -> /Volumes/s.eecs.harvard.edu/p1/0217.aff Nochg: 0 NUL: 51 LZMA: 72 old: 698586856 new: 638038917 LZred: 8.67% +/Volumes/TB/project1/affs/0219.aff -> /Volumes/s.eecs.harvard.edu/p1/0219.aff Nochg: 0 NUL: 0 LZMA: 65 old: 88573570 new: 59374722 LZred: 32.97% +/Volumes/TB/project1/affs/0220.aff -> /Volumes/s.eecs.harvard.edu/p1/0220.aff Nochg: 0 NUL: 0 LZMA: 63 old: 256171049 new: 186303721 LZred: 27.27% +/Volumes/TB/project1/affs/0221.aff -> /Volumes/s.eecs.harvard.edu/p1/0221.aff Nochg: 0 NUL: 0 LZMA: 17 old: 263806 new: 40041 LZred: 84.82% +/Volumes/TB/project1/affs/0222.aff -> /Volumes/s.eecs.harvard.edu/p1/0222.aff Nochg: 0 NUL: 0 LZMA: 17 old: 1365923 new: 213627 LZred: 84.36% +/Volumes/TB/project1/affs/0224.aff -> /Volumes/s.eecs.harvard.edu/p1/0224.aff Nochg: 0 NUL: 5 LZMA: 21 old: 128060574 new: 87554742 LZred: 31.63% +/Volumes/TB/project1/affs/0227.aff -> /Volumes/s.eecs.harvard.edu/p1/0227.aff Nochg: 0 NUL: 0 LZMA: 13 old: 177120976 new: 159156686 LZred: 10.14% +/Volumes/TB/project1/affs/0231.aff -> /Volumes/s.eecs.harvard.edu/p1/0231.aff Nochg: 0 NUL: 0 LZMA: 26 old: 206418128 new: 158877556 LZred: 23.03% +/Volumes/TB/project1/affs/0235.aff -> /Volumes/s.eecs.harvard.edu/p1/0235.aff Nochg: 0 NUL: 4 LZMA: 3 old: 34782266 new: 33770937 LZred: 2.91% +/Volumes/TB/project1/affs/0237.aff -> /Volumes/s.eecs.harvard.edu/p1/0237.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163882 new: 6012 LZred: 96.33% +/Volumes/TB/project1/affs/0238.aff -> /Volumes/s.eecs.harvard.edu/p1/0238.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163878 new: 6014 LZred: 96.33% +/Volumes/TB/project1/affs/0241.aff -> /Volumes/s.eecs.harvard.edu/p1/0241.aff Nochg: 0 NUL: 0 LZMA: 1 old: 80992 new: 3096 LZred: 96.18% +/Volumes/TB/project1/affs/0242.aff -> /Volumes/s.eecs.harvard.edu/p1/0242.aff Nochg: 0 NUL: 0 LZMA: 3 old: 10326739 new: 8198177 LZred: 20.61% +/Volumes/TB/project1/affs/0245.aff -> /Volumes/s.eecs.harvard.edu/p1/0245.aff Nochg: 0 NUL: 0 LZMA: 33 old: 220186553 new: 173306299 LZred: 21.29% +/Volumes/TB/project1/affs/0246.aff -> /Volumes/s.eecs.harvard.edu/p1/0246.aff Nochg: 0 NUL: 0 LZMA: 3 old: 4221346 new: 3229569 LZred: 23.49% +/Volumes/TB/project1/affs/0247.aff -> /Volumes/s.eecs.harvard.edu/p1/0247.aff Nochg: 0 NUL: 23 LZMA: 3 old: 140948 new: 12158 LZred: 91.37% +/Volumes/TB/project1/affs/0248.aff -> /Volumes/s.eecs.harvard.edu/p1/0248.aff Nochg: 0 NUL: 0 LZMA: 4 old: 16909582 new: 11534284 LZred: 31.79% +/Volumes/TB/project1/affs/0249.aff -> /Volumes/s.eecs.harvard.edu/p1/0249.aff Nochg: 0 NUL: 3 LZMA: 10 old: 36308444 new: 24003975 LZred: 33.89% +/Volumes/TB/project1/affs/0250.aff -> /Volumes/s.eecs.harvard.edu/p1/0250.aff Nochg: 0 NUL: 0 LZMA: 2 old: 164000 new: 6022 LZred: 96.33% +/Volumes/TB/project1/affs/0251.aff -> /Volumes/s.eecs.harvard.edu/p1/0251.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163991 new: 6014 LZred: 96.33% +/Volumes/TB/project1/affs/0252.aff -> /Volumes/s.eecs.harvard.edu/p1/0252.aff Nochg: 0 NUL: 0 LZMA: 259 old: 1956911625 new: 1497362059 LZred: 23.48% +/Volumes/TB/project1/affs/0256.aff -> /Volumes/s.eecs.harvard.edu/p1/0256.aff Nochg: 0 NUL: 144 LZMA: 1 old: 20258 new: 2986 LZred: 85.26% +/Volumes/TB/project1/affs/0258.aff -> /Volumes/s.eecs.harvard.edu/p1/0258.aff Nochg: 0 NUL: 0 LZMA: 143 old: 506375029 new: 361793163 LZred: 28.55% +/Volumes/TB/project1/affs/0259.aff -> /Volumes/s.eecs.harvard.edu/p1/0259.aff Nochg: 0 NUL: 382 LZMA: 3 old: 100880 new: 10623 LZred: 89.47% +/Volumes/TB/project1/affs/0263.aff -> /Volumes/s.eecs.harvard.edu/p1/0263.aff Nochg: 0 NUL: 156 LZMA: 1 old: 51908 new: 2987 LZred: 94.25% +/Volumes/TB/project1/affs/0264.aff -> /Volumes/s.eecs.harvard.edu/p1/0264.aff Nochg: 0 NUL: 257 LZMA: 1 old: 81068 new: 2986 LZred: 96.32% +/Volumes/TB/project1/affs/0265.aff -> /Volumes/s.eecs.harvard.edu/p1/0265.aff Nochg: 0 NUL: 0 LZMA: 194 old: 29926051 new: 6817541 LZred: 77.22% +/Volumes/TB/project1/affs/0266.aff -> /Volumes/s.eecs.harvard.edu/p1/0266.aff Nochg: 0 NUL: 150 LZMA: 2 old: 128910 new: 6022 LZred: 95.33% +/Volumes/TB/project1/affs/0267.aff -> /Volumes/s.eecs.harvard.edu/p1/0267.aff Nochg: 0 NUL: 0 LZMA: 194 old: 29924077 new: 6816712 LZred: 77.22% +/Volumes/TB/project1/affs/0268.aff -> /Volumes/s.eecs.harvard.edu/p1/0268.aff Nochg: 0 NUL: 82 LZMA: 5 old: 19323745 new: 11578039 LZred: 40.08% +/Volumes/TB/project1/affs/0269.aff -> /Volumes/s.eecs.harvard.edu/p1/0269.aff Nochg: 0 NUL: 127 LZMA: 3 old: 741452 new: 61641 LZred: 91.69% +/Volumes/TB/project1/affs/0270.aff -> /Volumes/s.eecs.harvard.edu/p1/0270.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163997 new: 6019 LZred: 96.33% +/Volumes/TB/project1/affs/0271.aff -> /Volumes/s.eecs.harvard.edu/p1/0271.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163991 new: 6019 LZred: 96.33% +/Volumes/TB/project1/affs/0272.aff -> /Volumes/s.eecs.harvard.edu/p1/0272.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163988 new: 6012 LZred: 96.33% +/Volumes/TB/project1/affs/0273.aff -> /Volumes/s.eecs.harvard.edu/p1/0273.aff Nochg: 0 NUL: 0 LZMA: 3 old: 221550 new: 8940 LZred: 95.96% +/Volumes/TB/project1/affs/0275.aff -> /Volumes/s.eecs.harvard.edu/p1/0275.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163998 new: 6019 LZred: 96.33% +/Volumes/TB/project1/affs/0276.aff -> /Volumes/s.eecs.harvard.edu/p1/0276.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163994 new: 6019 LZred: 96.33% +/Volumes/TB/project1/affs/0277.aff -> /Volumes/s.eecs.harvard.edu/p1/0277.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163996 new: 6019 LZred: 96.33% +/Volumes/TB/project1/affs/0278.aff -> /Volumes/s.eecs.harvard.edu/p1/0278.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163992 new: 6019 LZred: 96.33% +/Volumes/TB/project1/affs/0279.aff -> /Volumes/s.eecs.harvard.edu/p1/0279.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163992 new: 6017 LZred: 96.33% +/Volumes/TB/project1/affs/0281.aff -> /Volumes/s.eecs.harvard.edu/p1/0281.aff Nochg: 0 NUL: 0 LZMA: 2 old: 164001 new: 6019 LZred: 96.33% +/Volumes/TB/project1/affs/0283.aff -> /Volumes/s.eecs.harvard.edu/p1/0283.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163994 new: 6019 LZred: 96.33% +/Volumes/TB/project1/affs/0284.aff -> /Volumes/s.eecs.harvard.edu/p1/0284.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163999 new: 6019 LZred: 96.33% +/Volumes/TB/project1/affs/0286.aff -> /Volumes/s.eecs.harvard.edu/p1/0286.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163997 new: 6019 LZred: 96.33% +/Volumes/TB/project1/affs/0287.aff -> /Volumes/s.eecs.harvard.edu/p1/0287.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163992 new: 6018 LZred: 96.33% +/Volumes/TB/project1/affs/0289.aff -> /Volumes/s.eecs.harvard.edu/p1/0289.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163995 new: 6021 LZred: 96.33% +/Volumes/TB/project1/affs/0290.aff -> /Volumes/s.eecs.harvard.edu/p1/0290.aff Nochg: 0 NUL: 253 LZMA: 3 old: 291540 new: 226713 LZred: 22.24% +/Volumes/TB/project1/affs/0291.aff -> /Volumes/s.eecs.harvard.edu/p1/0291.aff Nochg: 8 NUL: 0 LZMA: 198 old: 2732269022 new: 2482687600 LZred: 9.13% +/Volumes/TB/project1/affs/0292.aff -> /Volumes/s.eecs.harvard.edu/p1/0292.aff Nochg: 0 NUL: 0 LZMA: 204 old: 2190135991 new: 2025730065 LZred: 7.51% +/Volumes/TB/project1/affs/0293.aff -> /Volumes/s.eecs.harvard.edu/p1/0293.aff Nochg: 0 NUL: 257 LZMA: 1 old: 81066 new: 2985 LZred: 96.32% +/Volumes/TB/project1/affs/0295.aff -> /Volumes/s.eecs.harvard.edu/p1/0295.aff Nochg: 0 NUL: 257 LZMA: 1 old: 81067 new: 2985 LZred: 96.32% +/Volumes/TB/project1/affs/0300.aff -> /Volumes/s.eecs.harvard.edu/p1/0300.aff Nochg: 0 NUL: 0 LZMA: 192 old: 1413529204 new: 1109771975 LZred: 21.49% +/Volumes/TB/project1/affs/0301.aff -> /Volumes/s.eecs.harvard.edu/p1/0301.aff Nochg: 0 NUL: 0 LZMA: 188 old: 2215990390 new: 692069946 LZred: 68.77% +/Volumes/TB/project1/affs/0302.aff -> /Volumes/s.eecs.harvard.edu/p1/0302.aff Nochg: 0 NUL: 192 LZMA: 1 old: 17677 new: 3630 LZred: 79.46% +/Volumes/TB/project1/affs/0303.aff -> /Volumes/s.eecs.harvard.edu/p1/0303.aff Nochg: 0 NUL: 188 LZMA: 1 old: 17677 new: 3631 LZred: 79.46% +/Volumes/TB/project1/affs/0305.aff -> /Volumes/s.eecs.harvard.edu/p1/0305.aff Nochg: 0 NUL: 0 LZMA: 1194 old: 20019793148 new: 161757662 LZred: 99.19% +/Volumes/TB/project1/affs/0308.aff -> /Volumes/s.eecs.harvard.edu/p1/0308.aff Nochg: 0 NUL: 0 LZMA: 1210 old: 1035508230 new: 873339685 LZred: 15.66% +/Volumes/TB/project1/affs/0310.aff -> /Volumes/s.eecs.harvard.edu/p1/0310.aff Nochg: 0 NUL: 0 LZMA: 49 old: 412219606 new: 349021658 LZred: 15.33% +/Volumes/TB/project1/affs/0312.aff -> /Volumes/s.eecs.harvard.edu/p1/0312.aff Nochg: 0 NUL: 86 LZMA: 315 old: 6809195 new: 1589324 LZred: 76.66% +/Volumes/TB/project1/affs/0313.aff -> /Volumes/s.eecs.harvard.edu/p1/0313.aff Nochg: 0 NUL: 5 LZMA: 1216 old: 3602356312 new: 2913252760 LZred: 19.13% +/Volumes/TB/project1/affs/0314.aff -> /Volumes/s.eecs.harvard.edu/p1/0314.aff Nochg: 0 NUL: 924 LZMA: 265 old: 2261609805 new: 1766365867 LZred: 21.90% +/Volumes/TB/project1/affs/0315.aff -> /Volumes/s.eecs.harvard.edu/p1/0315.aff Nochg: 1 NUL: 508 LZMA: 664 old: 8288377753 new: 7200653541 LZred: 13.12% +/Volumes/TB/project1/affs/0316.aff -> /Volumes/s.eecs.harvard.edu/p1/0316.aff Nochg: 0 NUL: 0 LZMA: 382 old: 2687301767 new: 1997252183 LZred: 25.68% +/Volumes/TB/project1/affs/0317.aff -> /Volumes/s.eecs.harvard.edu/p1/0317.aff Nochg: 0 NUL: 0 LZMA: 381 old: 1928221779 new: 1509138504 LZred: 21.73% +/Volumes/TB/project1/affs/0318.aff -> /Volumes/s.eecs.harvard.edu/p1/0318.aff Nochg: 0 NUL: 0 LZMA: 6 old: 16259483 new: 11548994 LZred: 28.97% +/Volumes/TB/project1/affs/0319.aff -> /Volumes/s.eecs.harvard.edu/p1/0319.aff Nochg: 0 NUL: 0 LZMA: 5 old: 19791659 new: 14750279 LZred: 25.47% +/Volumes/TB/project1/affs/0320.aff -> /Volumes/s.eecs.harvard.edu/p1/0320.aff Nochg: 0 NUL: 1127 LZMA: 92 old: 967962111 new: 833795826 LZred: 13.86% +/Volumes/TB/project1/affs/0321.aff -> /Volumes/s.eecs.harvard.edu/p1/0321.aff Nochg: 0 NUL: 0 LZMA: 381 old: 1680748486 new: 1224972116 LZred: 27.12% +/Volumes/TB/project1/affs/0322.aff -> /Volumes/s.eecs.harvard.edu/p1/0322.aff Nochg: 0 NUL: 0 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project1/affs/0323.aff -> /Volumes/s.eecs.harvard.edu/p1/0323.aff Nochg: 0 NUL: 0 LZMA: 385 old: 1660505326 new: 1097665864 LZred: 33.90% +/Volumes/TB/project1/affs/0324.aff -> /Volumes/s.eecs.harvard.edu/p1/0324.aff Nochg: 0 NUL: 0 LZMA: 383 old: 2906768928 new: 2256339011 LZred: 22.38% +/Volumes/TB/project1/affs/0325.aff -> /Volumes/s.eecs.harvard.edu/p1/0325.aff Nochg: 0 NUL: 0 LZMA: 383 old: 1708668249 new: 1308554129 LZred: 23.42% +/Volumes/TB/project1/affs/0326.aff -> /Volumes/s.eecs.harvard.edu/p1/0326.aff Nochg: 0 NUL: 0 LZMA: 385 old: 1516908302 new: 1054634212 LZred: 30.47% +/Volumes/TB/project1/affs/0327.aff -> /Volumes/s.eecs.harvard.edu/p1/0327.aff Nochg: 0 NUL: 0 LZMA: 61 old: 361728360 new: 270713134 LZred: 25.16% +/Volumes/TB/project1/affs/0328.aff -> /Volumes/s.eecs.harvard.edu/p1/0328.aff Nochg: 0 NUL: 21 LZMA: 124 old: 905334797 new: 713742894 LZred: 21.16% +/Volumes/TB/project1/affs/0329.aff -> /Volumes/s.eecs.harvard.edu/p1/0329.aff Nochg: 0 NUL: 0 LZMA: 71 old: 431830511 new: 316620019 LZred: 26.68% +/Volumes/TB/project1/affs/0330.aff -> /Volumes/s.eecs.harvard.edu/p1/0330.aff Nochg: 0 NUL: 0 LZMA: 1 old: 105663 new: 21279 LZred: 79.86% +/Volumes/TB/project1/affs/0331.aff -> /Volumes/s.eecs.harvard.edu/p1/0331.aff Nochg: 0 NUL: 74 LZMA: 83 old: 399979875 new: 292540299 LZred: 26.86% +/Volumes/TB/project1/affs/0332.aff -> /Volumes/s.eecs.harvard.edu/p1/0332.aff Nochg: 0 NUL: 91 LZMA: 37 old: 169626601 new: 120329369 LZred: 29.06% +/Volumes/TB/project1/affs/0333.aff -> /Volumes/s.eecs.harvard.edu/p1/0333.aff Nochg: 0 NUL: 96 LZMA: 289 old: 1803600795 new: 1201099867 LZred: 33.41% +/Volumes/TB/project1/affs/0334.aff -> /Volumes/s.eecs.harvard.edu/p1/0334.aff Nochg: 0 NUL: 0 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project1/affs/0335.aff -> /Volumes/s.eecs.harvard.edu/p1/0335.aff Nochg: 0 NUL: 0 LZMA: 41 old: 310714791 new: 243264931 LZred: 21.71% +/Volumes/TB/project1/affs/0338.aff -> /Volumes/s.eecs.harvard.edu/p1/0338.aff Nochg: 0 NUL: 0 LZMA: 809 old: 2721506317 new: 2206947975 LZred: 18.91% +/Volumes/TB/project1/affs/0339.aff -> /Volumes/s.eecs.harvard.edu/p1/0339.aff Nochg: 0 NUL: 0 LZMA: 540 old: 1084778531 new: 976174308 LZred: 10.01% +/Volumes/TB/project1/affs/0340.aff -> /Volumes/s.eecs.harvard.edu/p1/0340.aff Nochg: 0 NUL: 0 LZMA: 540 old: 1083625532 new: 975167294 LZred: 10.01% +/Volumes/TB/project1/affs/0341.aff -> /Volumes/s.eecs.harvard.edu/p1/0341.aff Nochg: 0 NUL: 0 LZMA: 540 old: 60367048 new: 3532052 LZred: 94.15% +/Volumes/TB/project1/affs/0342.aff -> /Volumes/s.eecs.harvard.edu/p1/0342.aff Nochg: 0 NUL: 0 LZMA: 540 old: 1086189007 new: 977936233 LZred: 9.97% +/Volumes/TB/project1/affs/0343.aff -> /Volumes/s.eecs.harvard.edu/p1/0343.aff Nochg: 0 NUL: 0 LZMA: 540 old: 1084485494 new: 976144576 LZred: 9.99% +/Volumes/TB/project1/affs/0344.aff -> /Volumes/s.eecs.harvard.edu/p1/0344.aff Nochg: 0 NUL: 0 LZMA: 540 old: 60384085 new: 3532564 LZred: 94.15% +/Volumes/TB/project1/affs/0345.aff -> /Volumes/s.eecs.harvard.edu/p1/0345.aff Nochg: 0 NUL: 0 LZMA: 540 old: 1084229690 new: 975798215 LZred: 10.00% +/Volumes/TB/project1/affs/0346.aff -> /Volumes/s.eecs.harvard.edu/p1/0346.aff Nochg: 0 NUL: 0 LZMA: 540 old: 1085614938 new: 977138339 LZred: 9.99% +/Volumes/TB/project1/affs/0347.aff -> /Volumes/s.eecs.harvard.edu/p1/0347.aff Nochg: 0 NUL: 0 LZMA: 540 old: 95460383 new: 21086472 LZred: 77.91% +/Volumes/TB/project1/affs/0348.aff -> /Volumes/s.eecs.harvard.edu/p1/0348.aff Nochg: 0 NUL: 0 LZMA: 540 old: 1082462847 new: 974590752 LZred: 9.97% +/Volumes/TB/project1/affs/0349.aff -> /Volumes/s.eecs.harvard.edu/p1/0349.aff Nochg: 0 NUL: 0 LZMA: 540 old: 1084125471 new: 976585573 LZred: 9.92% +/Volumes/TB/project1/affs/0350.aff -> /Volumes/s.eecs.harvard.edu/p1/0350.aff Nochg: 0 NUL: 0 LZMA: 540 old: 1082772737 new: 974957061 LZred: 9.96% +/Volumes/TB/project1/affs/0351.aff -> /Volumes/s.eecs.harvard.edu/p1/0351.aff Nochg: 0 NUL: 0 LZMA: 540 old: 1084029455 new: 976308645 LZred: 9.94% +/Volumes/TB/project1/affs/0354.aff -> /Volumes/s.eecs.harvard.edu/p1/0354.aff Nochg: 0 NUL: 0 LZMA: 540 old: 1085298608 new: 977028433 LZred: 9.98% +/Volumes/TB/project1/affs/0355.aff -> /Volumes/s.eecs.harvard.edu/p1/0355.aff Nochg: 0 NUL: 0 LZMA: 504 old: 1031231432 new: 928435878 LZred: 9.97% +/Volumes/TB/project1/affs/0356.aff -> /Volumes/s.eecs.harvard.edu/p1/0356.aff Nochg: 0 NUL: 0 LZMA: 540 old: 1083745688 new: 976016354 LZred: 9.94% +/Volumes/TB/project1/affs/0358.aff -> /Volumes/s.eecs.harvard.edu/p1/0358.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163997 new: 6017 LZred: 96.33% +/Volumes/TB/project1/affs/0359.aff -> /Volumes/s.eecs.harvard.edu/p1/0359.aff Nochg: 0 NUL: 0 LZMA: 2 old: 164004 new: 6015 LZred: 96.33% +/Volumes/TB/project1/affs/0360.aff -> /Volumes/s.eecs.harvard.edu/p1/0360.aff Nochg: 0 NUL: 0 LZMA: 3 old: 338195 new: 118902 LZred: 64.84% +/Volumes/TB/project1/affs/0362.aff -> /Volumes/s.eecs.harvard.edu/p1/0362.aff Nochg: 1 NUL: 0 LZMA: 26 old: 380945627 new: 375977075 LZred: 1.30% +/Volumes/TB/project1/affs/0363.aff -> /Volumes/s.eecs.harvard.edu/p1/0363.aff Nochg: 0 NUL: 0 LZMA: 25 old: 229284601 new: 205672115 LZred: 10.30% +/Volumes/TB/project1/affs/0364.aff -> /Volumes/s.eecs.harvard.edu/p1/0364.aff Nochg: 0 NUL: 16 LZMA: 10 old: 65083603 new: 47123024 LZred: 27.60% +/Volumes/TB/project1/affs/0365.aff -> /Volumes/s.eecs.harvard.edu/p1/0365.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163897 new: 6015 LZred: 96.33% +/Volumes/TB/project1/affs/0366.aff -> /Volumes/s.eecs.harvard.edu/p1/0366.aff Nochg: 0 NUL: 0 LZMA: 77 old: 662182555 new: 549195077 LZred: 17.06% +/Volumes/TB/project1/affs/0367.aff -> /Volumes/s.eecs.harvard.edu/p1/0367.aff Nochg: 0 NUL: 0 LZMA: 2 old: 165029 new: 6913 LZred: 95.81% +/Volumes/TB/project1/affs/0370.aff -> /Volumes/s.eecs.harvard.edu/p1/0370.aff Nochg: 0 NUL: 0 LZMA: 2 old: 164002 new: 6016 LZred: 96.33% +/Volumes/TB/project1/affs/0371.aff -> /Volumes/s.eecs.harvard.edu/p1/0371.aff Nochg: 0 NUL: 0 LZMA: 33 old: 277752383 new: 234716015 LZred: 15.49% +/Volumes/TB/project1/affs/0372.aff -> /Volumes/s.eecs.harvard.edu/p1/0372.aff Nochg: 0 NUL: 0 LZMA: 34 old: 237234997 new: 179823046 LZred: 24.20% +/Volumes/TB/project1/affs/0373.aff -> /Volumes/s.eecs.harvard.edu/p1/0373.aff Nochg: 0 NUL: 0 LZMA: 108 old: 751783203 new: 607352774 LZred: 19.21% +/Volumes/TB/project1/affs/0375.aff -> /Volumes/s.eecs.harvard.edu/p1/0375.aff Nochg: 0 NUL: 0 LZMA: 129 old: 18921950 new: 1936962 LZred: 89.76% +/Volumes/TB/project1/affs/0376.aff -> /Volumes/s.eecs.harvard.edu/p1/0376.aff Nochg: 0 NUL: 0 LZMA: 129 old: 18921603 new: 1936793 LZred: 89.76% +/Volumes/TB/project1/affs/0377.aff -> /Volumes/s.eecs.harvard.edu/p1/0377.aff Nochg: 0 NUL: 0 LZMA: 129 old: 18921973 new: 1936963 LZred: 89.76% +/Volumes/TB/project1/affs/0378.aff -> /Volumes/s.eecs.harvard.edu/p1/0378.aff Nochg: 0 NUL: 0 LZMA: 129 old: 18921954 new: 1936936 LZred: 89.76% +/Volumes/TB/project1/affs/0380.aff -> /Volumes/s.eecs.harvard.edu/p1/0380.aff Nochg: 0 NUL: 71 LZMA: 2 old: 1279672 new: 1041508 LZred: 18.61% +/Volumes/TB/project1/affs/0381.aff -> /Volumes/s.eecs.harvard.edu/p1/0381.aff Nochg: 0 NUL: 0 LZMA: 44 old: 679533250 new: 648822174 LZred: 4.52% +/Volumes/TB/project1/affs/0382.aff -> /Volumes/s.eecs.harvard.edu/p1/0382.aff Nochg: 0 NUL: 0 LZMA: 17 old: 76319484 new: 64832108 LZred: 15.05% +/Volumes/TB/project1/affs/0388.aff -> /Volumes/s.eecs.harvard.edu/p1/0388.aff Nochg: 0 NUL: 1 LZMA: 255 old: 1241373411 new: 566803620 LZred: 54.34% +/Volumes/TB/project1/affs/0389.aff -> /Volumes/s.eecs.harvard.edu/p1/0389.aff Nochg: 0 NUL: 253 LZMA: 3 old: 59403 new: 9899 LZred: 83.34% +/Volumes/TB/project1/affs/0390.aff -> /Volumes/s.eecs.harvard.edu/p1/0390.aff Nochg: 0 NUL: 0 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project1/affs/0391.aff -> /Volumes/s.eecs.harvard.edu/p1/0391.aff Nochg: 0 NUL: 0 LZMA: 6 old: 46691785 new: 40518080 LZred: 13.22% +/Volumes/TB/project1/affs/0392.aff -> /Volumes/s.eecs.harvard.edu/p1/0392.aff Nochg: 0 NUL: 253 LZMA: 3 old: 59398 new: 9897 LZred: 83.34% +/Volumes/TB/project1/affs/0394.aff -> /Volumes/s.eecs.harvard.edu/p1/0394.aff Nochg: 0 NUL: 0 LZMA: 145 old: 22778794 new: 5816074 LZred: 74.47% +/Volumes/TB/project1/affs/0397.aff -> /Volumes/s.eecs.harvard.edu/p1/0397.aff Nochg: 0 NUL: 0 LZMA: 1 old: 28139 new: 3084 LZred: 89.04% +/Volumes/TB/project1/affs/0399.aff -> /Volumes/s.eecs.harvard.edu/p1/0399.aff Nochg: 0 NUL: 0 LZMA: 251 old: 1986893644 new: 1581680832 LZred: 20.39% +/Volumes/TB/project1/affs/0400.aff -> /Volumes/s.eecs.harvard.edu/p1/0400.aff Nochg: 0 NUL: 0 LZMA: 1 old: 55182 new: 3279 LZred: 94.06% +/Volumes/TB/project1/affs/0401.aff -> /Volumes/s.eecs.harvard.edu/p1/0401.aff Nochg: 1 NUL: 505 LZMA: 101 old: 937243394 new: 802974373 LZred: 14.33% +/Volumes/TB/project1/affs/0402.aff -> /Volumes/s.eecs.harvard.edu/p1/0402.aff Nochg: 1 NUL: 577 LZMA: 28 old: 174520471 new: 150258964 LZred: 13.90% +/Volumes/TB/project1/affs/0403.aff -> /Volumes/s.eecs.harvard.edu/p1/0403.aff Nochg: 0 NUL: 577 LZMA: 28 old: 174578691 new: 150360133 LZred: 13.87% +/Volumes/TB/project1/affs/0404.aff -> /Volumes/s.eecs.harvard.edu/p1/0404.aff Nochg: 1 NUL: 556 LZMA: 49 old: 475670348 new: 425488337 LZred: 10.55% +/Volumes/TB/project1/affs/0405.aff -> /Volumes/s.eecs.harvard.edu/p1/0405.aff Nochg: 0 NUL: 0 LZMA: 1 old: 28136 new: 3085 LZred: 89.04% +/Volumes/TB/project1/affs/0406.aff -> /Volumes/s.eecs.harvard.edu/p1/0406.aff Nochg: 0 NUL: 578 LZMA: 26 old: 158179583 new: 133776691 LZred: 15.43% +/Volumes/TB/project1/affs/0407.aff -> /Volumes/s.eecs.harvard.edu/p1/0407.aff Nochg: 0 NUL: 0 LZMA: 65 old: 340149336 new: 246305696 LZred: 27.59% +/Volumes/TB/project1/affs/0409.aff -> /Volumes/s.eecs.harvard.edu/p1/0409.aff Nochg: 0 NUL: 0 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project1/affs/0410.aff -> /Volumes/s.eecs.harvard.edu/p1/0410.aff Nochg: 14 NUL: 0 LZMA: 433 old: 1782452854 new: 1521879895 LZred: 14.62% +/Volumes/TB/project1/affs/0411.aff -> /Volumes/s.eecs.harvard.edu/p1/0411.aff Nochg: 0 NUL: 0 LZMA: 1 old: 14741 new: 10627 LZred: 27.91% +/Volumes/TB/project1/affs/0412.aff -> /Volumes/s.eecs.harvard.edu/p1/0412.aff Nochg: 0 NUL: 0 LZMA: 152 old: 724870531 new: 563219638 LZred: 22.30% +/Volumes/TB/project1/affs/0413.aff -> /Volumes/s.eecs.harvard.edu/p1/0413.aff Nochg: 102 NUL: 0 LZMA: 24 old: 393447432 new: 342097006 LZred: 13.05% +/Volumes/TB/project1/affs/0414.aff -> /Volumes/s.eecs.harvard.edu/p1/0414.aff Nochg: 0 NUL: 43 LZMA: 8 old: 36448132 new: 27225221 LZred: 25.30% +/Volumes/TB/project1/affs/0415.aff -> /Volumes/s.eecs.harvard.edu/p1/0415.aff Nochg: 0 NUL: 0 LZMA: 255 old: 1095661669 new: 795359118 LZred: 27.41% +/Volumes/TB/project1/affs/0416.aff -> /Volumes/s.eecs.harvard.edu/p1/0416.aff Nochg: 0 NUL: 0 LZMA: 257 old: 2548628115 new: 2130832587 LZred: 16.39% +/Volumes/TB/project1/affs/0417.aff -> /Volumes/s.eecs.harvard.edu/p1/0417.aff Nochg: 0 NUL: 0 LZMA: 6 old: 30266265 new: 27032273 LZred: 10.69% +/Volumes/TB/project1/affs/0418.aff -> /Volumes/s.eecs.harvard.edu/p1/0418.aff Nochg: 0 NUL: 0 LZMA: 75 old: 737956458 new: 654884167 LZred: 11.26% +/Volumes/TB/project1/affs/0419.aff -> /Volumes/s.eecs.harvard.edu/p1/0419.aff Nochg: 0 NUL: 0 LZMA: 107 old: 1083119625 new: 955618238 LZred: 11.77% +/Volumes/TB/project1/affs/0420.aff -> /Volumes/s.eecs.harvard.edu/p1/0420.aff Nochg: 0 NUL: 0 LZMA: 14 old: 90511601 new: 70212516 LZred: 22.43% +/Volumes/TB/project1/affs/0421.aff -> /Volumes/s.eecs.harvard.edu/p1/0421.aff Nochg: 0 NUL: 0 LZMA: 11 old: 39409297 new: 27706094 LZred: 29.70% +/Volumes/TB/project1/affs/0423.aff -> /Volumes/s.eecs.harvard.edu/p1/0423.aff Nochg: 1 NUL: 3 LZMA: 253 old: 2133493239 new: 1787535475 LZred: 16.22% +/Volumes/TB/project1/affs/0424.aff -> /Volumes/s.eecs.harvard.edu/p1/0424.aff Nochg: 0 NUL: 0 LZMA: 378 old: 3400485057 new: 2576703769 LZred: 24.23% +/Volumes/TB/project1/affs/0425.aff -> /Volumes/s.eecs.harvard.edu/p1/0425.aff Nochg: 0 NUL: 255 LZMA: 3 old: 1857762 new: 255568 LZred: 86.24% +/Volumes/TB/project1/affs/0426.aff -> /Volumes/s.eecs.harvard.edu/p1/0426.aff Nochg: 0 NUL: 0 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project1/affs/0427.aff -> /Volumes/s.eecs.harvard.edu/p1/0427.aff Nochg: 0 NUL: 0 LZMA: 253 old: 2227202094 new: 1813002120 LZred: 18.60% +/Volumes/TB/project1/affs/0428.aff -> /Volumes/s.eecs.harvard.edu/p1/0428.aff Nochg: 0 NUL: 0 LZMA: 73 old: 635970066 new: 530440994 LZred: 16.59% +/Volumes/TB/project1/affs/0429.aff -> /Volumes/s.eecs.harvard.edu/p1/0429.aff Nochg: 0 NUL: 0 LZMA: 253 old: 2321478086 new: 1896936117 LZred: 18.29% +/Volumes/TB/project1/affs/0430.aff -> /Volumes/s.eecs.harvard.edu/p1/0430.aff Nochg: 0 NUL: 0 LZMA: 253 old: 2158841466 new: 1754549807 LZred: 18.73% +/Volumes/TB/project1/affs/0432.aff -> /Volumes/s.eecs.harvard.edu/p1/0432.aff Nochg: 0 NUL: 0 LZMA: 78 old: 604655291 new: 490663771 LZred: 18.85% +/Volumes/TB/project1/affs/0438.aff -> /Volumes/s.eecs.harvard.edu/p1/0438.aff Nochg: 0 NUL: 382 LZMA: 3 old: 1347109 new: 1131863 LZred: 15.98% +/Volumes/TB/project1/affs/0439.aff -> /Volumes/s.eecs.harvard.edu/p1/0439.aff Nochg: 0 NUL: 382 LZMA: 3 old: 1346619 new: 1131191 LZred: 16.00% +/Volumes/TB/project1/affs/0440.aff -> /Volumes/s.eecs.harvard.edu/p1/0440.aff Nochg: 0 NUL: 0 LZMA: 67 old: 485197900 new: 338570783 LZred: 30.22% +/Volumes/TB/project1/affs/0441.aff -> /Volumes/s.eecs.harvard.edu/p1/0441.aff Nochg: 0 NUL: 0 LZMA: 76 old: 607176983 new: 504623189 LZred: 16.89% +/Volumes/TB/project1/affs/0445.aff -> /Volumes/s.eecs.harvard.edu/p1/0445.aff Nochg: 0 NUL: 0 LZMA: 49 old: 231071883 new: 181881265 LZred: 21.29% +/Volumes/TB/project1/affs/0446.aff -> /Volumes/s.eecs.harvard.edu/p1/0446.aff Nochg: 0 NUL: 30 LZMA: 2 old: 261242 new: 216951 LZred: 16.95% +/Volumes/TB/project1/affs/0452.aff -> /Volumes/s.eecs.harvard.edu/p1/0452.aff Nochg: 0 NUL: 87 LZMA: 169 old: 5506701 new: 828802 LZred: 84.95% +/Volumes/TB/project1/affs/0453.aff -> /Volumes/s.eecs.harvard.edu/p1/0453.aff Nochg: 0 NUL: 87 LZMA: 169 old: 2758765 new: 414311 LZred: 84.98% +/Volumes/TB/project1/affs/0456.aff -> /Volumes/s.eecs.harvard.edu/p1/0456.aff Nochg: 0 NUL: 4 LZMA: 252 old: 4155934 new: 621562 LZred: 85.04% +/Volumes/TB/project1/affs/0457.aff -> /Volumes/s.eecs.harvard.edu/p1/0457.aff Nochg: 0 NUL: 4 LZMA: 252 old: 4155934 new: 621565 LZred: 85.04% +/Volumes/TB/project1/affs/0458.aff -> /Volumes/s.eecs.harvard.edu/p1/0458.aff Nochg: 0 NUL: 1 LZMA: 254 old: 185996061 new: 156236542 LZred: 16.00% +/Volumes/TB/project1/affs/0459.aff -> /Volumes/s.eecs.harvard.edu/p1/0459.aff Nochg: 0 NUL: 1 LZMA: 254 old: 149877040 new: 114907206 LZred: 23.33% +/Volumes/TB/project1/affs/0461.aff -> /Volumes/s.eecs.harvard.edu/p1/0461.aff Nochg: 0 NUL: 3 LZMA: 252 old: 722454903 new: 530770209 LZred: 26.53% +/Volumes/TB/project1/affs/0462.aff -> /Volumes/s.eecs.harvard.edu/p1/0462.aff Nochg: 0 NUL: 2 LZMA: 253 old: 246067151 new: 203612440 LZred: 17.25% +/Volumes/TB/project1/affs/0469.aff -> /Volumes/s.eecs.harvard.edu/p1/0469.aff Nochg: 0 NUL: 18 LZMA: 1195 old: 11767395553 new: 9457482099 LZred: 19.63% +/Volumes/TB/project1/affs/0470.aff -> /Volumes/s.eecs.harvard.edu/p1/0470.aff Nochg: 0 NUL: 0 LZMA: 148 old: 1262257911 new: 1043048718 LZred: 17.37% +/Volumes/TB/project1/affs/0471.aff -> /Volumes/s.eecs.harvard.edu/p1/0471.aff Nochg: 0 NUL: 0 LZMA: 248 old: 2327383189 new: 1979510583 LZred: 14.95% +/Volumes/TB/project1/affs/0472.aff -> /Volumes/s.eecs.harvard.edu/p1/0472.aff Nochg: 0 NUL: 0 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project1/affs/0473.aff -> /Volumes/s.eecs.harvard.edu/p1/0473.aff Nochg: 0 NUL: 0 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project1/affs/0474.aff -> /Volumes/s.eecs.harvard.edu/p1/0474.aff Nochg: 4 NUL: 0 LZMA: 280 old: 3778334940 new: 3410020289 LZred: 9.75% +/Volumes/TB/project1/affs/0475.aff -> /Volumes/s.eecs.harvard.edu/p1/0475.aff Nochg: 18 NUL: 0 LZMA: 207 old: 2382264218 new: 2186935932 LZred: 8.20% +/Volumes/TB/project1/affs/0476.aff -> /Volumes/s.eecs.harvard.edu/p1/0476.aff Nochg: 0 NUL: 2 LZMA: 451 old: 3031989383 new: 2372184988 LZred: 21.76% +/Volumes/TB/project1/affs/0478.aff -> /Volumes/s.eecs.harvard.edu/p1/0478.aff Nochg: 0 NUL: 2 LZMA: 253 old: 763345715 new: 624682321 LZred: 18.17% +/Volumes/TB/project1/affs/0479.aff -> /Volumes/s.eecs.harvard.edu/p1/0479.aff Nochg: 0 NUL: 0 LZMA: 185 old: 1315446623 new: 1117218428 LZred: 15.07% +/Volumes/TB/project1/affs/0481.aff -> /Volumes/s.eecs.harvard.edu/p1/0481.aff Nochg: 0 NUL: 0 LZMA: 312 old: 1946174527 new: 1364530920 LZred: 29.89% +/Volumes/TB/project1/affs/0482.aff -> /Volumes/s.eecs.harvard.edu/p1/0482.aff Nochg: 12 NUL: 0 LZMA: 165 old: 1338701914 new: 1102236809 LZred: 17.66% +/Volumes/TB/project1/affs/0485.aff -> /Volumes/s.eecs.harvard.edu/p1/0485.aff Nochg: 0 NUL: 0 LZMA: 153 old: 1087470978 new: 849168610 LZred: 21.91% +/Volumes/TB/project1/affs/0486.aff -> /Volumes/s.eecs.harvard.edu/p1/0486.aff Nochg: 0 NUL: 0 LZMA: 193 old: 494405045 new: 383235745 LZred: 22.49% +/Volumes/TB/project1/affs/0488.aff -> /Volumes/s.eecs.harvard.edu/p1/0488.aff Nochg: 0 NUL: 238 LZMA: 1 old: 51758 new: 2985 LZred: 94.23% +/Volumes/TB/project1/affs/0489.aff -> /Volumes/s.eecs.harvard.edu/p1/0489.aff Nochg: 0 NUL: 250 LZMA: 2 old: 58046 new: 6087 LZred: 89.51% +/Volumes/TB/project1/affs/0490.aff -> /Volumes/s.eecs.harvard.edu/p1/0490.aff Nochg: 0 NUL: 238 LZMA: 1 old: 51758 new: 2986 LZred: 94.23% +/Volumes/TB/project1/affs/0491.aff -> /Volumes/s.eecs.harvard.edu/p1/0491.aff Nochg: 0 NUL: 257 LZMA: 1 old: 81068 new: 2986 LZred: 96.32% +/Volumes/TB/project1/affs/0493.aff -> /Volumes/s.eecs.harvard.edu/p1/0493.aff Nochg: 0 NUL: 256 LZMA: 2 old: 97386 new: 5437 LZred: 94.42% +/Volumes/TB/project1/affs/0494.aff -> /Volumes/s.eecs.harvard.edu/p1/0494.aff Nochg: 0 NUL: 4 LZMA: 2 old: 97552 new: 5988 LZred: 93.86% +/Volumes/TB/project1/affs/0496.aff -> /Volumes/s.eecs.harvard.edu/p1/0496.aff Nochg: 0 NUL: 256 LZMA: 2 old: 114845 new: 12442 LZred: 89.17% +/Volumes/TB/project1/affs/0497.aff -> /Volumes/s.eecs.harvard.edu/p1/0497.aff Nochg: 0 NUL: 0 LZMA: 257 old: 615048978 new: 389440630 LZred: 36.68% +/Volumes/TB/project1/affs/0498.aff -> /Volumes/s.eecs.harvard.edu/p1/0498.aff Nochg: 0 NUL: 250 LZMA: 2 old: 396902 new: 316568 LZred: 20.24% +/Volumes/TB/project1/affs/0499.aff -> /Volumes/s.eecs.harvard.edu/p1/0499.aff Nochg: 0 NUL: 0 LZMA: 256 old: 1095775763 new: 795372115 LZred: 27.41% +/Volumes/TB/project1/affs/0511.aff -> /Volumes/s.eecs.harvard.edu/p1/0511.aff Nochg: 0 NUL: 124 LZMA: 2 old: 42634 new: 6617 LZred: 84.48% +/Volumes/TB/project1/affs/0512.aff -> /Volumes/s.eecs.harvard.edu/p1/0512.aff Nochg: 0 NUL: 0 LZMA: 1 old: 42400 new: 4263 LZred: 89.95% +/Volumes/TB/project1/affs/0513.aff -> /Volumes/s.eecs.harvard.edu/p1/0513.aff Nochg: 0 NUL: 151 LZMA: 2 old: 62068 new: 6615 LZred: 89.34% +/Volumes/TB/project1/affs/0514.aff -> /Volumes/s.eecs.harvard.edu/p1/0514.aff Nochg: 0 NUL: 0 LZMA: 153 old: 1175982636 new: 958637642 LZred: 18.48% +/Volumes/TB/project1/affs/0517.aff -> /Volumes/s.eecs.harvard.edu/p1/0517.aff Nochg: 0 NUL: 0 LZMA: 118 old: 1147182466 new: 946677905 LZred: 17.48% +/Volumes/TB/project1/affs/0518.aff -> /Volumes/s.eecs.harvard.edu/p1/0518.aff Nochg: 0 NUL: 121 LZMA: 3 old: 67658 new: 9721 LZred: 85.63% +/Volumes/TB/project1/affs/0519.aff -> /Volumes/s.eecs.harvard.edu/p1/0519.aff Nochg: 0 NUL: 124 LZMA: 2 old: 42638 new: 6617 LZred: 84.48% +/Volumes/TB/project1/affs/0520.aff -> /Volumes/s.eecs.harvard.edu/p1/0520.aff Nochg: 0 NUL: 151 LZMA: 2 old: 62067 new: 6615 LZred: 89.34% +/Volumes/TB/project1/affs/0521.aff -> /Volumes/s.eecs.harvard.edu/p1/0521.aff Nochg: 1 NUL: 0 LZMA: 498 old: 7335218536 new: 6893953235 LZred: 6.02% +/Volumes/TB/project1/affs/0522.aff -> /Volumes/s.eecs.harvard.edu/p1/0522.aff Nochg: 0 NUL: 0 LZMA: 189 old: 921553610 new: 717023119 LZred: 22.19% +/Volumes/TB/project1/affs/0523.aff -> /Volumes/s.eecs.harvard.edu/p1/0523.aff Nochg: 0 NUL: 0 LZMA: 4 old: 30599062 new: 29620170 LZred: 3.20% +/Volumes/TB/project1/affs/0524.aff -> /Volumes/s.eecs.harvard.edu/p1/0524.aff Nochg: 0 NUL: 0 LZMA: 192 old: 2148775715 new: 1885270866 LZred: 12.26% +/Volumes/TB/project1/affs/0525.aff -> /Volumes/s.eecs.harvard.edu/p1/0525.aff Nochg: 0 NUL: 382 LZMA: 3 old: 1382747 new: 1149204 LZred: 16.89% +/Volumes/TB/project1/affs/0526.aff -> /Volumes/s.eecs.harvard.edu/p1/0526.aff Nochg: 0 NUL: 383 LZMA: 2 old: 2035492 new: 1827250 LZred: 10.23% +/Volumes/TB/project1/affs/0527.aff -> /Volumes/s.eecs.harvard.edu/p1/0527.aff Nochg: 0 NUL: 382 LZMA: 3 old: 1348236 new: 1132081 LZred: 16.03% +/Volumes/TB/project1/affs/0528.aff -> /Volumes/s.eecs.harvard.edu/p1/0528.aff Nochg: 0 NUL: 84 LZMA: 2 old: 36928 new: 5679 LZred: 84.62% +/Volumes/TB/project1/affs/0529.aff -> /Volumes/s.eecs.harvard.edu/p1/0529.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163994 new: 6011 LZred: 96.33% +/Volumes/TB/project1/affs/0530.aff -> /Volumes/s.eecs.harvard.edu/p1/0530.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163996 new: 6012 LZred: 96.33% +/Volumes/TB/project1/affs/0531.aff -> /Volumes/s.eecs.harvard.edu/p1/0531.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163992 new: 6012 LZred: 96.33% +/Volumes/TB/project1/affs/0532.aff -> /Volumes/s.eecs.harvard.edu/p1/0532.aff Nochg: 0 NUL: 371 LZMA: 709 old: 1539150091 new: 1093862679 LZred: 28.93% +/Volumes/TB/project1/affs/0533.aff -> /Volumes/s.eecs.harvard.edu/p1/0533.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163991 new: 6012 LZred: 96.33% +/Volumes/TB/project1/affs/0534.aff -> /Volumes/s.eecs.harvard.edu/p1/0534.aff Nochg: 0 NUL: 0 LZMA: 19 old: 165403033 new: 145658257 LZred: 11.94% +/Volumes/TB/project1/affs/0535.aff -> /Volumes/s.eecs.harvard.edu/p1/0535.aff Nochg: 0 NUL: 292 LZMA: 255 old: 11115581 new: 5143121 LZred: 53.73% +/Volumes/TB/project1/affs/0536.aff -> /Volumes/s.eecs.harvard.edu/p1/0536.aff Nochg: 0 NUL: 1 LZMA: 128 old: 2091397 new: 314875 LZred: 84.94% +/Volumes/TB/project1/affs/0538.aff -> /Volumes/s.eecs.harvard.edu/p1/0538.aff Nochg: 0 NUL: 128 LZMA: 1 old: 17644 new: 2986 LZred: 83.08% +/Volumes/TB/project1/affs/0539.aff -> /Volumes/s.eecs.harvard.edu/p1/0539.aff Nochg: 0 NUL: 283 LZMA: 264 old: 287163970 new: 195795614 LZred: 31.82% +/Volumes/TB/project1/affs/0540.aff -> /Volumes/s.eecs.harvard.edu/p1/0540.aff Nochg: 0 NUL: 0 LZMA: 2 old: 164002 new: 6017 LZred: 96.33% +/Volumes/TB/project1/affs/0541.aff -> /Volumes/s.eecs.harvard.edu/p1/0541.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163989 new: 6013 LZred: 96.33% +/Volumes/TB/project1/affs/0542.aff -> /Volumes/s.eecs.harvard.edu/p1/0542.aff Nochg: 0 NUL: 0 LZMA: 21 old: 144352437 new: 110078179 LZred: 23.74% +/Volumes/TB/project1/affs/0543.aff -> /Volumes/s.eecs.harvard.edu/p1/0543.aff Nochg: 0 NUL: 0 LZMA: 23 old: 313976539 new: 289084346 LZred: 7.93% +/Volumes/TB/project1/affs/0544.aff -> /Volumes/s.eecs.harvard.edu/p1/0544.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163991 new: 6012 LZred: 96.33% +/Volumes/TB/project1/affs/0545.aff -> /Volumes/s.eecs.harvard.edu/p1/0545.aff Nochg: 0 NUL: 0 LZMA: 2 old: 164000 new: 6017 LZred: 96.33% +/Volumes/TB/project1/affs/0546.aff -> /Volumes/s.eecs.harvard.edu/p1/0546.aff Nochg: 0 NUL: 77 LZMA: 4 old: 102949 new: 10901 LZred: 89.41% +/Volumes/TB/project1/affs/0548.aff -> /Volumes/s.eecs.harvard.edu/p1/0548.aff Nochg: 0 NUL: 0 LZMA: 126 old: 288776355 new: 232652497 LZred: 19.44% +/Volumes/TB/project1/affs/0549.aff -> /Volumes/s.eecs.harvard.edu/p1/0549.aff Nochg: 0 NUL: 148 LZMA: 125 old: 345981263 new: 198440678 LZred: 42.64% +/Volumes/TB/project1/affs/0550.aff -> /Volumes/s.eecs.harvard.edu/p1/0550.aff Nochg: 0 NUL: 103 LZMA: 32 old: 169808496 new: 116682261 LZred: 31.29% +/Volumes/TB/project1/affs/0551.aff -> /Volumes/s.eecs.harvard.edu/p1/0551.aff Nochg: 0 NUL: 0 LZMA: 256 old: 140121530 new: 101851008 LZred: 27.31% +/Volumes/TB/project1/affs/0552.aff -> /Volumes/s.eecs.harvard.edu/p1/0552.aff Nochg: 0 NUL: 81 LZMA: 54 old: 173438546 new: 98666435 LZred: 43.11% +/Volumes/TB/project1/affs/0553.aff -> /Volumes/s.eecs.harvard.edu/p1/0553.aff Nochg: 1 NUL: 0 LZMA: 271 old: 620244727 new: 512126752 LZred: 17.43% +/Volumes/TB/project1/affs/0554.aff -> /Volumes/s.eecs.harvard.edu/p1/0554.aff Nochg: 1 NUL: 0 LZMA: 27 old: 181226360 new: 154946434 LZred: 14.50% +/Volumes/TB/project1/affs/0556.aff -> /Volumes/s.eecs.harvard.edu/p1/0556.aff Nochg: 0 NUL: 161 LZMA: 110 old: 587913291 new: 453478421 LZred: 22.87% +/Volumes/TB/project1/affs/0558.aff -> /Volumes/s.eecs.harvard.edu/p1/0558.aff Nochg: 0 NUL: 0 LZMA: 2 old: 185977 new: 9830 LZred: 94.71% +/Volumes/TB/project1/affs/0562.aff -> /Volumes/s.eecs.harvard.edu/p1/0562.aff Nochg: 0 NUL: 0 LZMA: 145 old: 927713079 new: 680152034 LZred: 26.69% +/Volumes/TB/project1/affs/0563.aff -> /Volumes/s.eecs.harvard.edu/p1/0563.aff Nochg: 2 NUL: 0 LZMA: 361 old: 4337441036 new: 3846834274 LZred: 11.31% +/Volumes/TB/project1/affs/0567.aff -> /Volumes/s.eecs.harvard.edu/p1/0567.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163996 new: 6015 LZred: 96.33% +/Volumes/TB/project1/affs/0568.aff -> /Volumes/s.eecs.harvard.edu/p1/0568.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163998 new: 6019 LZred: 96.33% +/Volumes/TB/project1/affs/0569.aff -> /Volumes/s.eecs.harvard.edu/p1/0569.aff Nochg: 0 NUL: 0 LZMA: 70 old: 498083763 new: 324455140 LZred: 34.86% +/Volumes/TB/project1/affs/0572.aff -> /Volumes/s.eecs.harvard.edu/p1/0572.aff Nochg: 0 NUL: 0 LZMA: 2 old: 185455 new: 12477 LZred: 93.27% +/Volumes/TB/project1/affs/0574.aff -> /Volumes/s.eecs.harvard.edu/p1/0574.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163991 new: 6014 LZred: 96.33% +/Volumes/TB/project1/affs/0576.aff -> /Volumes/s.eecs.harvard.edu/p1/0576.aff Nochg: 0 NUL: 0 LZMA: 2 old: 835451 new: 116495 LZred: 86.06% +/Volumes/TB/project1/affs/0577.aff -> /Volumes/s.eecs.harvard.edu/p1/0577.aff Nochg: 0 NUL: 0 LZMA: 102 old: 661240483 new: 390812601 LZred: 40.90% +/Volumes/TB/project1/affs/0578.aff -> /Volumes/s.eecs.harvard.edu/p1/0578.aff Nochg: 0 NUL: 0 LZMA: 102 old: 1134994287 new: 333522274 LZred: 70.61% +/Volumes/TB/project1/affs/0579.aff -> /Volumes/s.eecs.harvard.edu/p1/0579.aff Nochg: 0 NUL: 0 LZMA: 33 old: 246592768 new: 171360477 LZred: 30.51% +/Volumes/TB/project1/affs/0580.aff -> /Volumes/s.eecs.harvard.edu/p1/0580.aff Nochg: 0 NUL: 0 LZMA: 33 old: 217439833 new: 149643958 LZred: 31.18% +/Volumes/TB/project1/affs/0581.aff -> /Volumes/s.eecs.harvard.edu/p1/0581.aff Nochg: 0 NUL: 0 LZMA: 33 old: 282001986 new: 161098757 LZred: 42.87% +/Volumes/TB/project1/affs/0582.aff -> /Volumes/s.eecs.harvard.edu/p1/0582.aff Nochg: 0 NUL: 0 LZMA: 33 old: 243656947 new: 172145700 LZred: 29.35% +/Volumes/TB/project1/affs/0583.aff -> /Volumes/s.eecs.harvard.edu/p1/0583.aff Nochg: 0 NUL: 0 LZMA: 33 old: 262091485 new: 189454170 LZred: 27.71% +/Volumes/TB/project1/affs/0584.aff -> /Volumes/s.eecs.harvard.edu/p1/0584.aff Nochg: 0 NUL: 0 LZMA: 40 old: 209325779 new: 154573769 LZred: 26.16% +/Volumes/TB/project1/affs/0585.aff -> /Volumes/s.eecs.harvard.edu/p1/0585.aff Nochg: 0 NUL: 25 LZMA: 518 old: 2238945727 new: 1542375657 LZred: 31.11% +/Volumes/TB/project1/affs/0586.aff -> /Volumes/s.eecs.harvard.edu/p1/0586.aff Nochg: 0 NUL: 17 LZMA: 526 old: 2240837594 new: 1544355986 LZred: 31.08% +/Volumes/TB/project1/affs/0587.aff -> /Volumes/s.eecs.harvard.edu/p1/0587.aff Nochg: 0 NUL: 15 LZMA: 528 old: 2239779137 new: 1542857417 LZred: 31.12% +/Volumes/TB/project1/affs/0588.aff -> /Volumes/s.eecs.harvard.edu/p1/0588.aff Nochg: 0 NUL: 21 LZMA: 522 old: 2239347675 new: 1542366597 LZred: 31.12% +/Volumes/TB/project1/affs/0589.aff -> /Volumes/s.eecs.harvard.edu/p1/0589.aff Nochg: 0 NUL: 501 LZMA: 3 old: 59311 new: 10644 LZred: 82.05% +/Volumes/TB/project1/affs/0590.aff -> /Volumes/s.eecs.harvard.edu/p1/0590.aff Nochg: 0 NUL: 254 LZMA: 3 old: 85365 new: 9908 LZred: 88.39% +/Volumes/TB/project1/affs/0591.aff -> /Volumes/s.eecs.harvard.edu/p1/0591.aff Nochg: 0 NUL: 151 LZMA: 2 old: 46065 new: 7120 LZred: 84.54% +/Volumes/TB/project1/affs/0592.aff -> /Volumes/s.eecs.harvard.edu/p1/0592.aff Nochg: 0 NUL: 124 LZMA: 2 old: 44296 new: 7040 LZred: 84.11% +/Volumes/TB/project1/affs/0593.aff -> /Volumes/s.eecs.harvard.edu/p1/0593.aff Nochg: 0 NUL: 117 LZMA: 4 old: 136439 new: 13049 LZred: 90.44% +/Volumes/TB/project1/affs/0594.aff -> /Volumes/s.eecs.harvard.edu/p1/0594.aff Nochg: 0 NUL: 47 LZMA: 2 old: 69482 new: 5798 LZred: 91.66% +/Volumes/TB/project1/affs/0595.aff -> /Volumes/s.eecs.harvard.edu/p1/0595.aff Nochg: 0 NUL: 0 LZMA: 49 old: 412727570 new: 335635612 LZred: 18.68% +/Volumes/TB/project1/affs/0596.aff -> /Volumes/s.eecs.harvard.edu/p1/0596.aff Nochg: 0 NUL: 0 LZMA: 49 old: 72570007 new: 54377025 LZred: 25.07% +/Volumes/TB/project1/affs/0597.aff -> /Volumes/s.eecs.harvard.edu/p1/0597.aff Nochg: 0 NUL: 0 LZMA: 29 old: 240609159 new: 197401166 LZred: 17.96% +/Volumes/TB/project1/affs/0598.aff -> /Volumes/s.eecs.harvard.edu/p1/0598.aff Nochg: 0 NUL: 0 LZMA: 245 old: 895057125 new: 664755220 LZred: 25.73% +/Volumes/TB/project1/affs/0599.aff -> /Volumes/s.eecs.harvard.edu/p1/0599.aff Nochg: 1 NUL: 0 LZMA: 128 old: 986131277 new: 781697618 LZred: 20.73% +/Volumes/TB/project1/affs/0601.aff -> /Volumes/s.eecs.harvard.edu/p1/0601.aff Nochg: 0 NUL: 18 LZMA: 227 old: 90503084 new: 35783259 LZred: 60.46% +/Volumes/TB/project1/affs/0603.aff -> /Volumes/s.eecs.harvard.edu/p1/0603.aff Nochg: 26 NUL: 0 LZMA: 97 old: 1433822846 new: 1370926213 LZred: 4.39% +/Volumes/TB/project1/affs/0606.aff -> /Volumes/s.eecs.harvard.edu/p1/0606.aff Nochg: 0 NUL: 0 LZMA: 130 old: 720264975 new: 497393828 LZred: 30.94% +/Volumes/TB/project1/affs/0607.aff -> /Volumes/s.eecs.harvard.edu/p1/0607.aff Nochg: 0 NUL: 0 LZMA: 13 old: 56431760 new: 40917973 LZred: 27.49% +/Volumes/TB/project1/affs/0612.aff -> /Volumes/s.eecs.harvard.edu/p1/0612.aff Nochg: 0 NUL: 0 LZMA: 705 old: 7283524304 new: 6450996149 LZred: 11.43% +/Volumes/TB/project1/affs/0613.aff -> /Volumes/s.eecs.harvard.edu/p1/0613.aff Nochg: 0 NUL: 255 LZMA: 1 old: 22132 new: 2986 LZred: 86.51% +/Volumes/TB/project1/affs/0614.aff -> /Volumes/s.eecs.harvard.edu/p1/0614.aff Nochg: 0 NUL: 0 LZMA: 230 old: 3530235538 new: 3421216632 LZred: 3.09% +/Volumes/TB/project1/affs/0615.aff -> /Volumes/s.eecs.harvard.edu/p1/0615.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163993 new: 6020 LZred: 96.33% +/Volumes/TB/project1/affs/0616.aff -> /Volumes/s.eecs.harvard.edu/p1/0616.aff Nochg: 0 NUL: 0 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project1/affs/0617.aff -> /Volumes/s.eecs.harvard.edu/p1/0617.aff Nochg: 0 NUL: 0 LZMA: 32 old: 76299547 new: 34160967 LZred: 55.23% +/Volumes/TB/project1/affs/0618.aff -> /Volumes/s.eecs.harvard.edu/p1/0618.aff Nochg: 0 NUL: 0 LZMA: 11 old: 86118214 new: 69194339 LZred: 19.65% +/Volumes/TB/project1/affs/0619.aff -> /Volumes/s.eecs.harvard.edu/p1/0619.aff Nochg: 0 NUL: 0 LZMA: 8 old: 40273459 new: 30906975 LZred: 23.26% +/Volumes/TB/project1/affs/0620.aff -> /Volumes/s.eecs.harvard.edu/p1/0620.aff Nochg: 0 NUL: 0 LZMA: 11 old: 46641459 new: 29583178 LZred: 36.57% +/Volumes/TB/project1/affs/0621.aff -> /Volumes/s.eecs.harvard.edu/p1/0621.aff Nochg: 0 NUL: 0 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project1/affs/0622.aff -> /Volumes/s.eecs.harvard.edu/p1/0622.aff Nochg: 0 NUL: 0 LZMA: 95 old: 601900468 new: 450943453 LZred: 25.08% +/Volumes/TB/project1/affs/0623.aff -> /Volumes/s.eecs.harvard.edu/p1/0623.aff Nochg: 0 NUL: 0 LZMA: 229 old: 1987702830 new: 1678351826 LZred: 15.56% +/Volumes/TB/project1/affs/0624.aff -> /Volumes/s.eecs.harvard.edu/p1/0624.aff Nochg: 1 NUL: 0 LZMA: 188 old: 1956261581 new: 1715683372 LZred: 12.30% +/Volumes/TB/project1/affs/0625.aff -> /Volumes/s.eecs.harvard.edu/p1/0625.aff Nochg: 5 NUL: 0 LZMA: 185 old: 1442628343 new: 1185245936 LZred: 17.84% +/Volumes/TB/project1/affs/0626.aff -> /Volumes/s.eecs.harvard.edu/p1/0626.aff Nochg: 0 NUL: 192 LZMA: 1 old: 53029 new: 2986 LZred: 94.37% +/Volumes/TB/project1/affs/0627.aff -> /Volumes/s.eecs.harvard.edu/p1/0627.aff Nochg: 0 NUL: 0 LZMA: 191 old: 1667175833 new: 1460378987 LZred: 12.40% +/Volumes/TB/project1/affs/0628.aff -> /Volumes/s.eecs.harvard.edu/p1/0628.aff Nochg: 0 NUL: 0 LZMA: 102 old: 1680898 new: 250641 LZred: 85.09% +/Volumes/TB/project1/affs/0629.aff -> /Volumes/s.eecs.harvard.edu/p1/0629.aff Nochg: 0 NUL: 0 LZMA: 57 old: 997104 new: 141412 LZred: 85.82% +/Volumes/TB/project1/affs/0630.aff -> /Volumes/s.eecs.harvard.edu/p1/0630.aff Nochg: 0 NUL: 0 LZMA: 65 old: 1063473 new: 159916 LZred: 84.96% +/Volumes/TB/project1/affs/0631.aff -> /Volumes/s.eecs.harvard.edu/p1/0631.aff Nochg: 0 NUL: 25 LZMA: 72 old: 578126397 new: 482242103 LZred: 16.59% +/Volumes/TB/project1/affs/0632.aff -> /Volumes/s.eecs.harvard.edu/p1/0632.aff Nochg: 0 NUL: 0 LZMA: 126 old: 2064962 new: 309489 LZred: 85.01% +/Volumes/TB/project1/affs/0633.aff -> /Volumes/s.eecs.harvard.edu/p1/0633.aff Nochg: 0 NUL: 0 LZMA: 157 old: 2613693 new: 386613 LZred: 85.21% +/Volumes/TB/project1/affs/0634.aff -> /Volumes/s.eecs.harvard.edu/p1/0634.aff Nochg: 0 NUL: 0 LZMA: 157 old: 2598159 new: 385504 LZred: 85.16% +/Volumes/TB/project1/affs/0635.aff -> /Volumes/s.eecs.harvard.edu/p1/0635.aff Nochg: 0 NUL: 0 LZMA: 2 old: 156841 new: 6030 LZred: 96.16% +/Volumes/TB/project1/affs/0636.aff -> /Volumes/s.eecs.harvard.edu/p1/0636.aff Nochg: 0 NUL: 0 LZMA: 189 old: 3101152 new: 463963 LZred: 85.04% +/Volumes/TB/project1/affs/0637.aff -> /Volumes/s.eecs.harvard.edu/p1/0637.aff Nochg: 0 NUL: 0 LZMA: 185 old: 866198606 new: 623101846 LZred: 28.06% +/Volumes/TB/project1/affs/0638.aff -> /Volumes/s.eecs.harvard.edu/p1/0638.aff Nochg: 0 NUL: 0 LZMA: 186 old: 963419517 new: 699472653 LZred: 27.40% +/Volumes/TB/project1/affs/0639.aff -> /Volumes/s.eecs.harvard.edu/p1/0639.aff Nochg: 0 NUL: 4 LZMA: 188 old: 859326449 new: 622062889 LZred: 27.61% +/Volumes/TB/project1/affs/0640.aff -> /Volumes/s.eecs.harvard.edu/p1/0640.aff Nochg: 0 NUL: 0 LZMA: 191 old: 967036485 new: 660254645 LZred: 31.72% +/Volumes/TB/project1/affs/0641.aff -> /Volumes/s.eecs.harvard.edu/p1/0641.aff Nochg: 0 NUL: 0 LZMA: 192 old: 1598613390 new: 1247567636 LZred: 21.96% +/Volumes/TB/project1/affs/0643.aff -> /Volumes/s.eecs.harvard.edu/p1/0643.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163990 new: 6016 LZred: 96.33% +/Volumes/TB/project1/affs/0644.aff -> /Volumes/s.eecs.harvard.edu/p1/0644.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163992 new: 6014 LZred: 96.33% +/Volumes/TB/project1/affs/0645.aff -> /Volumes/s.eecs.harvard.edu/p1/0645.aff Nochg: 0 NUL: 125 LZMA: 1 old: 24958 new: 2984 LZred: 88.04% +/Volumes/TB/project1/affs/0646.aff -> /Volumes/s.eecs.harvard.edu/p1/0646.aff Nochg: 0 NUL: 125 LZMA: 1 old: 24956 new: 2984 LZred: 88.04% +/Volumes/TB/project1/affs/0647.aff -> /Volumes/s.eecs.harvard.edu/p1/0647.aff Nochg: 0 NUL: 125 LZMA: 1 old: 24959 new: 2985 LZred: 88.04% +/Volumes/TB/project1/affs/0648.aff -> /Volumes/s.eecs.harvard.edu/p1/0648.aff Nochg: 1 NUL: 96 LZMA: 88 old: 634950511 new: 503928048 LZred: 20.64% +/Volumes/TB/project1/affs/0649.aff -> /Volumes/s.eecs.harvard.edu/p1/0649.aff Nochg: 3 NUL: 0 LZMA: 109 old: 712196066 new: 528696233 LZred: 25.77% +/Volumes/TB/project1/affs/0650.aff -> /Volumes/s.eecs.harvard.edu/p1/0650.aff Nochg: 0 NUL: 90 LZMA: 98 old: 690723361 new: 531218097 LZred: 23.09% +/Volumes/TB/project1/affs/0651.aff -> /Volumes/s.eecs.harvard.edu/p1/0651.aff Nochg: 2 NUL: 0 LZMA: 207 old: 2809559614 new: 1004628618 LZred: 64.24% +/Volumes/TB/project1/affs/0652.aff -> /Volumes/s.eecs.harvard.edu/p1/0652.aff Nochg: 3 NUL: 0 LZMA: 125 old: 472742732 new: 383325027 LZred: 18.91% +/Volumes/TB/project1/affs/0653.aff -> /Volumes/s.eecs.harvard.edu/p1/0653.aff Nochg: 0 NUL: 54 LZMA: 138 old: 949293999 new: 710615378 LZred: 25.14% +/Volumes/TB/project1/affs/0654.aff -> /Volumes/s.eecs.harvard.edu/p1/0654.aff Nochg: 0 NUL: 255 LZMA: 1 old: 17547 new: 2990 LZred: 82.96% +/Volumes/TB/project1/affs/0655.aff -> /Volumes/s.eecs.harvard.edu/p1/0655.aff Nochg: 0 NUL: 0 LZMA: 381 old: 1915328220 new: 1398529233 LZred: 26.98% +/Volumes/TB/project1/affs/0656.aff -> /Volumes/s.eecs.harvard.edu/p1/0656.aff Nochg: 0 NUL: 0 LZMA: 385 old: 826660762 new: 535695838 LZred: 35.20% +/Volumes/TB/project1/affs/0657.aff -> /Volumes/s.eecs.harvard.edu/p1/0657.aff Nochg: 0 NUL: 0 LZMA: 21 old: 101790787 new: 71047534 LZred: 30.20% +/Volumes/TB/project1/affs/0658.aff -> /Volumes/s.eecs.harvard.edu/p1/0658.aff Nochg: 0 NUL: 0 LZMA: 164 old: 1287178315 new: 1006467929 LZred: 21.81% +/Volumes/TB/project1/affs/0659.aff -> /Volumes/s.eecs.harvard.edu/p1/0659.aff Nochg: 0 NUL: 0 LZMA: 345 old: 2187141172 new: 1539830999 LZred: 29.60% +/Volumes/TB/project1/affs/0660.aff -> /Volumes/s.eecs.harvard.edu/p1/0660.aff Nochg: 0 NUL: 0 LZMA: 380 old: 2778889041 new: 2055104307 LZred: 26.05% +/Volumes/TB/project1/affs/0661.aff -> /Volumes/s.eecs.harvard.edu/p1/0661.aff Nochg: 0 NUL: 0 LZMA: 380 old: 2900636307 new: 2021272775 LZred: 30.32% +/Volumes/TB/project1/affs/0662.aff -> /Volumes/s.eecs.harvard.edu/p1/0662.aff Nochg: 0 NUL: 0 LZMA: 384 old: 2765817203 new: 2018394682 LZred: 27.02% +/Volumes/TB/project1/affs/0663.aff -> /Volumes/s.eecs.harvard.edu/p1/0663.aff Nochg: 0 NUL: 0 LZMA: 380 old: 2663900024 new: 1978083688 LZred: 25.74% +/Volumes/TB/project1/affs/0664.aff -> /Volumes/s.eecs.harvard.edu/p1/0664.aff Nochg: 0 NUL: 0 LZMA: 383 old: 2391050020 new: 1816945978 LZred: 24.01% +/Volumes/TB/project1/affs/0665.aff -> /Volumes/s.eecs.harvard.edu/p1/0665.aff Nochg: 0 NUL: 0 LZMA: 329 old: 748733211 new: 515838018 LZred: 31.11% +/Volumes/TB/project1/affs/0667.aff -> /Volumes/s.eecs.harvard.edu/p1/0667.aff Nochg: 0 NUL: 0 LZMA: 385 old: 804843065 new: 551320021 LZred: 31.50% +/Volumes/TB/project1/affs/0668.aff -> /Volumes/s.eecs.harvard.edu/p1/0668.aff Nochg: 0 NUL: 4 LZMA: 381 old: 1909545930 new: 1272411248 LZred: 33.37% +/Volumes/TB/project1/affs/0669.aff -> /Volumes/s.eecs.harvard.edu/p1/0669.aff Nochg: 0 NUL: 0 LZMA: 352 old: 2000150042 new: 1523404717 LZred: 23.84% +/Volumes/TB/project1/affs/0670.aff -> /Volumes/s.eecs.harvard.edu/p1/0670.aff Nochg: 0 NUL: 0 LZMA: 383 old: 2762879492 new: 2057760509 LZred: 25.52% +/Volumes/TB/project1/affs/0671.aff -> /Volumes/s.eecs.harvard.edu/p1/0671.aff Nochg: 1 NUL: 0 LZMA: 382 old: 2546481860 new: 1885523970 LZred: 25.96% +/Volumes/TB/project1/affs/0672.aff -> /Volumes/s.eecs.harvard.edu/p1/0672.aff Nochg: 0 NUL: 0 LZMA: 366 old: 1980295387 new: 1526048287 LZred: 22.94% +/Volumes/TB/project1/affs/0673.aff -> /Volumes/s.eecs.harvard.edu/p1/0673.aff Nochg: 11 NUL: 0 LZMA: 532 old: 3138342621 new: 2430131930 LZred: 22.57% +/Volumes/TB/project1/affs/0674.aff -> /Volumes/s.eecs.harvard.edu/p1/0674.aff Nochg: 0 NUL: 0 LZMA: 381 old: 2147192378 new: 1674094507 LZred: 22.03% +/Volumes/TB/project1/affs/0676.aff -> /Volumes/s.eecs.harvard.edu/p1/0676.aff Nochg: 0 NUL: 0 LZMA: 383 old: 3090417037 new: 2436683680 LZred: 21.15% +/Volumes/TB/project1/affs/0677.aff -> /Volumes/s.eecs.harvard.edu/p1/0677.aff Nochg: 0 NUL: 0 LZMA: 189 old: 1332325465 new: 968157438 LZred: 27.33% +/Volumes/TB/project1/affs/0678.aff -> /Volumes/s.eecs.harvard.edu/p1/0678.aff Nochg: 0 NUL: 0 LZMA: 380 old: 2506686638 new: 1918633119 LZred: 23.46% +/Volumes/TB/project1/affs/0679.aff -> /Volumes/s.eecs.harvard.edu/p1/0679.aff Nochg: 0 NUL: 0 LZMA: 385 old: 1880094222 new: 1214640411 LZred: 35.39% +/Volumes/TB/project1/affs/0680.aff -> /Volumes/s.eecs.harvard.edu/p1/0680.aff Nochg: 0 NUL: 0 LZMA: 385 old: 2273217204 new: 1453109058 LZred: 36.08% +/Volumes/TB/project1/affs/0681.aff -> /Volumes/s.eecs.harvard.edu/p1/0681.aff Nochg: 1 NUL: 0 LZMA: 384 old: 6442450944 new: 1769247824 LZred: 72.54% +/Volumes/TB/project1/affs/0682.aff -> /Volumes/s.eecs.harvard.edu/p1/0682.aff Nochg: 0 NUL: 0 LZMA: 241 old: 4043309056 new: 987585396 LZred: 75.57% +/Volumes/TB/project1/affs/0683.aff -> /Volumes/s.eecs.harvard.edu/p1/0683.aff Nochg: 0 NUL: 8 LZMA: 341 old: 1807552341 new: 1421689036 LZred: 21.35% +/Volumes/TB/project1/affs/0684.aff -> /Volumes/s.eecs.harvard.edu/p1/0684.aff Nochg: 0 NUL: 17 LZMA: 4 old: 129611 new: 12123 LZred: 90.65% +/Volumes/TB/project1/affs/0685.aff -> /Volumes/s.eecs.harvard.edu/p1/0685.aff Nochg: 0 NUL: 283 LZMA: 27 old: 162259191 new: 113586821 LZred: 30.00% +/Volumes/TB/project1/affs/0686.aff -> /Volumes/s.eecs.harvard.edu/p1/0686.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163989 new: 6014 LZred: 96.33% +/Volumes/TB/project1/affs/0687.aff -> /Volumes/s.eecs.harvard.edu/p1/0687.aff Nochg: 0 NUL: 0 LZMA: 171 old: 1087105225 new: 789858317 LZred: 27.34% +/Volumes/TB/project1/affs/0689.aff -> /Volumes/s.eecs.harvard.edu/p1/0689.aff Nochg: 0 NUL: 4 LZMA: 92 old: 615559291 new: 463214759 LZred: 24.75% +/Volumes/TB/project1/affs/0690.aff -> /Volumes/s.eecs.harvard.edu/p1/0690.aff Nochg: 0 NUL: 96 LZMA: 289 old: 1822882779 new: 1247597786 LZred: 31.56% +/Volumes/TB/project1/affs/0691.aff -> /Volumes/s.eecs.harvard.edu/p1/0691.aff Nochg: 0 NUL: 0 LZMA: 383 old: 1460293226 new: 959038561 LZred: 34.33% +/Volumes/TB/project1/affs/0693.aff -> /Volumes/s.eecs.harvard.edu/p1/0693.aff Nochg: 0 NUL: 0 LZMA: 379 old: 2650755564 new: 1954184471 LZred: 26.28% +/Volumes/TB/project1/affs/0694.aff -> /Volumes/s.eecs.harvard.edu/p1/0694.aff Nochg: 0 NUL: 0 LZMA: 381 old: 1665252898 new: 1173529259 LZred: 29.53% +/Volumes/TB/project1/affs/0695.aff -> /Volumes/s.eecs.harvard.edu/p1/0695.aff Nochg: 0 NUL: 0 LZMA: 381 old: 1796568976 new: 1311072884 LZred: 27.02% +/Volumes/TB/project1/affs/0696.aff -> /Volumes/s.eecs.harvard.edu/p1/0696.aff Nochg: 0 NUL: 0 LZMA: 383 old: 2391120163 new: 1668958325 LZred: 30.20% +/Volumes/TB/project1/affs/0697.aff -> /Volumes/s.eecs.harvard.edu/p1/0697.aff Nochg: 0 NUL: 0 LZMA: 381 old: 2503320232 new: 1969903569 LZred: 21.31% +/Volumes/TB/project1/affs/0698.aff -> /Volumes/s.eecs.harvard.edu/p1/0698.aff Nochg: 0 NUL: 52 LZMA: 333 old: 1436224481 new: 1000025398 LZred: 30.37% +/Volumes/TB/project1/affs/0699.aff -> /Volumes/s.eecs.harvard.edu/p1/0699.aff Nochg: 0 NUL: 0 LZMA: 108 old: 748055225 new: 570165194 LZred: 23.78% +/Volumes/TB/project1/affs/0700.aff -> /Volumes/s.eecs.harvard.edu/p1/0700.aff Nochg: 0 NUL: 0 LZMA: 374 old: 2600172166 new: 1973044503 LZred: 24.12% +/Volumes/TB/project1/affs/0701.aff -> /Volumes/s.eecs.harvard.edu/p1/0701.aff Nochg: 0 NUL: 0 LZMA: 382 old: 1773127981 new: 1191939800 LZred: 32.78% +/Volumes/TB/project1/affs/0702.aff -> /Volumes/s.eecs.harvard.edu/p1/0702.aff Nochg: 0 NUL: 189 LZMA: 2 old: 1283796 new: 1133006 LZred: 11.75% +/Volumes/TB/project1/affs/0703.aff -> /Volumes/s.eecs.harvard.edu/p1/0703.aff Nochg: 0 NUL: 191 LZMA: 3 old: 2309563 new: 1844479 LZred: 20.14% +/Volumes/TB/project1/affs/0704.aff -> /Volumes/s.eecs.harvard.edu/p1/0704.aff Nochg: 0 NUL: 192 LZMA: 2 old: 1301726 new: 1126341 LZred: 13.47% +/Volumes/TB/project1/affs/0705.aff -> /Volumes/s.eecs.harvard.edu/p1/0705.aff Nochg: 0 NUL: 191 LZMA: 3 old: 2379052 new: 1936727 LZred: 18.59% +/Volumes/TB/project1/affs/0706.aff -> /Volumes/s.eecs.harvard.edu/p1/0706.aff Nochg: 0 NUL: 0 LZMA: 2 old: 164005 new: 6016 LZred: 96.33% +/Volumes/TB/project1/affs/0707.aff -> /Volumes/s.eecs.harvard.edu/p1/0707.aff Nochg: 0 NUL: 192 LZMA: 2 old: 1301705 new: 1126440 LZred: 13.46% +/Volumes/TB/project1/affs/0708.aff -> /Volumes/s.eecs.harvard.edu/p1/0708.aff Nochg: 0 NUL: 192 LZMA: 2 old: 1301709 new: 1126215 LZred: 13.48% +/Volumes/TB/project1/affs/0709.aff -> /Volumes/s.eecs.harvard.edu/p1/0709.aff Nochg: 0 NUL: 192 LZMA: 2 old: 1279681 new: 1126480 LZred: 11.97% +/Volumes/TB/project1/affs/0710.aff -> /Volumes/s.eecs.harvard.edu/p1/0710.aff Nochg: 0 NUL: 192 LZMA: 2 old: 1328642 new: 1141760 LZred: 14.07% +/Volumes/TB/project1/affs/0711.aff -> /Volumes/s.eecs.harvard.edu/p1/0711.aff Nochg: 0 NUL: 191 LZMA: 3 old: 1367371 new: 1135380 LZred: 16.97% +/Volumes/TB/project1/affs/0713.aff -> /Volumes/s.eecs.harvard.edu/p1/0713.aff Nochg: 9 NUL: 0 LZMA: 182 old: 1831917839 new: 1692152028 LZred: 7.63% +/Volumes/TB/project1/affs/0715.aff -> /Volumes/s.eecs.harvard.edu/p1/0715.aff Nochg: 0 NUL: 0 LZMA: 546 old: 5113752460 new: 3336145393 LZred: 34.76% +/Volumes/TB/project1/affs/0716.aff -> /Volumes/s.eecs.harvard.edu/p1/0716.aff Nochg: 3 NUL: 0 LZMA: 541 old: 3321023025 new: 2727866759 LZred: 17.86% +/Volumes/TB/project1/affs/0717.aff -> /Volumes/s.eecs.harvard.edu/p1/0717.aff Nochg: 0 NUL: 0 LZMA: 546 old: 3338862134 new: 2992857359 LZred: 10.36% +/Volumes/TB/project1/affs/0718.aff -> /Volumes/s.eecs.harvard.edu/p1/0718.aff Nochg: 3 NUL: 0 LZMA: 543 old: 3174606756 new: 2577692365 LZred: 18.80% +/Volumes/TB/project1/affs/0725.aff -> /Volumes/s.eecs.harvard.edu/p1/0725.aff Nochg: 0 NUL: 0 LZMA: 9 old: 60508426 new: 43873289 LZred: 27.49% +/Volumes/TB/project1/affs/0726.aff -> /Volumes/s.eecs.harvard.edu/p1/0726.aff Nochg: 0 NUL: 0 LZMA: 36 old: 323737867 new: 276730135 LZred: 14.52% +/Volumes/TB/project1/affs/0728.aff -> /Volumes/s.eecs.harvard.edu/p1/0728.aff Nochg: 0 NUL: 0 LZMA: 32 old: 198094829 new: 145752774 LZred: 26.42% +/Volumes/TB/project1/affs/0730.aff -> /Volumes/s.eecs.harvard.edu/p1/0730.aff Nochg: 0 NUL: 0 LZMA: 129 old: 2244446 new: 364869 LZred: 83.74% +/Volumes/TB/project1/affs/0731.aff -> /Volumes/s.eecs.harvard.edu/p1/0731.aff Nochg: 0 NUL: 0 LZMA: 129 old: 385626386 new: 282157396 LZred: 26.83% +/Volumes/TB/project1/affs/0732.aff -> /Volumes/s.eecs.harvard.edu/p1/0732.aff Nochg: 0 NUL: 0 LZMA: 129 old: 2244606 new: 364875 LZred: 83.74% +/Volumes/TB/project1/affs/0733.aff -> /Volumes/s.eecs.harvard.edu/p1/0733.aff Nochg: 0 NUL: 0 LZMA: 129 old: 2244407 new: 364805 LZred: 83.75% +/Volumes/TB/project1/affs/0734.aff -> /Volumes/s.eecs.harvard.edu/p1/0734.aff Nochg: 0 NUL: 0 LZMA: 129 old: 2244289 new: 364794 LZred: 83.75% +/Volumes/TB/project1/affs/0736.aff -> /Volumes/s.eecs.harvard.edu/p1/0736.aff Nochg: 0 NUL: 0 LZMA: 129 old: 385943419 new: 282265589 LZred: 26.86% +/Volumes/TB/project1/affs/0737.aff -> /Volumes/s.eecs.harvard.edu/p1/0737.aff Nochg: 0 NUL: 0 LZMA: 129 old: 386050661 new: 282227696 LZred: 26.89% +/Volumes/TB/project1/affs/0738.aff -> /Volumes/s.eecs.harvard.edu/p1/0738.aff Nochg: 0 NUL: 0 LZMA: 129 old: 385197515 new: 281840145 LZred: 26.83% +/Volumes/TB/project1/affs/0739.aff -> /Volumes/s.eecs.harvard.edu/p1/0739.aff Nochg: 1 NUL: 0 LZMA: 142 old: 1983787071 new: 1924118685 LZred: 3.01% +/Volumes/TB/project1/affs/0740.aff -> /Volumes/s.eecs.harvard.edu/p1/0740.aff Nochg: 0 NUL: 0 LZMA: 123 old: 1224533840 new: 1062199119 LZred: 13.26% +/Volumes/TB/project1/affs/0741.aff -> /Volumes/s.eecs.harvard.edu/p1/0741.aff Nochg: 0 NUL: 0 LZMA: 151 old: 1098862463 new: 796635858 LZred: 27.50% +/Volumes/TB/project1/affs/0742.aff -> /Volumes/s.eecs.harvard.edu/p1/0742.aff Nochg: 0 NUL: 13 LZMA: 135 old: 1680203087 new: 1587440680 LZred: 5.52% +/Volumes/TB/project1/affs/0743.aff -> /Volumes/s.eecs.harvard.edu/p1/0743.aff Nochg: 0 NUL: 4 LZMA: 147 old: 1171411133 new: 970188363 LZred: 17.18% +/Volumes/TB/project1/affs/0744.aff -> /Volumes/s.eecs.harvard.edu/p1/0744.aff Nochg: 0 NUL: 86 LZMA: 299 old: 2592657168 new: 1943585202 LZred: 25.04% +/Volumes/TB/project1/affs/0745.aff -> /Volumes/s.eecs.harvard.edu/p1/0745.aff Nochg: 0 NUL: 384 LZMA: 1 old: 54327 new: 2985 LZred: 94.51% +/Volumes/TB/project1/affs/0746.aff -> /Volumes/s.eecs.harvard.edu/p1/0746.aff Nochg: 0 NUL: 384 LZMA: 1 old: 58077 new: 2985 LZred: 94.86% +/Volumes/TB/project1/affs/0747.aff -> /Volumes/s.eecs.harvard.edu/p1/0747.aff Nochg: 0 NUL: 384 LZMA: 1 old: 62271 new: 2986 LZred: 95.20% +/Volumes/TB/project1/affs/0748.aff -> /Volumes/s.eecs.harvard.edu/p1/0748.aff Nochg: 0 NUL: 384 LZMA: 1 old: 58078 new: 2986 LZred: 94.86% +/Volumes/TB/project1/affs/0749.aff -> /Volumes/s.eecs.harvard.edu/p1/0749.aff Nochg: 0 NUL: 499 LZMA: 5 old: 1449262 new: 1138697 LZred: 21.43% +/Volumes/TB/project1/affs/0750.aff -> /Volumes/s.eecs.harvard.edu/p1/0750.aff Nochg: 0 NUL: 250 LZMA: 2 old: 1273803 new: 1126504 LZred: 11.56% +/Volumes/TB/project1/affs/0751.aff -> /Volumes/s.eecs.harvard.edu/p1/0751.aff Nochg: 0 NUL: 250 LZMA: 2 old: 1273800 new: 1126390 LZred: 11.57% +/Volumes/TB/project1/affs/0752.aff -> /Volumes/s.eecs.harvard.edu/p1/0752.aff Nochg: 0 NUL: 499 LZMA: 5 old: 16919227 new: 12898135 LZred: 23.77% +/Volumes/TB/project1/affs/0753.aff -> /Volumes/s.eecs.harvard.edu/p1/0753.aff Nochg: 0 NUL: 499 LZMA: 5 old: 6146516 new: 5325513 LZred: 13.36% +/Volumes/TB/project1/affs/0754.aff -> /Volumes/s.eecs.harvard.edu/p1/0754.aff Nochg: 0 NUL: 255 LZMA: 2 old: 1263130 new: 1126440 LZred: 10.82% +/Volumes/TB/project1/affs/0755.aff -> /Volumes/s.eecs.harvard.edu/p1/0755.aff Nochg: 0 NUL: 250 LZMA: 2 old: 2643701 new: 2175484 LZred: 17.71% +/Volumes/TB/project1/affs/0756.aff -> /Volumes/s.eecs.harvard.edu/p1/0756.aff Nochg: 0 NUL: 255 LZMA: 2 old: 5914486 new: 4829982 LZred: 18.34% +/Volumes/TB/project1/affs/0757.aff -> /Volumes/s.eecs.harvard.edu/p1/0757.aff Nochg: 0 NUL: 255 LZMA: 2 old: 1263133 new: 1126555 LZred: 10.81% +/Volumes/TB/project1/affs/0758.aff -> /Volumes/s.eecs.harvard.edu/p1/0758.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163989 new: 6014 LZred: 96.33% +/Volumes/TB/project1/affs/0759.aff -> /Volumes/s.eecs.harvard.edu/p1/0759.aff Nochg: 0 NUL: 512 LZMA: 2 old: 4909078 new: 4237151 LZred: 13.69% +/Volumes/TB/project1/affs/0760.aff -> /Volumes/s.eecs.harvard.edu/p1/0760.aff Nochg: 0 NUL: 206 LZMA: 3 old: 1375894 new: 1129807 LZred: 17.89% +/Volumes/TB/project1/affs/0761.aff -> /Volumes/s.eecs.harvard.edu/p1/0761.aff Nochg: 0 NUL: 255 LZMA: 2 old: 1263129 new: 1126546 LZred: 10.81% +/Volumes/TB/project1/affs/0762.aff -> /Volumes/s.eecs.harvard.edu/p1/0762.aff Nochg: 0 NUL: 255 LZMA: 3 old: 1468972 new: 1194332 LZred: 18.70% +/Volumes/TB/project1/affs/0763.aff -> /Volumes/s.eecs.harvard.edu/p1/0763.aff Nochg: 0 NUL: 255 LZMA: 3 old: 5980431 new: 5019195 LZred: 16.07% +/Volumes/TB/project1/affs/0764.aff -> /Volumes/s.eecs.harvard.edu/p1/0764.aff Nochg: 0 NUL: 0 LZMA: 77 old: 616381812 new: 506734704 LZred: 17.79% +/Volumes/TB/project1/affs/0765.aff -> /Volumes/s.eecs.harvard.edu/p1/0765.aff Nochg: 0 NUL: 0 LZMA: 50 old: 301975228 new: 220004471 LZred: 27.14% +/Volumes/TB/project1/affs/0766.aff -> /Volumes/s.eecs.harvard.edu/p1/0766.aff Nochg: 0 NUL: 0 LZMA: 58 old: 468386669 new: 372199021 LZred: 20.54% +/Volumes/TB/project1/affs/0767.aff -> /Volumes/s.eecs.harvard.edu/p1/0767.aff Nochg: 0 NUL: 149 LZMA: 104 old: 886516430 new: 758192900 LZred: 14.48% +/Volumes/TB/project1/affs/0768.aff -> /Volumes/s.eecs.harvard.edu/p1/0768.aff Nochg: 0 NUL: 0 LZMA: 122 old: 732869058 new: 524105467 LZred: 28.49% +/Volumes/TB/project1/affs/0770.aff -> /Volumes/s.eecs.harvard.edu/p1/0770.aff Nochg: 0 NUL: 0 LZMA: 433 old: 5172844510 new: 4688381005 LZred: 9.37% +/Volumes/TB/project1/affs/0771.aff -> /Volumes/s.eecs.harvard.edu/p1/0771.aff Nochg: 8 NUL: 0 LZMA: 239 old: 2174945519 new: 1825153826 LZred: 16.08% +/Volumes/TB/project1/affs/0773.aff -> /Volumes/s.eecs.harvard.edu/p1/0773.aff Nochg: 0 NUL: 0 LZMA: 119 old: 733325791 new: 603560861 LZred: 17.70% +/Volumes/TB/project1/affs/0775.aff -> /Volumes/s.eecs.harvard.edu/p1/0775.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163992 new: 6012 LZred: 96.33% +/Volumes/TB/project1/affs/0776.aff -> /Volumes/s.eecs.harvard.edu/p1/0776.aff Nochg: 0 NUL: 0 LZMA: 120 old: 448864610 new: 337901975 LZred: 24.72% +/Volumes/TB/project1/affs/0777.aff -> /Volumes/s.eecs.harvard.edu/p1/0777.aff Nochg: 0 NUL: 0 LZMA: 7 old: 78368060 new: 72244527 LZred: 7.81% +/Volumes/TB/project1/affs/0778.aff -> /Volumes/s.eecs.harvard.edu/p1/0778.aff Nochg: 0 NUL: 0 LZMA: 3 old: 245846 new: 8985 LZred: 96.35% +/Volumes/TB/project1/affs/0779.aff -> /Volumes/s.eecs.harvard.edu/p1/0779.aff Nochg: 2 NUL: 32 LZMA: 236 old: 1438175805 new: 925822304 LZred: 35.63% +/Volumes/TB/project1/affs/0780.aff -> /Volumes/s.eecs.harvard.edu/p1/0780.aff Nochg: 0 NUL: 0 LZMA: 2443 old: 6050314630 new: 4128855998 LZred: 31.76% +/Volumes/TB/project1/affs/0782.aff -> /Volumes/s.eecs.harvard.edu/p1/0782.aff Nochg: 0 NUL: 0 LZMA: 21 old: 151254046 new: 105606317 LZred: 30.18% +/Volumes/TB/project1/affs/0783.aff -> /Volumes/s.eecs.harvard.edu/p1/0783.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163996 new: 6015 LZred: 96.33% +/Volumes/TB/project1/affs/0784.aff -> /Volumes/s.eecs.harvard.edu/p1/0784.aff Nochg: 0 NUL: 47 LZMA: 3 old: 123811 new: 9261 LZred: 92.52% +/Volumes/TB/project1/affs/0790.aff -> /Volumes/s.eecs.harvard.edu/p1/0790.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163997 new: 6017 LZred: 96.33% +/Volumes/TB/project1/affs/0791.aff -> /Volumes/s.eecs.harvard.edu/p1/0791.aff Nochg: 0 NUL: 0 LZMA: 19 old: 113065149 new: 84222806 LZred: 25.51% +/Volumes/TB/project1/affs/0792.aff -> /Volumes/s.eecs.harvard.edu/p1/0792.aff Nochg: 0 NUL: 0 LZMA: 2 old: 166304 new: 7672 LZred: 95.39% +/Volumes/TB/project1/affs/0793.aff -> /Volumes/s.eecs.harvard.edu/p1/0793.aff Nochg: 0 NUL: 0 LZMA: 44 old: 240848844 new: 174719198 LZred: 27.46% +/Volumes/TB/project1/affs/0794.aff -> /Volumes/s.eecs.harvard.edu/p1/0794.aff Nochg: 0 NUL: 0 LZMA: 77 old: 714932094 new: 514614577 LZred: 28.02% +/Volumes/TB/project1/affs/0797.aff -> /Volumes/s.eecs.harvard.edu/p1/0797.aff Nochg: 0 NUL: 0 LZMA: 2 old: 241993 new: 25512 LZred: 89.46% +/Volumes/TB/project1/affs/0799.aff -> /Volumes/s.eecs.harvard.edu/p1/0799.aff Nochg: 6 NUL: 0 LZMA: 636 old: 2890048634 new: 2372175811 LZred: 17.92% diff --git a/stats/project2-report.txt b/stats/project2-report.txt new file mode 100644 index 0000000..0d6d90e --- /dev/null +++ b/stats/project2-report.txt @@ -0,0 +1,178 @@ +Name AF_IMAGESIZE Compressed Uncompressed Blank Bad +/access/p2/0802.aff 40060403712 36158 33554432 85 6144 +/access/p2/0805.aff 1750003712 836841205 1750003712 124109 6656 +/access/p2/0806.aff 20485398528 6869695412 20485398528 2324734 0 +/access/p2/0807.aff 540868608 6016 33554432 11 6144 +/access/p2/0808.aff 4304240640 2004450911 4304240640 197482 0 +/access/p2/0809.aff 4099866624 6017 33554432 11 6144 +/access/p2/0810.aff 4099866624 6017 33554432 11 6144 +/access/p2/0811.aff 4099866624 6015 33554432 11 6144 +/access/p2/0812.aff 4099866624 6019 33554432 11 6144 +/access/p2/0813.aff 4099866624 1338941796 4099866624 138552 0 +/access/p2/0814.aff 4099866624 2373516151 4099866624 168083 1024 +/access/p2/0815.aff 4099866624 6017 33554432 11 6144 +/access/p2/0816.aff 4099866624 6020 33554432 11 6144 +/access/p2/0817.aff 4099866624 6019 33554432 11 6144 +/access/p2/0818.aff 4099866624 6015 33554432 11 6144 +/access/p2/0819.aff 4099866624 2667891416 4099866624 167200 0 +/access/p2/0820.aff 4099866624 2214342827 4099866624 143004 0 +/access/p2/0821.aff 4099866624 6017 33554432 11 6144 +/access/p2/0822.aff 4099866624 6015 33554432 11 6144 +/access/p2/0823.aff 4099866624 1781827613 4099866624 173933 0 +/access/p2/0824.aff 3227148288 55855393 234881024 73489 6144 +/access/p2/0825.aff 1707761664 372198356 973078528 76745 6144 +/access/p2/0826.aff 730791936 6017 33554432 11 6144 +/access/p2/0827.aff 1624670208 1600818031 1624670208 330 0 +/access/p2/0828.aff 2111864832 1977291285 2111864832 325 0 +/access/p2/0829.aff 2167603200 442399770 1509949440 122937 6144 +/access/p2/0830.aff 2167603200 158774 922746880 1605503 6144 +/access/p2/0831.aff 2167603200 333547 1946157056 3600903 6144 +/access/p2/0832.aff 0 0 0 0 0 +/access/p2/0833.aff 2559836160 373741 2550136832 0 8704 +/access/p2/0834.aff 163928604672 1895610169 3238002688 146304 6144 +/access/p2/0835.aff 4311982080 900356448 4311982080 132632 0 +/access/p2/0836.aff 4311982080 1328383634 4311982080 148725 0 +/access/p2/0837.aff 4311982080 1186803163 4311982080 125039 0 +/access/p2/0838.aff 163928604672 3574222327 21256732672 558497 6144 +/access/p2/0839.aff 163928604672 6012 33554432 11 6144 +/access/p2/0840.aff 4311982080 1329480287 4311982080 141476 0 +/access/p2/0842.aff 4311982080 978711680 4311982080 137659 0 +/access/p2/0843.aff 163928604672 1224167111 4345298944 140463 6144 +/access/p2/0844.aff 30020272128 5102264634 30020272128 700114 0 +/access/p2/0845.aff 0 1604397288 0 0 0 +/access/p2/0846.aff 40020664320 12524 40020664320 78165359 0 +/access/p2/0847.aff 6448619520 4022653972 6448619520 261606 0 +/access/p2/0849.aff 40020664320 10881699121 40020664320 5355641 0 +/access/p2/0850.aff 40020664320 114038101 721420288 7485 6144 +/access/p2/0851.aff 40020664320 468482 40020664320 72154844 0 +/access/p2/0852.aff 0 0 0 0 0 +/access/p2/0853.aff 40020664320 11459104 3388997632 10968 7168 +/access/p2/0854.aff 0 2342318880 0 0 0 +/access/p2/0855.aff 40020664320 2296613780 4462739456 203248 6144 +/access/p2/0856.aff 80026361856 10734284864 26608664576 9784917 8704 +/access/p2/0857.aff 0 0 0 0 0 +/access/p2/0858.aff 0 3000 0 0 0 +/access/p2/0859.aff 6448619520 236884 33554432 76 9728 +/access/p2/0862.aff 4311982080 900666405 4311982080 127515 0 +/access/p2/0863.aff 40020664320 12525 40020664320 78165359 0 +/access/p2/0864.aff 4343641088 6016 33554432 11 6144 +/access/p2/0865.aff 6448619520 3969639723 6448619520 427426 0 +/access/p2/0866.aff 2192891392 104440887 251658240 21265 11264 +/access/p2/0867.aff 2200100864 6015 33554432 11 6144 +/access/p2/0868.aff 2192891392 104440887 251658240 21265 11264 +/access/p2/0879.aff 2143129560 6013 33554432 0 6144 +/access/p2/0881.aff 2143129560 6011 33554432 0 6144 +/access/p2/0882.aff 4327464960 3947040 33554432 120 6144 +/access/p2/0883.aff 3249340416 6171 671088640 1245195 6144 +/access/p2/0884.aff 3249340416 3757 3249340416 6346367 0 +/access/p2/0885.aff 3209084928 3749 3209084928 6267743 0 +/access/p2/0886.aff 3249340416 3758 3249340416 6346367 0 +/access/p2/0887.aff 3249340416 3758 3249340416 6346367 0 +/access/p2/0888.aff 3228696576 3754 3228696576 6306047 0 +/access/p2/0889.aff 3249340416 3756 3249340416 6346367 0 +/access/p2/0890.aff 3240566784 3757 3240566784 6329231 0 +/access/p2/0891.aff 3209084928 3749 3209084928 6267743 0 +/access/p2/0892.aff 3249340416 3759 3249340416 6346367 0 +/access/p2/0893.aff 3249340416 3756 3249340416 6346367 0 +/access/p2/0894.aff 3209084928 3750 3209084928 6267743 0 +/access/p2/0895.aff 3249340416 3757 3249340416 6346367 0 +/access/p2/0896.aff 3249340416 3758 3249340416 6346367 0 +/access/p2/0898.aff 3240050688 3757 3240050688 6328223 0 +/access/p2/0899.aff 3249340416 3757 3249340416 6346367 0 +/access/p2/0900.aff 3240646656 3757 3240646656 6329387 0 +/access/p2/0901.aff 3240050688 3758 3240050688 6328223 0 +/access/p2/0902.aff 3240566784 3757 3240566784 6329231 0 +/access/p2/0903.aff 3209084928 3749 3209084928 6267743 0 +/access/p2/0904.aff 421969408 117577248 419430400 49917 6656 +/access/p2/0905.aff 127983104 48680159 127983104 10008 6656 +/access/p2/0907.aff 127983104 50102880 127983104 7132 0 +/access/p2/0908.aff 212674560 6016 33554432 11 6144 +/access/p2/0909.aff 541384704 200071423 541384704 35032 6656 +/access/p2/0911.aff 853622784 6021 33554432 11 6144 +/access/p2/0913.aff 1705908224 221456739 419430400 53814 6144 +/access/p2/0914.aff 1279503360 6013 33554432 11 6144 +/access/p2/0915.aff 1281982464 1014569473 1281982464 16333 0 +/access/p2/0916.aff 1704665088 1300297419 1704665088 70397 0 +/access/p2/0917.aff 541384704 9352720 67108864 20200 6656 +/access/p2/0918.aff 1630295040 23854944 83886080 8224 6144 +/access/p2/0921.aff 0 0 0 0 0 +/access/p2/0922.aff 9104953344 5670 9104953344 17783111 0 +/access/p2/0924.aff 4321787904 2184651898 3556769792 0 0 +/access/p2/0925.aff 545685504 18386388 486539264 451170 9728 +/access/p2/0926.aff 548093952 85680339 548093952 487792 1536 +/access/p2/0927.aff 540170240 25756739 540170240 63852 6144 +/access/p2/0928.aff 853622784 6015 33554432 11 6144 +/access/p2/0929.aff 4311982080 64112402 2399141888 0 5856 +/access/p2/0930.aff 545685504 80594024 545685504 347851 0 +/access/p2/0931.aff 184287232 65937196 184287232 0 0 +/access/p2/0932.aff 4294817280 627225 4294817280 0 0 +/access/p2/0933.aff 0 0 0 0 0 +/access/p2/0935.aff 0 0 0 0 0 +/access/p2/0936.aff 0 0 0 0 0 +/access/p2/0937.aff 6448619520 6558739 6448619520 12576038 0 +/access/p2/0938.aff 6448619520 1128309 6448619520 12591907 0 +/access/p2/0939.aff 6448619520 1152294 6448619520 12575843 0 +/access/p2/0940.aff 6448619520 6015 33554432 11 6144 +/access/p2/0941.aff 3249340416 109539 67108864 0 6144 +/access/p2/0943.aff 4303272960 1108014876 4303272960 1103483 0 +/access/p2/0944.aff 4303272960 6879 3623878656 7034380 6656 +/access/p2/0946.aff 4303272960 1586288087 4303272960 557197 0 +/access/p2/0947.aff 4303272960 1543439738 4303272960 347132 0 +/access/p2/0948.aff 3249340416 1549101 738197504 1 6144 +/access/p2/0949.aff 4303272960 6188 4303272960 8401995 0 +/access/p2/0950.aff 3249340416 1156094492 3249340416 264010 0 +/access/p2/0951.aff 3249340416 219413848 3249340416 39082 0 +/access/p2/0952.aff 544997376 111411047 486539264 46235 6144 +/access/p2/0953.aff 541384704 16047012 520093696 727295 6144 +/access/p2/0954.aff 127983104 7050806 83886080 2194 6144 +/access/p2/0956.aff 212674560 6020 33554432 11 6144 +/access/p2/0957.aff 212674560 6017 33554432 11 6144 +/access/p2/0959.aff 127983104 7602620 100663296 2740 6144 +/access/p2/0960.aff 2132619264 74425 1811939328 3469924 6144 +/access/p2/0962.aff 212674560 6018 33554432 11 6144 +/access/p2/0967.aff 270434304 6019 33554432 11 6144 +/access/p2/0968.aff 170139648 692015 33554432 889 6144 +/access/p2/0969.aff 127983104 9128206 127983104 2522 6656 +/access/p2/0970.aff 365395968 6012 33554432 11 6144 +/access/p2/0972.aff 4303272960 714336856 4303272960 156210 8704 +/access/p2/0973.aff 4310433792 1712019765 4310433792 127816 7168 +/access/p2/0974.aff 4311982080 1174926240 4311982080 393485 0 +/access/p2/0975.aff 4299973632 1128464275 4299973632 2238654 0 +/access/p2/0977.aff 4310433792 1667346926 4310433792 961491 0 +/access/p2/0978.aff 4303272960 861353898 4303272960 381071 0 +/access/p2/0979.aff 4303272960 858266283 4303272960 317922 0 +/access/p2/0980.aff 4310433792 1927789753 4310433792 337189 0 +/access/p2/0981.aff 4223729664 566125632 3472883712 3812044 6144 +/access/p2/0982.aff 170649600 6013 33554432 11 6144 +/access/p2/0985.aff 365395968 6011 33554432 11 6144 +/access/p2/0986.aff 540868608 6014 33554432 11 6144 +/access/p2/0991.aff 760209214464 9819064 251658240 101693 6144 +/access/p2/0992.aff 170649600 6011 33554432 11 6144 +/access/p2/0993.aff 3249340416 3756 3249340416 6346367 0 +/access/p2/0995.aff 212674560 6019 33554432 11 6144 +/access/p2/0996.aff 341299200 6012 33554432 11 6144 +/access/p2/0997.aff 1549409458176 131498725 436207616 21457 6144 +/access/p2/0998.aff 212674560 6017 33554432 11 6144 +/access/p2/1000.aff 341299200 6010 33554432 11 6144 +/access/p2/1001.aff 760209214464 9359385 251658240 101326 6144 +/access/p2/1004.aff 0 0 0 0 0 +/access/p2/1006.aff 0 0 0 0 0 +/access/p2/1007.aff 20491075584 14590376098 20491075584 0 0 +/access/p2/1008.aff 20409532416 10597320845 20409532416 0 0 +/access/p2/1009.aff 22605004800 12174164693 22605004800 0 0 +/access/p2/1010.aff 14200977408 6981394524 14200977408 0 0 +/access/p2/1011.aff 15377080320 8516276880 15377080320 0 0 +/access/p2/1012.aff 37509857280 30029751690 37509857280 0 0 +/access/p2/1013.aff 40037760000 20816877569 40037760000 0 0 +/access/p2/1014.aff 0 1182856441 0 0 0 +/access/p2/1015.aff 15377080320 10281073473 15377080320 0 0 +/access/p2/1016.aff 20020396032 14552592635 20020396032 0 0 +/access/p2/1017.aff 10262568960 1993361940 10262568960 0 0 +/access/p2/1018.aff 30000000000 22556624591 30000000000 0 0 +/access/p2/1019.aff 40020664320 30088385672 40020664320 0 0 +/access/p2/1020.aff 10141286400 4577456002 10141286400 0 0 +/access/p2/1021.aff 30738677760 30343451492 30738677760 0 0 +/access/p2/1022.aff 10141286400 4577455023 10141286400 0 0 +/access/p2/1023.aff 13578485760 935430451 13578485760 0 0 +/access/p2/1024.aff 10005037056 2047232801 10005037056 0 0 +/access/p2/1025.aff 13578485760 10769081414 13578485760 0 0 diff --git a/stats/project2.afcompare.preen.txt b/stats/project2.afcompare.preen.txt new file mode 100644 index 0000000..3f7cc47 --- /dev/null +++ b/stats/project2.afcompare.preen.txt @@ -0,0 +1,177 @@ +/Volumes/TB/project2/affs/0802.aff -> /Volumes/s.eecs.harvard.edu/p2/0802.aff Nochg: 0 NUL: 0 LZMA: 2 old: 240742 new: 36158 LZred: 84.98% +/Volumes/TB/project2/affs/0805.aff -> /Volumes/s.eecs.harvard.edu/p2/0805.aff Nochg: 0 NUL: 0 LZMA: 103 old: 975550557 new: 803286773 LZred: 17.66% +/Volumes/TB/project2/affs/0806.aff -> /Volumes/s.eecs.harvard.edu/p2/0806.aff Nochg: 1 NUL: 0 LZMA: 1219 old: 9471852613 new: 6819363764 LZred: 28.00% +/Volumes/TB/project2/affs/0807.aff -> /Volumes/s.eecs.harvard.edu/p2/0807.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163998 new: 6016 LZred: 96.33% +/Volumes/TB/project2/affs/0808.aff -> /Volumes/s.eecs.harvard.edu/p2/0808.aff Nochg: 0 NUL: 0 LZMA: 250 old: 2312996631 new: 1887010399 LZred: 18.42% +/Volumes/TB/project2/affs/0809.aff -> /Volumes/s.eecs.harvard.edu/p2/0809.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163996 new: 6017 LZred: 96.33% +/Volumes/TB/project2/affs/0810.aff -> /Volumes/s.eecs.harvard.edu/p2/0810.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163992 new: 6017 LZred: 96.33% +/Volumes/TB/project2/affs/0811.aff -> /Volumes/s.eecs.harvard.edu/p2/0811.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163996 new: 6015 LZred: 96.33% +/Volumes/TB/project2/affs/0812.aff -> /Volumes/s.eecs.harvard.edu/p2/0812.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163998 new: 6019 LZred: 96.33% +/Volumes/TB/project2/affs/0813.aff -> /Volumes/s.eecs.harvard.edu/p2/0813.aff Nochg: 0 NUL: 0 LZMA: 238 old: 1541818497 new: 1221501284 LZred: 20.78% +/Volumes/TB/project2/affs/0814.aff -> /Volumes/s.eecs.harvard.edu/p2/0814.aff Nochg: 20 NUL: 0 LZMA: 219 old: 2293546997 new: 1937308535 LZred: 15.53% +/Volumes/TB/project2/affs/0815.aff -> /Volumes/s.eecs.harvard.edu/p2/0815.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163996 new: 6017 LZred: 96.33% +/Volumes/TB/project2/affs/0816.aff -> /Volumes/s.eecs.harvard.edu/p2/0816.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163993 new: 6020 LZred: 96.33% +/Volumes/TB/project2/affs/0817.aff -> /Volumes/s.eecs.harvard.edu/p2/0817.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163998 new: 6019 LZred: 96.33% +/Volumes/TB/project2/affs/0818.aff -> /Volumes/s.eecs.harvard.edu/p2/0818.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163991 new: 6015 LZred: 96.33% +/Volumes/TB/project2/affs/0819.aff -> /Volumes/s.eecs.harvard.edu/p2/0819.aff Nochg: 0 NUL: 0 LZMA: 242 old: 2876885968 new: 2617559768 LZred: 9.01% +/Volumes/TB/project2/affs/0820.aff -> /Volumes/s.eecs.harvard.edu/p2/0820.aff Nochg: 9 NUL: 0 LZMA: 227 old: 2228167097 new: 1912352939 LZred: 14.17% +/Volumes/TB/project2/affs/0821.aff -> /Volumes/s.eecs.harvard.edu/p2/0821.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163990 new: 6017 LZred: 96.33% +/Volumes/TB/project2/affs/0822.aff -> /Volumes/s.eecs.harvard.edu/p2/0822.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163995 new: 6015 LZred: 96.33% +/Volumes/TB/project2/affs/0823.aff -> /Volumes/s.eecs.harvard.edu/p2/0823.aff Nochg: 1 NUL: 0 LZMA: 235 old: 2092184539 new: 1614055453 LZred: 22.85% +/Volumes/TB/project2/affs/0824.aff -> /Volumes/s.eecs.harvard.edu/p2/0824.aff Nochg: 0 NUL: 0 LZMA: 14 old: 69143002 new: 55855393 LZred: 19.22% +/Volumes/TB/project2/affs/0825.aff -> /Volumes/s.eecs.harvard.edu/p2/0825.aff Nochg: 0 NUL: 0 LZMA: 58 old: 468386665 new: 372198356 LZred: 20.54% +/Volumes/TB/project2/affs/0826.aff -> /Volumes/s.eecs.harvard.edu/p2/0826.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163993 new: 6017 LZred: 96.33% +/Volumes/TB/project2/affs/0827.aff -> /Volumes/s.eecs.harvard.edu/p2/0827.aff Nochg: 85 NUL: 0 LZMA: 11 old: 171843581 new: 157977455 LZred: 8.07% +/Volumes/TB/project2/affs/0828.aff -> /Volumes/s.eecs.harvard.edu/p2/0828.aff Nochg: 76 NUL: 0 LZMA: 48 old: 789708075 new: 668668437 LZred: 15.33% +/Volumes/TB/project2/affs/0829.aff -> /Volumes/s.eecs.harvard.edu/p2/0829.aff Nochg: 0 NUL: 0 LZMA: 90 old: 585544857 new: 442399770 LZred: 24.45% +/Volumes/TB/project2/affs/0830.aff -> /Volumes/s.eecs.harvard.edu/p2/0830.aff Nochg: 0 NUL: 0 LZMA: 55 old: 1051890 new: 158774 LZred: 84.91% +/Volumes/TB/project2/affs/0831.aff -> /Volumes/s.eecs.harvard.edu/p2/0831.aff Nochg: 0 NUL: 0 LZMA: 116 old: 2073978 new: 333547 LZred: 83.92% +/Volumes/TB/project2/affs/0832.aff -> /Volumes/s.eecs.harvard.edu/p2/0832.aff Nochg: 0 NUL: 0 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project2/affs/0833.aff -> /Volumes/s.eecs.harvard.edu/p2/0833.aff Nochg: 0 NUL: 0 LZMA: 152 old: 2565754 new: 373741 LZred: 85.43% +/Volumes/TB/project2/affs/0834.aff -> /Volumes/s.eecs.harvard.edu/p2/0834.aff Nochg: 4 NUL: 0 LZMA: 186 old: 1991510322 new: 1778169657 LZred: 10.71% +/Volumes/TB/project2/affs/0835.aff -> /Volumes/s.eecs.harvard.edu/p2/0835.aff Nochg: 0 NUL: 0 LZMA: 254 old: 1074903704 new: 833247584 LZred: 22.48% +/Volumes/TB/project2/affs/0836.aff -> /Volumes/s.eecs.harvard.edu/p2/0836.aff Nochg: 0 NUL: 0 LZMA: 254 old: 1549583551 new: 1261274770 LZred: 18.61% +/Volumes/TB/project2/affs/0837.aff -> /Volumes/s.eecs.harvard.edu/p2/0837.aff Nochg: 0 NUL: 0 LZMA: 254 old: 1435933454 new: 1119694299 LZred: 22.02% +/Volumes/TB/project2/affs/0838.aff -> /Volumes/s.eecs.harvard.edu/p2/0838.aff Nochg: 1 NUL: 0 LZMA: 1260 old: 4214730263 new: 3456781815 LZred: 17.98% +/Volumes/TB/project2/affs/0839.aff -> /Volumes/s.eecs.harvard.edu/p2/0839.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163986 new: 6012 LZred: 96.33% +/Volumes/TB/project2/affs/0840.aff -> /Volumes/s.eecs.harvard.edu/p2/0840.aff Nochg: 0 NUL: 0 LZMA: 253 old: 1568616513 new: 1245594207 LZred: 20.59% +/Volumes/TB/project2/affs/0842.aff -> /Volumes/s.eecs.harvard.edu/p2/0842.aff Nochg: 0 NUL: 0 LZMA: 254 old: 1164956632 new: 911602816 LZred: 21.75% +/Volumes/TB/project2/affs/0843.aff -> /Volumes/s.eecs.harvard.edu/p2/0843.aff Nochg: 0 NUL: 0 LZMA: 253 old: 1367009114 new: 1123503815 LZred: 17.81% +/Volumes/TB/project2/affs/0844.aff -> /Volumes/s.eecs.harvard.edu/p2/0844.aff Nochg: 1 NUL: 0 LZMA: 1784 old: 6353685821 new: 5001601338 LZred: 21.28% +/Volumes/TB/project2/affs/0845.aff -> /Volumes/s.eecs.harvard.edu/p2/0845.aff Nochg: 0 NUL: 0 LZMA: 1837 old: 1981800014 new: 1537288424 LZred: 22.43% +/Volumes/TB/project2/affs/0846.aff -> /Volumes/s.eecs.harvard.edu/p2/0846.aff Nochg: 0 NUL: 2385 LZMA: 1 old: 54835 new: 2984 LZred: 94.56% +/Volumes/TB/project2/affs/0847.aff -> /Volumes/s.eecs.harvard.edu/p2/0847.aff Nochg: 6 NUL: 0 LZMA: 274 old: 2597256796 new: 2160382996 LZred: 16.82% +/Volumes/TB/project2/affs/0849.aff -> /Volumes/s.eecs.harvard.edu/p2/0849.aff Nochg: 0 NUL: 22 LZMA: 2350 old: 13846835197 new: 10646818009 LZred: 23.11% +/Volumes/TB/project2/affs/0850.aff -> /Volumes/s.eecs.harvard.edu/p2/0850.aff Nochg: 0 NUL: 0 LZMA: 43 old: 144683582 new: 114038101 LZred: 21.18% +/Volumes/TB/project2/affs/0851.aff -> /Volumes/s.eecs.harvard.edu/p2/0851.aff Nochg: 0 NUL: 2201 LZMA: 185 old: 7537197 new: 459678 LZred: 93.90% +/Volumes/TB/project2/affs/0852.aff -> /Volumes/s.eecs.harvard.edu/p2/0852.aff Nochg: 0 NUL: 0 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project2/affs/0853.aff -> /Volumes/s.eecs.harvard.edu/p2/0853.aff Nochg: 0 NUL: 0 LZMA: 202 old: 23616279 new: 11459104 LZred: 51.48% +/Volumes/TB/project2/affs/0854.aff -> /Volumes/s.eecs.harvard.edu/p2/0854.aff Nochg: 0 NUL: 0 LZMA: 177 old: 2190839214 new: 2057106208 LZred: 6.10% +/Volumes/TB/project2/affs/0855.aff -> /Volumes/s.eecs.harvard.edu/p2/0855.aff Nochg: 0 NUL: 0 LZMA: 261 old: 2511808521 new: 2212727700 LZred: 11.91% +/Volumes/TB/project2/affs/0856.aff -> /Volumes/s.eecs.harvard.edu/p2/0856.aff Nochg: 7 NUL: 200 LZMA: 1366 old: 12763088371 new: 10398739744 LZred: 18.52% +/Volumes/TB/project2/affs/0857.aff -> /Volumes/s.eecs.harvard.edu/p2/0857.aff Nochg: 0 NUL: 0 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project2/affs/0858.aff -> /Volumes/s.eecs.harvard.edu/p2/0858.aff Nochg: 0 NUL: 0 LZMA: 1 old: 81992 new: 3000 LZred: 96.34% +/Volumes/TB/project2/affs/0859.aff -> /Volumes/s.eecs.harvard.edu/p2/0859.aff Nochg: 0 NUL: 0 LZMA: 2 old: 505687 new: 236884 LZred: 53.16% +/Volumes/TB/project2/affs/0862.aff -> /Volumes/s.eecs.harvard.edu/p2/0862.aff Nochg: 0 NUL: 0 LZMA: 254 old: 1075307382 new: 833557541 LZred: 22.48% +/Volumes/TB/project2/affs/0863.aff -> /Volumes/s.eecs.harvard.edu/p2/0863.aff Nochg: 0 NUL: 2385 LZMA: 1 old: 54838 new: 2985 LZred: 94.56% +/Volumes/TB/project2/affs/0864.aff -> /Volumes/s.eecs.harvard.edu/p2/0864.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163990 new: 6016 LZred: 96.33% +/Volumes/TB/project2/affs/0865.aff -> /Volumes/s.eecs.harvard.edu/p2/0865.aff Nochg: 8 NUL: 0 LZMA: 272 old: 2463573717 new: 2073814315 LZred: 15.82% +/Volumes/TB/project2/affs/0866.aff -> /Volumes/s.eecs.harvard.edu/p2/0866.aff Nochg: 0 NUL: 0 LZMA: 14 old: 108451310 new: 87663671 LZred: 19.17% +/Volumes/TB/project2/affs/0867.aff -> /Volumes/s.eecs.harvard.edu/p2/0867.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163998 new: 6015 LZred: 96.33% +/Volumes/TB/project2/affs/0868.aff -> /Volumes/s.eecs.harvard.edu/p2/0868.aff Nochg: 0 NUL: 0 LZMA: 14 old: 108451310 new: 87663671 LZred: 19.17% +/Volumes/TB/project2/affs/0879.aff -> /Volumes/s.eecs.harvard.edu/p2/0879.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163993 new: 6013 LZred: 96.33% +/Volumes/TB/project2/affs/0881.aff -> /Volumes/s.eecs.harvard.edu/p2/0881.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163991 new: 6011 LZred: 96.33% +/Volumes/TB/project2/affs/0882.aff -> /Volumes/s.eecs.harvard.edu/p2/0882.aff Nochg: 0 NUL: 0 LZMA: 2 old: 4426221 new: 3947040 LZred: 10.83% +/Volumes/TB/project2/affs/0883.aff -> /Volumes/s.eecs.harvard.edu/p2/0883.aff Nochg: 0 NUL: 38 LZMA: 2 old: 164000 new: 6019 LZred: 96.33% +/Volumes/TB/project2/affs/0884.aff -> /Volumes/s.eecs.harvard.edu/p2/0884.aff Nochg: 0 NUL: 193 LZMA: 1 old: 38041 new: 2985 LZred: 92.15% +/Volumes/TB/project2/affs/0885.aff -> /Volumes/s.eecs.harvard.edu/p2/0885.aff Nochg: 0 NUL: 191 LZMA: 1 old: 64017 new: 2985 LZred: 95.34% +/Volumes/TB/project2/affs/0886.aff -> /Volumes/s.eecs.harvard.edu/p2/0886.aff Nochg: 0 NUL: 193 LZMA: 1 old: 38043 new: 2986 LZred: 92.15% +/Volumes/TB/project2/affs/0887.aff -> /Volumes/s.eecs.harvard.edu/p2/0887.aff Nochg: 0 NUL: 193 LZMA: 1 old: 38043 new: 2986 LZred: 92.15% +/Volumes/TB/project2/affs/0888.aff -> /Volumes/s.eecs.harvard.edu/p2/0888.aff Nochg: 0 NUL: 192 LZMA: 1 old: 53027 new: 2986 LZred: 94.37% +/Volumes/TB/project2/affs/0889.aff -> /Volumes/s.eecs.harvard.edu/p2/0889.aff Nochg: 0 NUL: 193 LZMA: 1 old: 38042 new: 2984 LZred: 92.16% +/Volumes/TB/project2/affs/0890.aff -> /Volumes/s.eecs.harvard.edu/p2/0890.aff Nochg: 0 NUL: 193 LZMA: 1 old: 72049 new: 2985 LZred: 95.86% +/Volumes/TB/project2/affs/0891.aff -> /Volumes/s.eecs.harvard.edu/p2/0891.aff Nochg: 0 NUL: 191 LZMA: 1 old: 64016 new: 2985 LZred: 95.34% +/Volumes/TB/project2/affs/0892.aff -> /Volumes/s.eecs.harvard.edu/p2/0892.aff Nochg: 0 NUL: 193 LZMA: 1 old: 38041 new: 2987 LZred: 92.15% +/Volumes/TB/project2/affs/0893.aff -> /Volumes/s.eecs.harvard.edu/p2/0893.aff Nochg: 0 NUL: 193 LZMA: 1 old: 38039 new: 2984 LZred: 92.16% +/Volumes/TB/project2/affs/0894.aff -> /Volumes/s.eecs.harvard.edu/p2/0894.aff Nochg: 0 NUL: 191 LZMA: 1 old: 64017 new: 2986 LZred: 95.34% +/Volumes/TB/project2/affs/0895.aff -> /Volumes/s.eecs.harvard.edu/p2/0895.aff Nochg: 0 NUL: 193 LZMA: 1 old: 38041 new: 2985 LZred: 92.15% +/Volumes/TB/project2/affs/0896.aff -> /Volumes/s.eecs.harvard.edu/p2/0896.aff Nochg: 0 NUL: 193 LZMA: 1 old: 38042 new: 2986 LZred: 92.15% +/Volumes/TB/project2/affs/0898.aff -> /Volumes/s.eecs.harvard.edu/p2/0898.aff Nochg: 0 NUL: 193 LZMA: 1 old: 74046 new: 2985 LZred: 95.97% +/Volumes/TB/project2/affs/0899.aff -> /Volumes/s.eecs.harvard.edu/p2/0899.aff Nochg: 0 NUL: 193 LZMA: 1 old: 38039 new: 2985 LZred: 92.15% +/Volumes/TB/project2/affs/0900.aff -> /Volumes/s.eecs.harvard.edu/p2/0900.aff Nochg: 0 NUL: 193 LZMA: 1 old: 71740 new: 2985 LZred: 95.84% +/Volumes/TB/project2/affs/0901.aff -> /Volumes/s.eecs.harvard.edu/p2/0901.aff Nochg: 0 NUL: 193 LZMA: 1 old: 74045 new: 2986 LZred: 95.97% +/Volumes/TB/project2/affs/0902.aff -> /Volumes/s.eecs.harvard.edu/p2/0902.aff Nochg: 0 NUL: 193 LZMA: 1 old: 72049 new: 2985 LZred: 95.86% +/Volumes/TB/project2/affs/0903.aff -> /Volumes/s.eecs.harvard.edu/p2/0903.aff Nochg: 0 NUL: 191 LZMA: 1 old: 64014 new: 2985 LZred: 95.34% +/Volumes/TB/project2/affs/0904.aff -> /Volumes/s.eecs.harvard.edu/p2/0904.aff Nochg: 0 NUL: 0 LZMA: 25 old: 161931770 new: 117577248 LZred: 27.39% +/Volumes/TB/project2/affs/0905.aff -> /Volumes/s.eecs.harvard.edu/p2/0905.aff Nochg: 0 NUL: 0 LZMA: 8 old: 62024390 new: 48680159 LZred: 21.51% +/Volumes/TB/project2/affs/0907.aff -> /Volumes/s.eecs.harvard.edu/p2/0907.aff Nochg: 0 NUL: 0 LZMA: 8 old: 63170531 new: 50102880 LZred: 20.69% +/Volumes/TB/project2/affs/0908.aff -> /Volumes/s.eecs.harvard.edu/p2/0908.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163990 new: 6016 LZred: 96.33% +/Volumes/TB/project2/affs/0909.aff -> /Volumes/s.eecs.harvard.edu/p2/0909.aff Nochg: 0 NUL: 0 LZMA: 33 old: 257639430 new: 200071423 LZred: 22.34% +/Volumes/TB/project2/affs/0911.aff -> /Volumes/s.eecs.harvard.edu/p2/0911.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163997 new: 6021 LZred: 96.33% +/Volumes/TB/project2/affs/0913.aff -> /Volumes/s.eecs.harvard.edu/p2/0913.aff Nochg: 0 NUL: 0 LZMA: 25 old: 245883046 new: 221456739 LZred: 9.93% +/Volumes/TB/project2/affs/0914.aff -> /Volumes/s.eecs.harvard.edu/p2/0914.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163989 new: 6013 LZred: 96.33% +/Volumes/TB/project2/affs/0915.aff -> /Volumes/s.eecs.harvard.edu/p2/0915.aff Nochg: 0 NUL: 0 LZMA: 77 old: 1031790524 new: 1014569473 LZred: 1.67% +/Volumes/TB/project2/affs/0916.aff -> /Volumes/s.eecs.harvard.edu/p2/0916.aff Nochg: 0 NUL: 0 LZMA: 99 old: 1319799974 new: 1249965771 LZred: 5.29% +/Volumes/TB/project2/affs/0917.aff -> /Volumes/s.eecs.harvard.edu/p2/0917.aff Nochg: 0 NUL: 0 LZMA: 4 old: 12367708 new: 9352720 LZred: 24.38% +/Volumes/TB/project2/affs/0918.aff -> /Volumes/s.eecs.harvard.edu/p2/0918.aff Nochg: 0 NUL: 0 LZMA: 5 old: 29094857 new: 23854944 LZred: 18.01% +/Volumes/TB/project2/affs/0921.aff -> /Volumes/s.eecs.harvard.edu/p2/0921.aff Nochg: 0 NUL: 0 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project2/affs/0922.aff -> /Volumes/s.eecs.harvard.edu/p2/0922.aff Nochg: 0 NUL: 542 LZMA: 1 old: 37202 new: 3502 LZred: 90.59% +/Volumes/TB/project2/affs/0924.aff -> /Volumes/s.eecs.harvard.edu/p2/0924.aff Nochg: 1 NUL: 0 LZMA: 211 old: 2623467971 new: 2167874682 LZred: 17.37% +/Volumes/TB/project2/affs/0925.aff -> /Volumes/s.eecs.harvard.edu/p2/0925.aff Nochg: 0 NUL: 0 LZMA: 29 old: 36529725 new: 18386388 LZred: 49.67% +/Volumes/TB/project2/affs/0926.aff -> /Volumes/s.eecs.harvard.edu/p2/0926.aff Nochg: 0 NUL: 0 LZMA: 33 old: 131414056 new: 85680339 LZred: 34.80% +/Volumes/TB/project2/affs/0927.aff -> /Volumes/s.eecs.harvard.edu/p2/0927.aff Nochg: 0 NUL: 0 LZMA: 33 old: 544831974 new: 25756739 LZred: 95.27% +/Volumes/TB/project2/affs/0928.aff -> /Volumes/s.eecs.harvard.edu/p2/0928.aff Nochg: 0 NUL: 0 LZMA: 2 old: 16859209 new: 6015 LZred: 99.96% +/Volumes/TB/project2/affs/0929.aff -> /Volumes/s.eecs.harvard.edu/p2/0929.aff Nochg: 0 NUL: 13 LZMA: 130 old: 95292250 new: 64112350 LZred: 32.72% +/Volumes/TB/project2/affs/0930.aff -> /Volumes/s.eecs.harvard.edu/p2/0930.aff Nochg: 0 NUL: 0 LZMA: 33 old: 120906667 new: 80594024 LZred: 33.34% +/Volumes/TB/project2/affs/0931.aff -> /Volumes/s.eecs.harvard.edu/p2/0931.aff Nochg: 0 NUL: 0 LZMA: 11 old: 83442031 new: 65937196 LZred: 20.98% +/Volumes/TB/project2/affs/0932.aff -> /Volumes/s.eecs.harvard.edu/p2/0932.aff Nochg: 0 NUL: 0 LZMA: 256 old: 4178374 new: 627225 LZred: 84.99% +/Volumes/TB/project2/affs/0933.aff -> /Volumes/s.eecs.harvard.edu/p2/0933.aff Nochg: 0 NUL: 0 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project2/affs/0935.aff -> /Volumes/s.eecs.harvard.edu/p2/0935.aff Nochg: 0 NUL: 0 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project2/affs/0936.aff -> /Volumes/s.eecs.harvard.edu/p2/0936.aff Nochg: 0 NUL: 0 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project2/affs/0937.aff -> /Volumes/s.eecs.harvard.edu/p2/0937.aff Nochg: 0 NUL: 382 LZMA: 3 old: 7076384 new: 6557211 LZred: 7.34% +/Volumes/TB/project2/affs/0938.aff -> /Volumes/s.eecs.harvard.edu/p2/0938.aff Nochg: 0 NUL: 383 LZMA: 2 old: 1300478 new: 1126777 LZred: 13.36% +/Volumes/TB/project2/affs/0939.aff -> /Volumes/s.eecs.harvard.edu/p2/0939.aff Nochg: 0 NUL: 382 LZMA: 3 old: 1387459 new: 1150766 LZred: 17.06% +/Volumes/TB/project2/affs/0940.aff -> /Volumes/s.eecs.harvard.edu/p2/0940.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163989 new: 6015 LZred: 96.33% +/Volumes/TB/project2/affs/0941.aff -> /Volumes/s.eecs.harvard.edu/p2/0941.aff Nochg: 0 NUL: 0 LZMA: 4 old: 513045 new: 109539 LZred: 78.65% +/Volumes/TB/project2/affs/0943.aff -> /Volumes/s.eecs.harvard.edu/p2/0943.aff Nochg: 0 NUL: 3 LZMA: 251 old: 1545710408 new: 1057683216 LZred: 31.57% +/Volumes/TB/project2/affs/0944.aff -> /Volumes/s.eecs.harvard.edu/p2/0944.aff Nochg: 0 NUL: 214 LZMA: 2 old: 120928 new: 6023 LZred: 95.02% +/Volumes/TB/project2/affs/0946.aff -> /Volumes/s.eecs.harvard.edu/p2/0946.aff Nochg: 4 NUL: 0 LZMA: 247 old: 1828409440 new: 1418515927 LZred: 22.42% +/Volumes/TB/project2/affs/0947.aff -> /Volumes/s.eecs.harvard.edu/p2/0947.aff Nochg: 1 NUL: 0 LZMA: 253 old: 1836181924 new: 1476330874 LZred: 19.60% +/Volumes/TB/project2/affs/0948.aff -> /Volumes/s.eecs.harvard.edu/p2/0948.aff Nochg: 0 NUL: 0 LZMA: 44 old: 6745056 new: 1549101 LZred: 77.03% +/Volumes/TB/project2/affs/0949.aff -> /Volumes/s.eecs.harvard.edu/p2/0949.aff Nochg: 0 NUL: 256 LZMA: 1 old: 61711 new: 5164 LZred: 91.63% +/Volumes/TB/project2/affs/0950.aff -> /Volumes/s.eecs.harvard.edu/p2/0950.aff Nochg: 0 NUL: 0 LZMA: 194 old: 1461672684 new: 1156094492 LZred: 20.91% +/Volumes/TB/project2/affs/0951.aff -> /Volumes/s.eecs.harvard.edu/p2/0951.aff Nochg: 0 NUL: 0 LZMA: 192 old: 237079243 new: 185859416 LZred: 21.60% +/Volumes/TB/project2/affs/0952.aff -> /Volumes/s.eecs.harvard.edu/p2/0952.aff Nochg: 0 NUL: 0 LZMA: 29 old: 283177530 new: 111411047 LZred: 60.66% +/Volumes/TB/project2/affs/0953.aff -> /Volumes/s.eecs.harvard.edu/p2/0953.aff Nochg: 0 NUL: 10 LZMA: 21 old: 45163616 new: 16046972 LZred: 64.47% +/Volumes/TB/project2/affs/0954.aff -> /Volumes/s.eecs.harvard.edu/p2/0954.aff Nochg: 0 NUL: 0 LZMA: 5 old: 15171872 new: 7050806 LZred: 53.53% +/Volumes/TB/project2/affs/0956.aff -> /Volumes/s.eecs.harvard.edu/p2/0956.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163996 new: 6020 LZred: 96.33% +/Volumes/TB/project2/affs/0957.aff -> /Volumes/s.eecs.harvard.edu/p2/0957.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163997 new: 6017 LZred: 96.33% +/Volumes/TB/project2/affs/0959.aff -> /Volumes/s.eecs.harvard.edu/p2/0959.aff Nochg: 0 NUL: 0 LZMA: 6 old: 16916662 new: 7602620 LZred: 55.06% +/Volumes/TB/project2/affs/0960.aff -> /Volumes/s.eecs.harvard.edu/p2/0960.aff Nochg: 0 NUL: 104 LZMA: 4 old: 270019 new: 74009 LZred: 72.59% +/Volumes/TB/project2/affs/0962.aff -> /Volumes/s.eecs.harvard.edu/p2/0962.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163993 new: 6018 LZred: 96.33% +/Volumes/TB/project2/affs/0967.aff -> /Volumes/s.eecs.harvard.edu/p2/0967.aff Nochg: 0 NUL: 0 LZMA: 2 old: 164000 new: 6019 LZred: 96.33% +/Volumes/TB/project2/affs/0968.aff -> /Volumes/s.eecs.harvard.edu/p2/0968.aff Nochg: 0 NUL: 0 LZMA: 2 old: 968255 new: 692015 LZred: 28.53% +/Volumes/TB/project2/affs/0969.aff -> /Volumes/s.eecs.harvard.edu/p2/0969.aff Nochg: 0 NUL: 0 LZMA: 8 old: 22024442 new: 9128206 LZred: 58.55% +/Volumes/TB/project2/affs/0970.aff -> /Volumes/s.eecs.harvard.edu/p2/0970.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163988 new: 6012 LZred: 96.33% +/Volumes/TB/project2/affs/0972.aff -> /Volumes/s.eecs.harvard.edu/p2/0972.aff Nochg: 0 NUL: 0 LZMA: 254 old: 862849916 new: 664005208 LZred: 23.05% +/Volumes/TB/project2/affs/0973.aff -> /Volumes/s.eecs.harvard.edu/p2/0973.aff Nochg: 0 NUL: 0 LZMA: 255 old: 2347882745 new: 1678465333 LZred: 28.51% +/Volumes/TB/project2/affs/0974.aff -> /Volumes/s.eecs.harvard.edu/p2/0974.aff Nochg: 0 NUL: 0 LZMA: 258 old: 1598463735 new: 1174926240 LZred: 26.50% +/Volumes/TB/project2/affs/0975.aff -> /Volumes/s.eecs.harvard.edu/p2/0975.aff Nochg: 0 NUL: 63 LZMA: 188 old: 1265973854 new: 1027800727 LZred: 18.81% +/Volumes/TB/project2/affs/0977.aff -> /Volumes/s.eecs.harvard.edu/p2/0977.aff Nochg: 0 NUL: 17 LZMA: 240 old: 2469588926 new: 1667346858 LZred: 32.48% +/Volumes/TB/project2/affs/0978.aff -> /Volumes/s.eecs.harvard.edu/p2/0978.aff Nochg: 0 NUL: 0 LZMA: 254 old: 1019410977 new: 811022250 LZred: 20.44% +/Volumes/TB/project2/affs/0979.aff -> /Volumes/s.eecs.harvard.edu/p2/0979.aff Nochg: 0 NUL: 0 LZMA: 257 old: 1129816757 new: 858266283 LZred: 24.03% +/Volumes/TB/project2/affs/0980.aff -> /Volumes/s.eecs.harvard.edu/p2/0980.aff Nochg: 0 NUL: 0 LZMA: 244 old: 2096277105 new: 1709685945 LZred: 18.44% +/Volumes/TB/project2/affs/0981.aff -> /Volumes/s.eecs.harvard.edu/p2/0981.aff Nochg: 0 NUL: 69 LZMA: 138 old: 755589071 new: 566125356 LZred: 25.07% +/Volumes/TB/project2/affs/0982.aff -> /Volumes/s.eecs.harvard.edu/p2/0982.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163997 new: 6013 LZred: 96.33% +/Volumes/TB/project2/affs/0985.aff -> /Volumes/s.eecs.harvard.edu/p2/0985.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163986 new: 6011 LZred: 96.33% +/Volumes/TB/project2/affs/0986.aff -> /Volumes/s.eecs.harvard.edu/p2/0986.aff Nochg: 0 NUL: 0 LZMA: 2 old: 164000 new: 6014 LZred: 96.33% +/Volumes/TB/project2/affs/0991.aff -> /Volumes/s.eecs.harvard.edu/p2/0991.aff Nochg: 0 NUL: 0 LZMA: 15 old: 24957732 new: 9819064 LZred: 60.66% +/Volumes/TB/project2/affs/0992.aff -> /Volumes/s.eecs.harvard.edu/p2/0992.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163994 new: 6011 LZred: 96.33% +/Volumes/TB/project2/affs/0993.aff -> /Volumes/s.eecs.harvard.edu/p2/0993.aff Nochg: 0 NUL: 193 LZMA: 1 old: 38040 new: 2984 LZred: 92.16% +/Volumes/TB/project2/affs/0995.aff -> /Volumes/s.eecs.harvard.edu/p2/0995.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163989 new: 6019 LZred: 96.33% +/Volumes/TB/project2/affs/0996.aff -> /Volumes/s.eecs.harvard.edu/p2/0996.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163992 new: 6012 LZred: 96.33% +/Volumes/TB/project2/affs/0997.aff -> /Volumes/s.eecs.harvard.edu/p2/0997.aff Nochg: 0 NUL: 0 LZMA: 26 old: 197635302 new: 131498725 LZred: 33.46% +/Volumes/TB/project2/affs/0998.aff -> /Volumes/s.eecs.harvard.edu/p2/0998.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163989 new: 6017 LZred: 96.33% +/Volumes/TB/project2/affs/1000.aff -> /Volumes/s.eecs.harvard.edu/p2/1000.aff Nochg: 0 NUL: 0 LZMA: 2 old: 163989 new: 6010 LZred: 96.34% +/Volumes/TB/project2/affs/1001.aff -> /Volumes/s.eecs.harvard.edu/p2/1001.aff Nochg: 0 NUL: 0 LZMA: 15 old: 24803943 new: 9359385 LZred: 62.27% +/Volumes/TB/project2/affs/1004.aff -> /Volumes/s.eecs.harvard.edu/p2/1004.aff Nochg: 0 NUL: 0 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project2/affs/1006.aff -> /Volumes/s.eecs.harvard.edu/p2/1006.aff Nochg: 0 NUL: 0 LZMA: 0 old: 0 new: 0 LZred: nan% +/Volumes/TB/project2/affs/1007.aff -> /Volumes/s.eecs.harvard.edu/p2/1007.aff Nochg: 0 NUL: 0 LZMA: 1193 old: 15262253937 new: 14103836834 LZred: 7.59% +/Volumes/TB/project2/affs/1008.aff -> /Volumes/s.eecs.harvard.edu/p2/1008.aff Nochg: 18 NUL: 0 LZMA: 1161 old: 11404830361 new: 9657796749 LZred: 15.32% +/Volumes/TB/project2/affs/1009.aff -> /Volumes/s.eecs.harvard.edu/p2/1009.aff Nochg: 11 NUL: 30 LZMA: 1255 old: 12994582744 new: 11117199965 LZred: 14.45% +/Volumes/TB/project2/affs/1010.aff -> /Volumes/s.eecs.harvard.edu/p2/1010.aff Nochg: 2 NUL: 0 LZMA: 812 old: 7578649990 new: 6394191964 LZred: 15.63% +/Volumes/TB/project2/affs/1011.aff -> /Volumes/s.eecs.harvard.edu/p2/1011.aff Nochg: 56 NUL: 33 LZMA: 818 old: 8860084925 new: 7408980492 LZred: 16.38% +/Volumes/TB/project2/affs/1012.aff -> /Volumes/s.eecs.harvard.edu/p2/1012.aff Nochg: 0 NUL: 0 LZMA: 2236 old: 34202486980 new: 30029751690 LZred: 12.20% +/Volumes/TB/project2/affs/1013.aff -> /Volumes/s.eecs.harvard.edu/p2/1013.aff Nochg: 7 NUL: 837 LZMA: 1503 old: 21081733051 new: 20028345069 LZred: 5.00% +/Volumes/TB/project2/affs/1013.aff -> /Volumes/s.eecs.harvard.edu/p2/1013.aff Nochg: 7 NUL: 837 LZMA: 1503 old: 21081733051 new: 20028345069 LZred: 5.00% +/Volumes/TB/project2/affs/1015.aff -> /Volumes/s.eecs.harvard.edu/p2/1015.aff Nochg: 50 NUL: 0 LZMA: 761 old: 8629799116 new: 7663827777 LZred: 11.19% +/Volumes/TB/project2/affs/1016.aff -> /Volumes/s.eecs.harvard.edu/p2/1016.aff Nochg: 77 NUL: 0 LZMA: 1081 old: 13644925677 new: 12656767227 LZred: 7.24% +/Volumes/TB/project2/affs/1017.aff -> /Volumes/s.eecs.harvard.edu/p2/1017.aff Nochg: 0 NUL: 152 LZMA: 457 old: 2688499294 new: 1943029684 LZred: 27.73% +/Volumes/TB/project2/affs/1018.aff -> /Volumes/s.eecs.harvard.edu/p2/1018.aff Nochg: 46 NUL: 0 LZMA: 1698 old: 23662862798 new: 21029897935 LZred: 11.13% +/Volumes/TB/project2/affs/1019.aff -> /Volumes/s.eecs.harvard.edu/p2/1019.aff Nochg: 262 NUL: 82 LZMA: 1852 old: 24480828240 new: 22505083712 LZred: 8.07% +/Volumes/TB/project2/affs/1020.aff -> /Volumes/s.eecs.harvard.edu/p2/1020.aff Nochg: 8 NUL: 18 LZMA: 552 old: 4799530311 new: 3990253370 LZred: 16.86% +/Volumes/TB/project2/affs/1021.aff -> /Volumes/s.eecs.harvard.edu/p2/1021.aff Nochg: 1135 NUL: 0 LZMA: 581 old: 9386805230 new: 9338377060 LZred: 0.52% +/Volumes/TB/project2/affs/1022.aff -> /Volumes/s.eecs.harvard.edu/p2/1022.aff Nochg: 8 NUL: 18 LZMA: 552 old: 4799530199 new: 3990252391 LZred: 16.86% +/Volumes/TB/project2/affs/1023.aff -> /Volumes/s.eecs.harvard.edu/p2/1023.aff Nochg: 0 NUL: 0 LZMA: 810 old: 1265565721 new: 935430451 LZred: 26.09% +/Volumes/TB/project2/affs/1024.aff -> /Volumes/s.eecs.harvard.edu/p2/1024.aff Nochg: 0 NUL: 0 LZMA: 579 old: 8286378301 new: 1745242913 LZred: 78.94% +/Volumes/TB/project2/affs/1025.aff -> /Volumes/s.eecs.harvard.edu/p2/1025.aff Nochg: 49 NUL: 0 LZMA: 726 old: 9921942352 new: 9359795270 LZred: 5.67% diff --git a/stats/project3-report.txt b/stats/project3-report.txt new file mode 100644 index 0000000..f9711a7 --- /dev/null +++ b/stats/project3-report.txt @@ -0,0 +1,113 @@ +Name AF_IMAGESIZE Compressed Uncompressed Blank Bad +/access/p3/1026.aff 13020069888 2478856028 13020069888 0 0 +/access/p3/1027.aff 10245537792 9071965795 10245537792 0 0 +/access/p3/1028.aff 13578485760 7696182839 13578485760 0 0 +/access/p3/1029.aff 18042716160 15059422864 18042716160 0 0 +/access/p3/1030.aff 15377080320 15306171861 15377080320 0 0 +/access/p3/1031.aff 13578485760 8423414324 13578485760 0 0 +/access/p3/1032.aff 10005037056 9795216323 10005037056 0 0 +/access/p3/1033.aff 40020664320 20239018598 40020664320 0 0 +/access/p3/1034.aff 20490559488 17787310540 20490559488 0 0 +/access/p3/1035.aff 40060403712 1657190312 40060403712 0 0 +/access/p3/1036.aff 10141286400 4978582196 10141286400 0 0 +/access/p3/1037.aff 10005037056 1129337679 10005037056 0 0 +/access/p3/1038.aff 40982151168 27803054991 40982151168 0 0 +/access/p3/1039.aff 61492838400 53296717556 61492838400 0 0 +/access/p3/1040.aff 20404101120 3088145332 20404101120 0 0 +/access/p3/1041.aff 20491075584 18494855503 20491075584 0 0 +/access/p3/1042.aff 10141286400 4569625678 10141286400 0 0 +/access/p3/1043.aff 10141286400 9257435534 10141286400 0 0 +/access/p3/1044.aff 10205282304 2311020624 10205282304 0 0 +/access/p3/1046.aff 7510164480 1127358 7510164480 14665236 0 +/access/p3/1047.aff 7683637248 8353578 7683637248 14988020 0 +/access/p3/1048.aff 7509196800 1125099 7509196800 14663347 0 +/access/p3/1049.aff 50331648 8975 50331648 2048 163840 +/access/p3/1050.aff 7683637248 5801804 7683637248 14988114 0 +/access/p3/1051.aff 268435456 55805430 268435456 23489 163840 +/access/p3/1052.aff 620756992 253649861 620756992 44500 163840 +/access/p3/1053.aff 503316480 163892020 503316480 37989 163840 +/access/p3/1054.aff 4325529600 1163903121 4211081216 544102 655360 +/access/p3/1055.aff 4325529600 731687777 4325529600 2546868 0 +/access/p3/1057.aff 4325529600 1123968794 4325529600 268775 26924 +/access/p3/1058.aff 4325529600 1271378599 4325529600 3277776 0 +/access/p3/1059.aff 4325529600 1844396446 4325529600 451449 0 +/access/p3/1060.aff 117440512 25420186 117440512 14721 163840 +/access/p3/1061.aff 4325529600 1410054191 4325529600 295448 0 +/access/p3/1062.aff 4325529600 2028132782 4325529600 222390 0 +/access/p3/1063.aff 4325529600 1375511574 4325529600 2135781 0 +/access/p3/1064.aff 2248146944 1055351048 2248146944 129073 163840 +/access/p3/1065.aff 520093696 220691222 520093696 40904 163840 +/access/p3/1066.aff 4325529600 1467809310 4325529600 365083 0 +/access/p3/1067.aff 4325529600 1362442307 4325529600 316365 360448 +/access/p3/1068.aff 1124073472 508186343 1124073472 75097 163840 +/access/p3/1069.aff 4325529600 2365780060 4325529600 297468 0 +/access/p3/1070.aff 4325529600 1360874182 4325529600 606855 0 +/access/p3/1071.aff 4325529600 1248376849 4325529600 645883 0 +/access/p3/1073.aff 117440512 36582140 117440512 16968 163840 +/access/p3/1074.aff 4325529600 1372671208 4325529600 298200 0 +/access/p3/1075.aff 4325529600 1399297006 4325529600 517050 0 +/access/p3/1076.aff 4325529600 1162915029 4325529600 433626 0 +/access/p3/1077.aff 4325529600 1313629179 4325529600 327510 0 +/access/p3/1078.aff 4325529600 1214450419 4325529600 478067 0 +/access/p3/1079.aff 4093640704 1329580303 4093640704 378849 163840 +/access/p3/1080.aff 369098752 164397644 369098752 27268 163840 +/access/p3/1081.aff 6448619520 1127672 6448619520 12591906 0 +/access/p3/1082.aff 6448619520 1151064 6448619520 12575843 0 +/access/p3/1083.aff 6448619520 1150962 6448619520 12575843 0 +/access/p3/1084.aff 6448619520 1128679 6448619520 12575843 0 +/access/p3/1085.aff 6448619520 1125357 6448619520 12591907 0 +/access/p3/1086.aff 6448619520 3786642 6448619520 12576949 0 +/access/p3/1088.aff 6448619520 6046074 6448619520 12576027 0 +/access/p3/1089.aff 6448619520 1128734 6448619520 12575843 0 +/access/p3/1090.aff 6448619520 1150829 6448619520 12575843 0 +/access/p3/1091.aff 3228696576 1140167348 3228696576 262581 0 +/access/p3/1092.aff 4303756800 1623644778 4303756800 352614 0 +/access/p3/1094.aff 4303756800 1770575589 4303756800 424797 0 +/access/p3/1101.aff 0 11538573 0 0 294912 +/access/p3/1103.aff 536870912 660416 536870912 24658 163840 +/access/p3/1104.aff 3204448256 926250782 3204448256 73098 622592 +/access/p3/1105.aff 4325529600 1067220631 4325529600 1157629 0 +/access/p3/1106.aff 4328562176 1493958112 4328562176 1585493 163840 +/access/p3/1107.aff 6448619520 5991610 6448619520 24656 0 +/access/p3/1108.aff 20416757760 11251400920 20416757760 3367310 0 +/access/p3/1109.aff 2111864832 266987840 2111864832 3201457 0 +/access/p3/1110.aff 1624670208 3392 1624670208 3141377 65536 +/access/p3/1111.aff 1624670208 388 1624670208 3173183 0 +/access/p3/1112.aff 1624670208 388 1624670208 3173183 0 +/access/p3/1113.aff 50331648 8940 50331648 0 163840 +/access/p3/1114.aff 1624670208 24358 1624670208 3026633 294912 +/access/p3/1115.aff 8455200768 2016 8455200768 16514063 0 +/access/p3/1116.aff 20520493056 4896 20520493056 40079087 0 +/access/p3/1117.aff 8455200768 4717 8455200768 16514062 0 +/access/p3/1122.aff 815394816 1038756 815394816 1570381 32768 +/access/p3/1123.aff 100663296 11843045 100663296 12605 163840 +/access/p3/1126.aff 3080060928 4959 3080060928 6015346 0 +/access/p3/1129.aff 1443004416 38781103 1443004416 23720 0 +/access/p3/1130.aff 4325529600 1067220631 4325529600 1157629 0 +/access/p3/1131.aff 0 176322198 0 0 0 +/access/p3/1132.aff 0 10620136 0 0 0 +/access/p3/1133.aff 0 15722086 0 0 0 +/access/p3/1134.aff 810786816 174168054 810786816 540233 0 +/access/p3/1135.aff 2147837440 899671868 2147837440 10091 0 +/access/p3/1136.aff 2139710976 478880679 2139710976 31660 98304 +/access/p3/1137.aff 0 24647719 0 0 0 +/access/p3/1138.aff 2147837440 1129738776 2147837440 27347 0 +/access/p3/1139.aff 2577383424 418499175 1543503872 168150 393216 +/access/p3/1140.aff 50331648 8937 50331648 0 163840 +/access/p3/1141.aff 10239860736 6943184445 10239860736 390936 0 +/access/p3/1142.aff 2111864832 551783911 2111864832 351731 0 +/access/p3/1144.aff 3249340416 1352830218 3249340416 233808 0 +/access/p3/1145.aff 9100369920 6461235683 9100369920 350100 0 +/access/p3/1146.aff 4000776192 2088084952 4000776192 218044 79616 +/access/p3/1147.aff 3249340416 1553368803 3249340416 219780 0 +/access/p3/1148.aff 4310433792 1028 4310433792 8418815 0 +/access/p3/1149.aff 3249340416 1146929753 3249340416 149331 0 +/access/p3/1150.aff 4311982080 1032 4311982080 8421839 0 +/access/p3/1151.aff 4311982080 1032 4311982080 8421839 0 +/access/p3/1152.aff 4311982080 1032 4311982080 8421839 0 +/access/p3/1153.aff 4311982080 1032 4311982080 8421839 0 +/access/p3/1154.aff 4311982080 743840297 2231369728 137072 524288 +/access/p3/1155.aff 4311982080 1032 4311982080 8421839 0 +/access/p3/1156.aff 4311982080 1032 4311982080 8421839 0 +/access/p3/1157.aff 4311982080 1032 4311982080 8421839 0 +/access/p3/1158.aff 4311982080 1032 4311982080 8421839 0 diff --git a/stats/project3.afcompare.preen.txt b/stats/project3.afcompare.preen.txt new file mode 100644 index 0000000..c993661 --- /dev/null +++ b/stats/project3.afcompare.preen.txt @@ -0,0 +1,114 @@ +/project3/affs/1145.aff -> /access/p3/1145.aff Nochg: 1 NUL: 0 LZMA: 537 old: 7018536416 new: 6360572387 LZred: 9.37% +/project3/affs/1026.aff -> /access/p3/1026.aff Nochg: 0 NUL: 198 LZMA: 560 old: 2349142562 new: 2160088132 LZred: 8.05% +/project3/affs/1027.aff -> /access/p3/1027.aff Nochg: 8 NUL: 0 LZMA: 581 old: 8832405742 new: 8568649315 LZred: 2.99% +/project3/affs/1030.aff -> /access/p3/1030.aff Nochg: 906 NUL: 0 LZMA: 8 old: 56090041 new: 55682517 LZred: 0.73% +/project3/affs/1029.aff -> /access/p3/1029.aff Nochg: 0 NUL: 0 LZMA: 1066 old: 15486154083 new: 14891650704 LZred: 3.84% +/project3/affs/1028.aff -> /access/p3/1028.aff Nochg: 1 NUL: 0 LZMA: 802 old: 8337750083 new: 7561965111 LZred: 9.30% +/project3/affs/1032.aff -> /access/p3/1032.aff Nochg: 13 NUL: 0 LZMA: 534 old: 8759081060 new: 8738251715 LZred: 0.24% +/project3/affs/1033.aff -> /access/p3/1033.aff Nochg: 5 NUL: 38 LZMA: 2216 old: 18049411129 new: 18024425934 LZred: 0.14% +/project3/affs/1034.aff -> /access/p3/1034.aff Nochg: 18 NUL: 4 LZMA: 1173 old: 17789618342 new: 17032335804 LZred: 4.26% +/project3/affs/1035.aff -> /access/p3/1035.aff Nochg: 62 NUL: 2079 LZMA: 236 old: 442254980 new: 432445228 LZred: 2.22% +/project3/affs/1036.aff -> /access/p3/1036.aff Nochg: 97 NUL: 242 LZMA: 213 old: 2602025817 new: 2461998828 LZred: 5.38% +/project3/affs/1037.aff -> /access/p3/1037.aff Nochg: 0 NUL: 417 LZMA: 180 old: 1780740908 new: 1129336011 LZred: 36.58% +/project3/affs/1031.aff -> /access/p3/1031.aff Nochg: 17 NUL: 0 LZMA: 772 old: 10592210207 new: 7785880116 LZred: 26.49% +/project3/affs/1040.aff -> /access/p3/1040.aff Nochg: 0 NUL: 658 LZMA: 557 old: 3958059406 new: 3054588268 LZred: 22.83% +/project3/affs/1041.aff -> /access/p3/1041.aff Nochg: 72 NUL: 50 LZMA: 832 old: 12883613129 new: 12790601863 LZred: 0.72% +/project3/affs/1042.aff -> /access/p3/1042.aff Nochg: 0 NUL: 0 LZMA: 600 old: 5397286728 new: 4485739598 LZred: 16.89% +/project3/affs/1043.aff -> /access/p3/1043.aff Nochg: 0 NUL: 0 LZMA: 605 old: 9366991055 new: 9257435534 LZred: 1.17% +/project3/affs/1044.aff -> /access/p3/1044.aff Nochg: 0 NUL: 281 LZMA: 320 old: 2712258629 new: 2176801772 LZred: 19.74% +/project3/affs/1038.aff -> /access/p3/1038.aff Nochg: 233 NUL: 40 LZMA: 1940 old: 21460084567 new: 20035203823 LZred: 6.64% +/project3/affs/1039.aff -> /access/p3/1039.aff Nochg: 60 NUL: 1 LZMA: 3410 old: 50523161674 new: 49018527472 LZred: 2.98% +/project3/affs/1046.aff -> /access/p3/1046.aff Nochg: 0 NUL: 446 LZMA: 2 old: 1252110 new: 1125574 LZred: 10.11% +/project3/affs/1047.aff -> /access/p3/1047.aff Nochg: 0 NUL: 456 LZMA: 2 old: 9247297 new: 8351754 LZred: 9.68% +/project3/affs/1048.aff -> /access/p3/1048.aff Nochg: 0 NUL: 447 LZMA: 1 old: 1241144 new: 1123311 LZred: 9.49% +/project3/affs/1050.aff -> /access/p3/1050.aff Nochg: 0 NUL: 456 LZMA: 2 old: 6808020 new: 5799980 LZred: 14.81% +/project3/affs/1049.aff -> /access/p3/1049.aff Nochg: 0 NUL: 0 LZMA: 3 old: 219261 new: 8975 LZred: 95.91% +/project3/affs/1051.aff -> /access/p3/1051.aff Nochg: 0 NUL: 0 LZMA: 16 old: 75980624 new: 55805430 LZred: 26.55% +/project3/affs/1052.aff -> /access/p3/1052.aff Nochg: 0 NUL: 0 LZMA: 34 old: 248571465 new: 203318213 LZred: 18.21% +/project3/affs/1053.aff -> /access/p3/1053.aff Nochg: 0 NUL: 0 LZMA: 30 old: 206450131 new: 163892020 LZred: 20.61% +/project3/affs/1054.aff -> /access/p3/1054.aff Nochg: 0 NUL: 0 LZMA: 249 old: 1515515281 new: 1130348689 LZred: 25.41% +/project3/affs/1055.aff -> /access/p3/1055.aff Nochg: 0 NUL: 0 LZMA: 256 old: 1082121218 new: 698133345 LZred: 35.48% +/project3/affs/1057.aff -> /access/p3/1057.aff Nochg: 0 NUL: 0 LZMA: 256 old: 1733232800 new: 1090414362 LZred: 37.09% +/project3/affs/1058.aff -> /access/p3/1058.aff Nochg: 2 NUL: 65 LZMA: 183 old: 1310591568 new: 1103606179 LZred: 15.79% +/project3/affs/1059.aff -> /access/p3/1059.aff Nochg: 0 NUL: 0 LZMA: 252 old: 2151398947 new: 1743733150 LZred: 18.95% +/project3/affs/1060.aff -> /access/p3/1060.aff Nochg: 0 NUL: 0 LZMA: 7 old: 31995260 new: 25420186 LZred: 20.55% +/project3/affs/1061.aff -> /access/p3/1061.aff Nochg: 0 NUL: 0 LZMA: 251 old: 1771758603 new: 1292613679 LZred: 27.04% +/project3/affs/1062.aff -> /access/p3/1062.aff Nochg: 0 NUL: 1 LZMA: 245 old: 2241013555 new: 1826806186 LZred: 18.48% +/project3/affs/1063.aff -> /access/p3/1063.aff Nochg: 0 NUL: 0 LZMA: 254 old: 1616263839 new: 1308402710 LZred: 19.05% +/project3/affs/1064.aff -> /access/p3/1064.aff Nochg: 0 NUL: 0 LZMA: 129 old: 1161730180 new: 971464968 LZred: 16.38% +/project3/affs/1065.aff -> /access/p3/1065.aff Nochg: 0 NUL: 0 LZMA: 29 old: 229966883 new: 187136790 LZred: 18.62% +/project3/affs/1067.aff -> /access/p3/1067.aff Nochg: 0 NUL: 0 LZMA: 256 old: 1964333232 new: 1328887875 LZred: 32.35% +/project3/affs/1066.aff -> /access/p3/1066.aff Nochg: 0 NUL: 0 LZMA: 256 old: 1736353995 new: 1434254878 LZred: 17.40% +/project3/affs/1068.aff -> /access/p3/1068.aff Nochg: 0 NUL: 0 LZMA: 65 old: 566900071 new: 474631911 LZred: 16.28% +/project3/affs/1069.aff -> /access/p3/1069.aff Nochg: 8 NUL: 0 LZMA: 243 old: 2476400779 new: 2114121820 LZred: 14.63% +/project3/affs/1070.aff -> /access/p3/1070.aff Nochg: 0 NUL: 0 LZMA: 256 old: 1742232043 new: 1327319750 LZred: 23.81% +/project3/affs/1071.aff -> /access/p3/1071.aff Nochg: 0 NUL: 0 LZMA: 252 old: 1715262776 new: 1147713553 LZred: 33.09% +/project3/affs/1074.aff -> /access/p3/1074.aff Nochg: 0 NUL: 0 LZMA: 256 old: 1895227788 new: 1339116776 LZred: 29.34% +/project3/affs/1073.aff -> /access/p3/1073.aff Nochg: 0 NUL: 0 LZMA: 7 old: 39740101 new: 36582140 LZred: 7.95% +/project3/affs/1076.aff -> /access/p3/1076.aff Nochg: 0 NUL: 0 LZMA: 256 old: 1611129344 new: 1129360597 LZred: 29.90% +/project3/affs/1075.aff -> /access/p3/1075.aff Nochg: 0 NUL: 0 LZMA: 256 old: 1879050525 new: 1365742574 LZred: 27.32% +/project3/affs/1078.aff -> /access/p3/1078.aff Nochg: 0 NUL: 0 LZMA: 256 old: 1681298906 new: 1180895987 LZred: 29.76% +/project3/affs/1077.aff -> /access/p3/1077.aff Nochg: 0 NUL: 0 LZMA: 256 old: 1718052480 new: 1280074747 LZred: 25.49% +/project3/affs/1079.aff -> /access/p3/1079.aff Nochg: 0 NUL: 0 LZMA: 242 old: 1772224996 new: 1296025871 LZred: 26.87% +/project3/affs/1080.aff -> /access/p3/1080.aff Nochg: 0 NUL: 0 LZMA: 20 old: 155572166 new: 130843212 LZred: 15.90% +/project3/affs/1081.aff -> /access/p3/1081.aff Nochg: 0 NUL: 383 LZMA: 2 old: 1257816 new: 1126140 LZred: 10.47% +/project3/affs/1082.aff -> /access/p3/1082.aff Nochg: 0 NUL: 382 LZMA: 3 old: 1335360 new: 1149536 LZred: 13.92% +/project3/affs/1083.aff -> /access/p3/1083.aff Nochg: 0 NUL: 382 LZMA: 3 old: 1335357 new: 1149434 LZred: 13.92% +/project3/affs/1084.aff -> /access/p3/1084.aff Nochg: 0 NUL: 382 LZMA: 3 old: 1263319 new: 1127151 LZred: 10.78% +/project3/affs/1085.aff -> /access/p3/1085.aff Nochg: 0 NUL: 384 LZMA: 1 old: 1241045 new: 1123821 LZred: 9.45% +/project3/affs/1086.aff -> /access/p3/1086.aff Nochg: 0 NUL: 382 LZMA: 3 old: 4777025 new: 3785114 LZred: 20.76% +/project3/affs/1088.aff -> /access/p3/1088.aff Nochg: 0 NUL: 382 LZMA: 3 old: 6604407 new: 6044546 LZred: 8.48% +/project3/affs/1094.aff -> /access/p3/1094.aff Nochg: 0 NUL: 1 LZMA: 254 old: 2102038576 new: 1737021153 LZred: 17.36% +/project3/affs/1089.aff -> /access/p3/1089.aff Nochg: 0 NUL: 382 LZMA: 3 old: 1263795 new: 1127206 LZred: 10.81% +/project3/affs/1090.aff -> /access/p3/1090.aff Nochg: 0 NUL: 382 LZMA: 3 old: 1335358 new: 1149301 LZred: 13.93% +/project3/affs/1091.aff -> /access/p3/1091.aff Nochg: 0 NUL: 0 LZMA: 190 old: 1393904043 new: 1089835700 LZred: 21.81% +/project3/affs/1092.aff -> /access/p3/1092.aff Nochg: 0 NUL: 0 LZMA: 254 old: 1908830605 new: 1573313130 LZred: 17.58% +/project3/affs/1103.aff -> /access/p3/1103.aff Nochg: 0 NUL: 0 LZMA: 32 old: 4625863 new: 660416 LZred: 85.72% +/project3/affs/1104.aff -> /access/p3/1104.aff Nochg: 0 NUL: 0 LZMA: 191 old: 1031134734 new: 926250782 LZred: 10.17% +/project3/affs/1105.aff -> /access/p3/1105.aff Nochg: 0 NUL: 13 LZMA: 241 old: 1221488719 new: 1000111715 LZred: 18.12% +/project3/affs/1106.aff -> /access/p3/1106.aff Nochg: 0 NUL: 18 LZMA: 232 old: 1718145843 new: 1342963096 LZred: 21.84% +/project3/affs/1107.aff -> /access/p3/1107.aff Nochg: 0 NUL: 0 LZMA: 385 old: 56383968 new: 5991610 LZred: 89.37% +/project3/affs/1108.aff -> /access/p3/1108.aff Nochg: 9 NUL: 61 LZMA: 1126 old: 12394337889 new: 10748084196 LZred: 13.28% +/project3/affs/1109.aff -> /access/p3/1109.aff Nochg: 0 NUL: 92 LZMA: 27 old: 183573592 new: 149546960 LZred: 18.54% +/project3/affs/1110.aff -> /access/p3/1110.aff Nochg: 0 NUL: 96 LZMA: 1 old: 73489 new: 3008 LZred: 95.91% +/project3/affs/1111.aff -> /access/p3/1111.aff Nochg: 0 NUL: 97 LZMA: 0 old: 0 new: 0 LZred: nan% +/project3/affs/1112.aff -> /access/p3/1112.aff Nochg: 0 NUL: 97 LZMA: 0 old: 0 new: 0 LZred: nan% +/project3/affs/1113.aff -> /access/p3/1113.aff Nochg: 0 NUL: 0 LZMA: 3 old: 221550 new: 8940 LZred: 95.96% +/project3/affs/1114.aff -> /access/p3/1114.aff Nochg: 0 NUL: 89 LZMA: 8 old: 393291 new: 24002 LZred: 93.90% +/project3/affs/1115.aff -> /access/p3/1115.aff Nochg: 0 NUL: 504 LZMA: 0 old: 0 new: 0 LZred: nan% +/project3/affs/1116.aff -> /access/p3/1116.aff Nochg: 0 NUL: 1224 LZMA: 0 old: 0 new: 0 LZred: nan% +/project3/affs/1117.aff -> /access/p3/1117.aff Nochg: 0 NUL: 503 LZMA: 1 old: 16615 new: 2705 LZred: 83.72% +/project3/affs/1126.aff -> /access/p3/1126.aff Nochg: 0 NUL: 183 LZMA: 1 old: 20059 new: 4227 LZred: 78.93% +/project3/affs/1101.aff -> /access/p3/1101.aff Nochg: 0 NUL: 0 LZMA: 2 old: 15445873 new: 11538573 LZred: 25.30% +/project3/affs/1130.aff -> /access/p3/1130.aff Nochg: 0 NUL: 13 LZMA: 241 old: 1221488719 new: 1000111715 LZred: 18.12% +/project3/affs/1131.aff -> /access/p3/1131.aff Nochg: 0 NUL: 0 LZMA: 31 old: 235257945 new: 176322198 LZred: 25.05% +/project3/affs/1132.aff -> /access/p3/1132.aff Nochg: 0 NUL: 0 LZMA: 2 old: 14256295 new: 10620136 LZred: 25.51% +/project3/affs/1133.aff -> /access/p3/1133.aff Nochg: 0 NUL: 0 LZMA: 3 old: 22312510 new: 15722086 LZred: 29.54% +/project3/affs/1134.aff -> /access/p3/1134.aff Nochg: 0 NUL: 13 LZMA: 36 old: 244053645 new: 174168002 LZred: 28.64% +/project3/affs/1135.aff -> /access/p3/1135.aff Nochg: 0 NUL: 0 LZMA: 129 old: 1005266800 new: 899671868 LZred: 10.50% +/project3/affs/1136.aff -> /access/p3/1136.aff Nochg: 0 NUL: 0 LZMA: 128 old: 624587394 new: 478880679 LZred: 23.33% +/project3/affs/1137.aff -> /access/p3/1137.aff Nochg: 0 NUL: 0 LZMA: 2 old: 25252254 new: 24647719 LZred: 2.39% +/project3/affs/1138.aff -> /access/p3/1138.aff Nochg: 0 NUL: 0 LZMA: 129 old: 1164485965 new: 1129738776 LZred: 2.98% +/project3/affs/1139.aff -> /access/p3/1139.aff Nochg: 0 NUL: 0 LZMA: 89 old: 469252937 new: 368167527 LZred: 21.54% +/project3/affs/1140.aff -> /access/p3/1140.aff Nochg: 0 NUL: 0 LZMA: 3 old: 221541 new: 8937 LZred: 95.97% +/project3/affs/1142.aff -> /access/p3/1142.aff Nochg: 0 NUL: 0 LZMA: 126 old: 802634925 new: 551783911 LZred: 31.25% +/project3/affs/1141.aff -> /access/p3/1141.aff Nochg: 1 NUL: 0 LZMA: 596 old: 7917067413 new: 6691526205 LZred: 15.48% +/project3/affs/1122.aff -> /access/p3/1122.aff Nochg: 0 NUL: 46 LZMA: 3 old: 1099181 new: 1038572 LZred: 5.51% +/project3/affs/1123.aff -> /access/p3/1123.aff Nochg: 0 NUL: 0 LZMA: 6 old: 15478242 new: 11843045 LZred: 23.49% +/project3/affs/1129.aff -> /access/p3/1129.aff Nochg: 0 NUL: 0 LZMA: 87 old: 58661036 new: 38781103 LZred: 33.89% +/project3/affs/1149.aff -> /access/p3/1149.aff Nochg: 0 NUL: 0 LZMA: 191 old: 1286244025 new: 1096598105 LZred: 14.74% +/project3/affs/1148.aff -> /access/p3/1148.aff Nochg: 0 NUL: 257 LZMA: 0 old: 0 new: 0 LZred: nan% +/project3/affs/1147.aff -> /access/p3/1147.aff Nochg: 0 NUL: 0 LZMA: 191 old: 1825388243 new: 1503037155 LZred: 17.66% +/project3/affs/1146.aff -> /access/p3/1146.aff Nochg: 0 NUL: 0 LZMA: 232 old: 2316708221 new: 1970644440 LZred: 14.94% +/project3/affs/1144.aff -> /access/p3/1144.aff Nochg: 0 NUL: 0 LZMA: 192 old: 1566104080 new: 1319275786 LZred: 15.76% +/project3/affs/1154.aff -> /access/p3/1154.aff Nochg: 0 NUL: 0 LZMA: 133 old: 980852201 new: 743840297 LZred: 24.16% +/project3/affs/1150.aff -> /access/p3/1150.aff Nochg: 0 NUL: 258 LZMA: 0 old: 0 new: 0 LZred: nan% +/project3/affs/1151.aff -> /access/p3/1151.aff Nochg: 0 NUL: 258 LZMA: 0 old: 0 new: 0 LZred: nan% +/project3/affs/1152.aff -> /access/p3/1152.aff Nochg: 0 NUL: 258 LZMA: 0 old: 0 new: 0 LZred: nan% +/project3/affs/1153.aff -> /access/p3/1153.aff Nochg: 0 NUL: 258 LZMA: 0 old: 0 new: 0 LZred: nan% +/project3/affs/1155.aff -> /access/p3/1155.aff Nochg: 0 NUL: 258 LZMA: 0 old: 0 new: 0 LZred: nan% +/project3/affs/1156.aff -> /access/p3/1156.aff Nochg: 0 NUL: 258 LZMA: 0 old: 0 new: 0 LZred: nan% +/project3/affs/1157.aff -> /access/p3/1157.aff Nochg: 0 NUL: 258 LZMA: 0 old: 0 new: 0 LZred: nan% +/project3/affs/1158.aff -> /access/p3/1158.aff Nochg: 0 NUL: 258 LZMA: 0 old: 0 new: 0 LZred: nan% +======================== +Only in /project3/affs/ diff --git a/tests/Makefile.am b/tests/Makefile.am new file mode 100644 index 0000000..896dadc --- /dev/null +++ b/tests/Makefile.am @@ -0,0 +1,4 @@ +TESTS = test_decryption.sh +EXTRA_DIST = makeimage.cpp speedtests.py verify.py encrypted.aff encrypted.iso test_decryption.sh + + diff --git a/tests/README.txt b/tests/README.txt new file mode 100644 index 0000000..1c9a8f7 --- /dev/null +++ b/tests/README.txt @@ -0,0 +1,7 @@ +This directory contains some additional tests which can be run to validate the AFF distribution. + +The file encrypted.aff is a standardized encrypted file that is +encrypted with the passphrase 'password'. + +NOTE: Remember to set the svn:executable property on all of the shell scripts so that + they get checked out with the execute bit set diff --git a/tests/makeimage.cpp b/tests/makeimage.cpp new file mode 100644 index 0000000..d3a5578 --- /dev/null +++ b/tests/makeimage.cpp @@ -0,0 +1,38 @@ +/* + * makeimage.cpp: + * + * Make an image with a given number of sectors. + * This file is a work of a US government employee and as such is in the Public domain. + * Simson L. Garfinkel, March 12, 2012 + */ + +#include +#include +#include +#include + +const char *progname = "makeimage"; + +void usage() +{ + errx(1,"usage: %s file blockcount\n",progname); +} + +int main(int argc,char **argv) +{ + if(argc!=3) usage(); + + int count = atoi(argv[2]); + char buf[512]; + + FILE *out = fopen(argv[1],"wb"); + if(!out) err(1,"fopen(%s)",argv[1]); + + memset(buf,' ',sizeof(buf)); + buf[511] = '\000'; + for(int i=0;i %s" % (i2,iso2)) + cmd("cmp %s %s" % (iso,iso2)) + cmd("rm -rf %s %s %s %s" % (iso,iso2,i1,i2)) + + +if(__name__=='__main__'): + runtest("aff","") + runtest("aff","-M33554432b") + runtest("afd","") + runtest("afd","-M33554432b") + runtest("afm","") + runtest("afm","-M33554432b") + diff --git a/tools/Makefile.am b/tools/Makefile.am new file mode 100644 index 0000000..09ab831 --- /dev/null +++ b/tools/Makefile.am @@ -0,0 +1,38 @@ +bin_PROGRAMS = affcat affcompare affconvert affcopy affcrypto affix affinfo affsegment \ + affstats affverify affxml affuse affrecover affsign affdiskprint + +EXTRA_DIST = test_make_random_iso.sh test_crypto.sh test_signing.sh test_recovery.sh \ + test_passphrase.sh test_afsegment.sh + +TESTS = test_signing.sh test_recovery.sh test_passphrase.sh test_afsegment.sh test_crypto.sh + +# See http://www.gnu.org/software/libtool/manual.html # Using-Automake + +AM_LDFLAGS = -static # staticly link our tools (easier debugging) + +affcat_SOURCES = affcat.cpp + +affcrypto_SOURCES = affcrypto.cpp +affcompare_SOURCES = affcompare.cpp unix4win32.h +affconvert_SOURCES = affconvert.cpp unix4win32.h +affcopy_SOURCES = affcopy.cpp unix4win32.h aff_bom.h aff_bom.cpp +affdiskprint_SOURCES = affdiskprint.cpp unix4win32.h hashextent.h +affix_SOURCES = affix.cpp unix4win32.h +affuse_SOURCES = affuse.c +affinfo_SOURCES = affinfo.cpp unix4win32.h +affrecover_SOURCES = affrecover.cpp unix4win32.h +affsegment_SOURCES = affsegment.cpp +affsign_SOURCES = affsign.cpp aff_bom.h aff_bom.cpp +affstats_SOURCES = affstats.cpp +affverify_SOURCES = affverify.cpp aff_bom.h aff_bom.cpp +affxml_SOURCES = affxml.cpp unix4win32.h + +INCLUDES = -I@top_srcdir@/lib/ +LDADD = @top_builddir@/lib/libafflib.la + +affuse_CFLAGS = @FUSE_CFLAGS@ +affuse_LDADD = @top_builddir@/lib/libafflib.la @FUSE_LIBS@ +affuse_LINK = $(CXXLINK) + +CLEANFILES = deskclerk.pem test_password.sh blank.iso archives.pem password_signed.aff signed.aff \ + random.iso agent.pem r3.iso random*.aff blank.aff blanke.aff words r3.iso evidence1.aff diff --git a/tools/aff_bom.cpp b/tools/aff_bom.cpp new file mode 100644 index 0000000..3e1bd9d --- /dev/null +++ b/tools/aff_bom.cpp @@ -0,0 +1,257 @@ +/* + * aff_bom.cpp: + * + * PUBLIC DOMAIN SOFTWARE. + * + * The software provided here is released by the Naval Postgraduate + * School (NPS), an agency of the US Department of the Navy, USA. The + * software bears no warranty, either expressed or implied. NPS does + * not assume legal liability nor responsibility for a User's use of + * the software or the results of such use. Please note that within + * the United States, copyright protection, under Section 105 of the + * United States Code, Title 17, is not available for any work of the + * United States Government and/or for any works created by United + * States Government employees. User acknowledges that this software + * contains work which was created by NPS employee(s) and is therefore + * in the public domain and not subject to copyright. + * -------------------------------------------------------------------- + * + * Change History: + * Simson L. Garfinkel - 2008 - Created + */ + + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" +#include "utils.h" +#ifdef HAVE_ERR_H +#include "err.h" +#endif + +#include "aff_bom.h" + +#ifdef HAVE_READLINE_READLINE_H +#include +#endif + +using namespace std; + +int parse_chain(const string &name) +{ + char ch; + int num; + if(sscanf(name.c_str(),AF_BOM_SEG"%c",&num,&ch)==1) return num; + return -1; +} + +int highest_chain(aff::seglist &segments) +{ + int highest_chain = -1; + for(aff::seglist::const_iterator seg = segments.begin(); seg!=segments.end() ;seg++){ + /* Are any of the segments in the input file signed? + * If so, we can't use the AFFLIB signing mechanisms, only the + * chain-of-custody mechanisms in this program. + */ + /* Are there any chain of custody segments? */ + int num= parse_chain(seg->name); + if(num>highest_chain) highest_chain = num; + } + return highest_chain; + +} + +#ifdef HAVE_OPENSSL_BIO_H +/* BIO_xmlescape: + * sends str to the bio, escaping for XML. + */ +int BIO_write_xml_escape(BIO *bio,const char *str,int parsed) +{ + while(*str){ + switch(*str){ + case '&': BIO_write(bio,"&",5);break; + case '<': BIO_write(bio,"<",4);break; + case '>': BIO_write(bio,">",4);break; + case '"': BIO_write(bio,""",6);break; + case '\'': BIO_write(bio,"'",6);break; + case '\\': + if(parsed) BIO_write(bio,str,1); + else BIO_write(bio,"\\\\",2); + break; + default: BIO_write(bio,str,1);break; + } + str++; + } + return 0; +} +#endif + +#ifdef USE_AFFSIGS +char *aff_bom::get_notes() +{ + if(isatty(fileno(stdin))){ + printf("Enter notes. Terminate input with a '.' on a line by itself:\n"); + } + if(notes) return notes; + notes = (char *)calloc(1,1); + while(notes){ + char buf2[1024]; + char *val=0; + +#ifdef HAVE_LIBREADLINE + if(isatty(fileno(stdin))){ + val = readline(""); + } +#endif + if(val==0){ + memset(buf2,0,sizeof(buf2)); + val = fgets(buf2,sizeof(buf2)-1,stdin); + if(val==0) break; + } + if(strcmp(val,".")==0) break; + notes = (char *)realloc(notes,strlen(notes)+strlen(val)+1); + strcat(notes,val); + } + printf("Thank you.\n"); + return notes; +} + +#ifdef HAVE_OPENSSL_BIO_H +int aff_bom::read_files(const char *cert_file,const char *key_file) +{ + BIO *bp_cert = BIO_new_file(cert_file,"r"); // read the certfile + PEM_read_bio_X509(bp_cert,&cert,0,0); // get an x509 cert + BIO_free(bp_cert); + if(!cert) return -1; // can't read certificate file + + /* Now read the private key */ + BIO *bp_privkey = BIO_new_file(key_file,"r"); + privkey = PEM_read_bio_PrivateKey(bp_privkey,0,0,0); + BIO_free(bp_privkey); + if(privkey==0){ + X509_free(cert); + cert = 0; + return -1; + } + + bom_open = true; + xml = BIO_new(BIO_s_mem()); // where we are writing + time_t clock = time(0); + struct tm *tm = localtime(&clock); + char timebuf[1024]; + strftime(timebuf,sizeof(timebuf),"%FT%T",tm); + + BIO_printf(xml,"<%s version=\"1\">\n",AF_XML_AFFBOM); + BIO_printf(xml," %s\n",timebuf); + BIO_printf(xml," afcopy\n"); + if(opt_note){ + BIO_printf(xml," "); + BIO_write_xml_escape(xml,get_notes(),0); + BIO_printf(xml," \n"); + } + BIO_printf(xml," \n"); + PEM_write_bio_X509(xml,cert); + BIO_printf(xml," \n"); + BIO_printf(xml," \n"); + return 0; +} + +/* Add to the Bill of Materials */ +void aff_bom::add(const char *segname,int sigmode,const u_char *seghash,size_t seghash_len) +{ + BIO_printf(xml,"<%s segname='%s' sigmode='%d' alg='sha256'>\n", + AF_XML_SEGMENT_HASH,segname,sigmode); + if(BIO_flush(xml)!=1) return; // something is wrong + BIO *b64 = BIO_new(BIO_f_base64()); + xml = BIO_push(b64,xml); + BIO_write(xml,seghash,seghash_len); + if(BIO_flush(xml)!=1) return; // another error... + xml = BIO_pop(b64); + BIO_printf(xml,"\n",AF_XML_SEGMENT_HASH); +} + +void aff_bom::close() +{ + /* Terminate the XML block*/ + BIO_printf(xml,"\n"); + BIO_printf(xml,"\n",AF_XML_AFFBOM); + + OpenSSL_add_all_digests(); + const EVP_MD *sha256 = EVP_get_digestbyname("SHA256"); + + if(sha256){ + /* now sign the XML */ + char *xbuf=0; + size_t xlen = BIO_get_mem_data(xml,&xbuf); + unsigned char sig[1024]; + u_int siglen = sizeof(sig); + + EVP_MD_CTX md; + EVP_SignInit(&md,sha256); + EVP_SignUpdate(&md,xbuf,xlen); + EVP_SignFinal(&md,sig,&siglen,privkey); + + /* Write the signature in base64 encoding... */ + BIO *b64 = BIO_new(BIO_f_base64()); + xml = BIO_push(b64,xml); + BIO_write(xml,sig,siglen); + if(BIO_flush(xml)!=1) return; // something wrong + + /* Remove the base64 bio */ + xml = BIO_pop(b64); + } + bom_open = false; +} + +int aff_bom::write(AFFILE *af,aff::seglist &segments) +{ + assert(!bom_open); + char segname[AF_MAX_NAME_LEN]; + snprintf(segname,sizeof(segname),AF_BOM_SEG,highest_chain(segments)+1); + return af_update_seg_frombio(af,segname,0,xml); +} + + +void aff_bom::make_hash(u_char seghash[32], uint32_t arg,const char *segname, + const u_char *segbuf, uint32_t segsize) +{ + OpenSSL_add_all_digests(); // probably a good idea + const EVP_MD *sha256 = EVP_get_digestbyname("SHA256"); + + if(sha256){ + unsigned int seghash_len = sizeof(seghash); + uint32_t arg_net = htonl(arg); + EVP_MD_CTX md; /* EVP message digest */ + EVP_DigestInit(&md,sha256); + EVP_DigestUpdate(&md,(const unsigned char *)segname,strlen(segname)+1); + EVP_DigestUpdate(&md,(const unsigned char *)&arg_net,sizeof(arg_net)); + EVP_DigestUpdate(&md,segbuf,segsize); + EVP_DigestFinal(&md,seghash,&seghash_len); + } +} + +int aff_bom::add(AFFILE *af,const char *segname) +{ + /* Get the segment length first */ + size_t datalen = 0; + if(af_get_seg(af,segname,0,0,&datalen)<0) return -1; + uint32_t arg; + u_char *segdata = (u_char *)malloc(datalen);/* Allocate memory */ + if(segdata<0) return -1; + if(af_get_seg(af,segname,&arg,segdata,&datalen)<0){ + free(segdata); + return -1; + } + u_char seghash[32]; + make_hash(seghash,arg,segname,segdata,datalen); + add(segname,AF_SIGNATURE_MODE0,seghash,sizeof(seghash)); + free(segdata); + return(0); + +} + + +#endif /* have_openssl_bio_h */ +#endif /* use_affsigs */ + + diff --git a/tools/aff_bom.h b/tools/aff_bom.h new file mode 100644 index 0000000..0a25066 --- /dev/null +++ b/tools/aff_bom.h @@ -0,0 +1,84 @@ +/* + * aff_bom.h + * + * PUBLIC DOMAIN SOFTWARE. + * + * The software provided here is released by the Naval Postgraduate + * School (NPS), an agency of the US Department of the Navy, USA. The + * software bears no warranty, either expressed or implied. NPS does + * not assume legal liability nor responsibility for a User's use of + * the software or the results of such use. Please note that within + * the United States, copyright protection, under Section 105 of the + * United States Code, Title 17, is not available for any work of the + * United States Government and/or for any works created by United + * States Government employees. User acknowledges that this software + * contains work which was created by NPS employee(s) and is therefore + * in the public domain and not subject to copyright. + * -------------------------------------------------------------------- + * + * Change History: + * Simson L. Garfinkel - 2008 - Created + */ + + + +#ifndef AFF_BOM_H +#define AFF_BOM_H + +#include +#include +#include +#include +#include +#include +#include +#ifdef HAVE_OPENSSL_PEM_H +#include +#include +#else +typedef void X509; +typedef void EVP_PKEY; +typedef void BIO; +#define BIO_free free +#endif + + + +class outelement { +public: + outelement(){} + AFFILE *af; // where output goes + aff::seglist segs; // list of existing segments in output +}; + typedef std::vector outlist; + +int parse_chain(const std::string &name); +int highest_chain(aff::seglist &slist); + +#ifdef USE_AFFSIGS +class aff_bom { + X509 *cert; + EVP_PKEY *privkey; + char *notes; + bool bom_open; +public: + static void make_hash(u_char seghash[32], uint32_t arg,const char *segname, + const u_char *pagebuf, uint32_t pagesize); + bool opt_note; + BIO *xml; + aff_bom(bool flag):cert(0),privkey(0),notes(0),bom_open(false),opt_note(flag),xml(0) { } + ~aff_bom(){ + assert(!bom_open); + if(notes) free(notes); + if(xml) BIO_free(xml); + } + int read_files(const char *cert_file,const char *key_file); // returns 0 if success + void add(const char *segname,int sigmode,const u_char *seghash,size_t seghash_len); // add to BOM + int add(AFFILE *af,const char *segname); // get the seg, hash it, and add it to the BOM + void close(); // close the BoM + int write(AFFILE *af,aff::seglist &segments); // write the BoM + char *get_notes(); +}; +#endif + +#endif diff --git a/tools/affcat.cpp b/tools/affcat.cpp new file mode 100644 index 0000000..cfd1618 --- /dev/null +++ b/tools/affcat.cpp @@ -0,0 +1,359 @@ +/* + * afcat.cpp: + * + * cat the contents of an AFF file... + * Distributed under the Berkeley 4-part license + */ + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" + +#include +#include +#include +#include +#ifdef HAVE_CSTRING +#include +#endif + + +using namespace std; + +vector pages; + +const char *progname = "afcat"; +int opt_info = 0; +char *opt_segname=0; +int64_t opt_pagenum = -1; +int opt_quiet = 1; +int opt_list= 0 ; +int opt_list_long = 0; +int opt_debug = 0; +int64_t opt_sector = -1; +int opt_badflag = 0; +vector opt_r; + + +void usage() +{ + printf("afcat version %s\n",PACKAGE_VERSION); + printf("usage: afcat [options] infile [... more infiles]\n"); + printf("options:\n"); + printf(" -s name --- Just output segment name\n"); + printf(" -p ### --- just output data page number ###\n"); + printf(" -S ### --- Just output data sector ### (assumes 512-byte sectors). Sector #0 is first\n"); + printf(" -q --- quiet; don't print to STDERR if a page is skipped\n"); + printf(" -n --- noisy; tell when pages are skipped.\n"); + printf(" -l --- List all of the segment names\n"); + printf(" -L --- List segment names, lengths, and args\n"); + printf(" -d --- debug. Print the page numbers to stderr as data goes to stdout\n"); + printf(" -b --- Output BADFALG for bad blocks (default is NULLs)\n"); + printf(" -v --- Just print the version number and exit.\n"); + printf(" -r offset:count --- seek to offset and output count characters in each file; may be repeated\n"); + exit(0); +} + + +const char *current_fname = 0; +int64_t current_page = -1; +void sig_info(int arg) +{ + fprintf(stderr,"afcat "); + if(current_fname) fprintf(stderr,"%s: ",current_fname); + if(current_page>=0) fprintf(stderr,"[%"PRId64"] ",current_page); + fflush(stderr); +} + + + +int compar(const void *a_,const void *b_) +{ + int64_t a = *(int *)a_; + int64_t b = *(int *)b_; + if(ab) return 1; + return 0; +} + +struct afm_private { + AFFILE *aff; // the AFFILE we use for the actual metadata + AFFILE *sr; // the AFFILE we use for the splitraw + int sr_initialized; // has the split-raw been setup from AFM? +}; + +int output_page(AFFILE *af,FILE *outfile,int64_t pagenum) +{ + current_fname = af_filename(af); + current_page = pagenum; + unsigned char *buf = (unsigned char *)malloc(af->image_pagesize); + if(buf==0){ + err(1,"malloc(%d) failed",(int)af->image_pagesize); + } + uint64_t offset = (uint64_t)pagenum * af->image_pagesize; // go to that location + + + af_seek(af,offset,SEEK_SET); + + + int bytes = af_read(af,buf,af->image_pagesize); // read what we can + + if(bytes<0){ + if(opt_debug) fprintf(stderr,"afcat: cannot read page %"I64d"\n",pagenum); + return -1; + } + + if(opt_debug){ + fprintf(stderr,"afcat: page:%"I64d" bytes: %d offset:%"I64d"\n", + pagenum, bytes,offset); + } + + /* Check each sector to see if it is badflag or not. + * If it is and if opt_badflag is not set, make it all NULs. + */ + for(unsigned char *cc=buf;ccimage_sectorsize){ + if(af_is_badsector(af,cc) && opt_badflag==0){ + memset(cc,0,af->image_sectorsize); + } + } + + if(opt_debug) fprintf(stderr," outputing %d bytes\n",bytes); + int count = fwrite(buf,1,bytes,outfile); // send to the output + if(count!=bytes) fprintf(stderr,"fwrite(buf,1,%d,outfile) only wrote %d bytes\n",bytes,count); + free(buf); + return bytes; +} + + +int afcat(AFFILE *af) +{ + int64_t total_bytes_written = 0; + + /* Read all of the pages from beginning to end and capture + * all the segment numbers... + */ + +#ifdef WIN32 + _setmode(fileno(stdout),_O_BINARY); +#endif + if(opt_debug) fprintf(stderr,"afcat(%s)\n",af_filename(af)); + + if(opt_segname){ + /* First figure out how big the segment is */ + size_t datalen = 0; + if(af_get_seg(af,opt_segname,0,0,&datalen)){ + fprintf(stderr,"%s: segment '%s' does not exist\n", + af_filename(af),opt_segname); + return -1; + } + unsigned char *data = (unsigned char *)malloc(datalen); + if(data==0) err(1,"malloc"); + if(af_get_seg(af,opt_segname,0,data,&datalen)){ + free(data); + fprintf(stderr,"%s: could not read segment '%s'\n", + af_filename(af),opt_segname); + return -1; + } + int count = fwrite(data,1,datalen,stdout); + if(count!=(ssize_t)datalen){ + fprintf(stderr,"fwrite(buf,1,%d,outfile) only wrote %d bytes\n",(int)datalen,count); + } + free(data); + return 0; + } + + if(opt_pagenum != -1){ // just write a particular page? + int r = output_page(af,stdout,opt_pagenum); + return r>=0 ? 0 : -1; + } + + if(opt_sector>=0){ + unsigned char *buf = (unsigned char *)malloc(af->image_sectorsize); + af_seek(af,(uint64_t)opt_sector*af->image_sectorsize,SEEK_SET); + int bytes_read = af_read(af,buf,af->image_sectorsize); + if(bytes_read>0){ + int bytes_written = fwrite(buf,1,bytes_read,stdout); + if(bytes_read!=bytes_written){ + fprintf(stderr,"fwrite(buf,1,%d,outfile) only wrote %d bytes\n", + bytes_read,bytes_written); + } + } + free(buf); + return 0; + } + + /* Get a list of all the segments. If we are doing a list, just print them. + * If we are not doing a list, capture the data pages and put their numbers + * into an array. + */ + + if(opt_debug) fprintf(stderr,"af_rewind_seg()\n"); + + if(opt_r.size()>0){ + unsigned char *buf = (unsigned char *)malloc(af->image_pagesize); + for(vector::const_iterator offset_count=opt_r.begin(); offset_count != opt_r.end(); offset_count++){ + string opts = *offset_count; + const char *opt = opts.c_str(); + uint64_t offset=0; + int count=0; + if(sscanf(opt,"%"I64u":%d",&offset,&count)!=2){ + err(1,"Cannot decode '%s'\n",opt); + } + af_seek(af,offset,SEEK_SET); + int r= af_read(af,buf,count); + if(r>0){ + int bytes_written = fwrite(buf,1,r,stdout); + if(bytes_written!=r) { + fprintf(stderr,"fwrite(buf,1,%d,outfile) only wrote %d bytes\n",r,bytes_written); + } + + } + } + free(buf); + return 0; + } + + + af_rewind_seg(af); // start at the beginning + char segname[AF_MAX_NAME_LEN]; + uint32_t arg; + size_t datalen = 0; + memset(segname,0,sizeof(segname)); + + int encrypted_segments = 0; + while(af_get_next_seg(af,segname,sizeof(segname),&arg,0,&datalen)==0){ + if(opt_debug) fprintf(stderr,"af_get_next_seg found segment %s\n",segname); + if(segname[0]==0) continue; // ignore sector + if(opt_list){ + printf("%s",segname); + if(opt_list_long){ + printf("\targ:%"PRIu32"\tlen:%d",arg,(int)datalen); + } + putchar('\n'); + } + else { + int64_t pagenum = af_segname_page_number(segname); + if(pagenum>=0) pages.push_back(pagenum); + if(af_is_encrypted_segment(segname)) encrypted_segments++; + } + datalen = 0; // allow to get the next one + } + if(opt_list) return 0; // that's all that was wanted. + + + sort(pages.begin(),pages.end()); + + if(pages.size()==0 && encrypted_segments){ + fprintf(stderr,"afcat: This file has %d encrypted segments.\n",encrypted_segments); + fprintf(stderr,"afcat: No unencrypted pages could be found.\n"); + } + + /* Now I have a list of pages; cat each one */ + int next_page = 0; // starting page number + int64_t imagesize = af_get_imagesize(af); + for(vector::iterator i = pages.begin(); i != pages.end(); i++){ + + int page = *i; + if(page != next_page && opt_quiet==0){ + if(page == next_page+1 ){ + fprintf(stderr,"afcat: page %d not in file\n",next_page); + } + else{ + fprintf(stderr,"afcat: pages %d through %d not in file\n", + next_page,page-1); + } + } + int r = output_page(af,stdout,page); + if(r<0) return -1; + total_bytes_written += r; + next_page = page + 1; // note what should be next + + //fprintf(stderr,"bytes written=%qd imagesize=%qd\n",total_bytes_written,imagesize); + if((total_bytes_written > imagesize) && (imagesize>0)){ + err(1,"afcat internal error. bytes written=%"I64d" imagesize=%" I64d, + (int64_t)total_bytes_written, + (int64_t)imagesize); + return -1; + } + } + return 0; +} + + +int64_t atoi64(const char *buf) +{ + int64_t r=0; + char ch; + if(sscanf(buf,"%"I64d"%c",&r,&ch)==1) return r; + fprintf(stderr,"Cannot parse '%s'\n",buf); + exit(0); +} + + +int main(int argc,char **argv) +{ + int ch; + +#ifdef SIGINFO + signal(SIGINFO,sig_info); +#endif + + while ((ch = getopt(argc, argv, "s:S:p:lLh?dqnvr:")) != -1) { + switch (ch) { + case 's': + opt_segname = optarg; + break; + case 'S': + opt_sector = atoi64(optarg); + break; + case 'p': + opt_pagenum = atoi64(optarg); + break; + case 'q': + opt_quiet = 1; + break; + case 'n': + opt_quiet = 0; + break; + case 'l': + opt_list = 1; + break; + case 'r': + opt_r.push_back(optarg); + break; + case 'L': + opt_list = 1; + opt_list_long = 1; + break; + case 'b': + opt_badflag = 1; + break; + case 'd': + opt_debug++; + break; + case 'h': + case '?': + default: + usage(); + break; + case 'v': + printf("%s version %s\n",progname,PACKAGE_VERSION); + exit(0); + } + } + argc -= optind; + argv += optind; + + if(argc<1){ + usage(); + } + + while(*argv){ + AFFILE *af = af_open(*argv,O_RDONLY,0); + if(!af) af_err(1,"afcat(%s)",*argv); + if(afcat(af)) err(1,"afcat"); + af_close(af); + argv++; + argc--; + } +} diff --git a/tools/affcompare.cpp b/tools/affcompare.cpp new file mode 100644 index 0000000..7d1bfe5 --- /dev/null +++ b/tools/affcompare.cpp @@ -0,0 +1,791 @@ +/* + * acompare.cpp: + * + * Compare the contents of an ISO file to an AFF file. + * Optionally, if they are equal, delete the ISO file + * + * Distributed under the Berkeley 4-part license + */ + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" +#include "utils.h" + +using namespace std; +using namespace aff; + + +#ifdef WIN32 +#include "unix4win32.h" +#endif + + +#ifdef UNIX +#include +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_CSTRING +#include +#endif + +#ifdef linux +#include +#endif + + + +const char *progname = "affcompare"; + +int opt_quiet = 0; +int opt_all = 0; +int opt_print_sectors = 0; +int opt_print_sector_contents = 0; +int opt_page = -1; +int opt_preen = 0; +int opt_exist = 0; +int opt_ignore_metadata = 0; +int opt_s3 = 0; +int opt_verbose = 0; +const char *batch_ext = ""; + +vector errors; + +const char *current_source = 0; + +void sig_info(int arg) +{ + if(current_source){ + printf("%s... ",current_source); + } + printf("\n"); + fflush(stdout); +} + +void print_title(char *title) +{ + if(title[0]){ + puts(title); + title[0] = 0; + } +} + +void usage() +{ + printf("affcompare version %s\n",PACKAGE_VERSION); + printf("\n"); + printf("usage: affcompare [options] file1 file2\n"); + printf(" compares file1 with file2\n"); + printf("\n"); + printf("or affcompare [options] -r dir1 dir2\n"); + printf(" comparses similarly-named files in dir1 and dir2\n"); + printf("\n"); + printf("or affcompare [options] -s file1 file2...\n"); + printf(" Reports if file was successfully copied to Amazon S3\n"); + printf(" checking only for existence, not reading back the bytes.\n"); + printf(" (Because all writes to S3 are validated by the MD5 of the object\n"); +#ifndef USE_S3 + printf(" NOTE: S3 support is not provided in this version\n"); +#endif + + printf("fast options:\n"); + printf("(These compare segments but not their contents.)\n"); + printf(" -p --- report about the results of preening\n"); + printf(" -e --- Just report about existence (use with -r)\n"); + printf(" -s --- Just see if all of the segments are present, but don't\n"); + printf(" validate the contents. (Primarily for use with Amazon S3)\n"); + + printf("other options:\n"); + printf(" -V --- just print the version number and exit\n"); + printf(" -v --- Verbose; each file as it is compared.\n"); + printf(" -q --- Quiet. No output except for errors\n"); + printf(" -a --- print what's the same (all)\n"); + printf(" -b --- print the numbers of differing sectors\n"); + printf(" -c --- print the contents of differing sectors\n"); + printf(" -m --- Just report about the data (ignore metadata)\n"); + printf(" -P ### --- Just examine the differences on page ###\n"); + printf(" -q --- Quiet; no output except for errors.\n"); + printf("\n"); + printf("Options documented above:\n"); + printf(" -r dir1 dir2 --- recursively compare what's in dir1 with dir2, and\n"); + printf(" report what's in dir1 that's not in dir2\n"); + printf(" -s --- Check to see if named files are on Amazon S3\n"); + printf("\n"); + printf(" affcompare file1.aff file2.aff --- compare file1.aff and file2.aff\n"); + printf(" affcompare f1.aff f2.aff dir1/ --- compare f1.aff with dir1/f1.aff and f2.aff with dir2/f2.aff\n"); + printf(" note: dir1/ must end with a slash.\n"); + printf(" affcompare -b img file.aff --- compare file.aff and file.img\n"); + printf(" affcompare -b img file1.aff file2.aff... --- compare file1.aff, file1.img, etc.\n"); + printf(" affcompare -re dir1 dir2 --- report AFF files in dir1 but not in dir2\n"); + printf(" affcompare -rse dir1 s3:/// --- report AFF files in dir1 but not on S3 (low bandwidth)\n"); + printf(" affcompare -rs dir1 s3:/// --- report AFF files in dir1 but incomplete on on S3 (more bandwidth)\n"); + printf("\n"); + exit(0); +} + +void print_sector(AFFILE *af,unsigned char *buf) +{ + for(unsigned int i=0;iimage_sectorsize;i++){ + if(isprint(buf[i])){ + putchar(buf[i]); + } + else { + putchar('.'); + } + if(i%64==63) putchar('\n'); + } +} + + +void print_info(char dir,const char *segname,uint32_t arg,size_t len, + unsigned char *data,int mcr) +{ + printf(" %c %s arg=%"PRIu32" len=%d\n",dir,segname,arg,(int)len); + printf(" "); + if((arg == AF_SEG_QUADWORD) && (len==8)){ + printf("data=%"I64d" as a 64-bit value\n",af_decode_q(data)); + return; + } + /* Otherwise, just print some stuff... */ + for(unsigned int i=0;i',segname,arg2,data2_len,data2,mcr); + if(mcr){ + printf(" *** Metadata segment are different "); + if(strcmp(segname,AF_BADFLAG)==0){ + printf("(bad flags should be different!)"); + } + putchar('\n'); + } + putchar('\n'); + ret = 1; + } + else { + if(opt_all){ + print_title(title); + printf(" %s (same in both) \n",segname); + } + } + free(data1); + free(data2); + return ret; +} + +int compare_aff_data_segments(char *title,AFFILE *af1,AFFILE *af2,int64_t pagenum,int mode) +{ + int ret = 0; + char pagename[65]; + snprintf(pagename,sizeof(pagename),AF_PAGE,pagenum); + + char segname[65]; + snprintf(segname,sizeof(segname),AF_SEG_D,pagenum); + + uint32_t arg1=0; + size_t data1_len=0; + int r1 = af_get_seg(af1,pagename,&arg1,0,&data1_len); + if(r1==-1) r1=af_get_seg(af1,segname,&arg1,0,&data1_len); + + uint32_t arg2=0; + size_t data2_len=0; + int r2 = af_get_seg(af2,pagename,&arg2,0,&data2_len); + if(r2 == -1) r2=af_get_seg(af2,segname,&arg2,0,&data2_len); + + if(r1<0 && r2<0) return 0; // no data segment in either file + if(r1==0 && r2!=0){ + if(mode==1){ + print_title(title); + printf(" %s \n",pagename); + } + return 1; + } + + if(r2==0 && r1!=0){ + if(mode==2){ + print_title(title); + printf(" %s \n",pagename); + } + return 1; + } + if(mode!=3) return 0; // only report differences in mode 3 + + /* Get the actual data... */ + unsigned char *data1 = (unsigned char *)malloc(af_page_size(af1)); + unsigned char *data2 = (unsigned char *)malloc(af_page_size(af2)); + + data1_len = af_page_size(af1); + data2_len = af_page_size(af2); + + uint64_t start_sector_number = (pagenum * data1_len) / af1->image_sectorsize; + + /* Find the size of each page, then get the page */ + if(af_get_page(af1,pagenum,0,&data1_len)<0) + err(1,"Cannot read page %"I64d" size from %s\n",pagenum,af_filename(af1)); + if(af_get_page(af1,pagenum,data1,&data1_len)<0) + err(1,"Cannot read page %"I64d" from %s",pagenum,af_filename(af1)); + + if(af_get_page(af2,pagenum,0,&data2_len)<0) + err(1,"Cannot read page %"I64d" size from %s\n",pagenum,af_filename(af2)); + if(af_get_page(af2,pagenum,data2,&data2_len)<0) + err(1,"Cannot read page %"I64d" from %s",pagenum,af_filename(af2)); + + if(data1_len != data2_len){ + printf("page %"I64d" size %d != size %d\n",pagenum,(int)data1_len,(int)data2_len); + return 1; + } + + /* Now look at the pages sector-by-sector. */ + int af1_bad=0; + int af2_bad=0; + int matching_bad_sectors = 0; + int matching_sectors = 0; + int total_sectors = 0; + int no_match = 0; + vector different_sectors; + + for(unsigned int offset=0;offsetimage_sectorsize){ + uint64_t this_sector = start_sector_number + offset/af1->image_sectorsize; + total_sectors++; + if(af_is_badsector(af1,data1+offset) && + af_is_badsector(af2,data2+offset)){ + matching_bad_sectors++; + continue; + } + if(af_is_badsector(af1,data1+offset)){ + af1_bad++; + continue; + } + if(af_is_badsector(af2,data2+offset)){ + af2_bad++; + continue; + } + if(memcmp(data1+offset,data2+offset,af1->image_sectorsize)==0){ + matching_sectors++; + continue; + } + no_match++; + different_sectors.push_back(this_sector); + } + + char outline[256]; + outline[0] = 0; + if(opt_all || (no_match>0) || af1_bad || af2_bad){ + snprintf(outline,sizeof(outline), + " page%"I64d" sectors:%4d matching: %3d different:%3d", + pagenum,total_sectors,matching_sectors,no_match); + } + if(af1_bad){ + snprintf(outline+strlen(outline),sizeof(outline)-strlen(outline), + " file 1 bad: %3d ",af1_bad); + } + if(af2_bad){ + snprintf(outline+strlen(outline),sizeof(outline)-strlen(outline), + " file 2 bad: %3d ",af2_bad); + } + if(matching_bad_sectors){ + if(opt_all){ + snprintf(outline+strlen(outline),sizeof(outline)-strlen(outline), + " bad both:%3d ",matching_bad_sectors); + } + } + + if(outline[0]){ + print_title(title); + puts(outline); + } + if(opt_print_sectors && different_sectors.size()>0){ + print_title(title); + printf(" Sectors with differences:"); + int i=0; + for(vector::iterator j = different_sectors.begin(); + j != different_sectors.end(); + j++){ + if(i==0){ + printf("\n "); + } + printf(" %"I64d,*j); + i = (i+1) % 10; + } + putchar('\n'); + ret = 1; + } + if(opt_print_sector_contents && different_sectors.size()>0){ + print_title(title); + printf(" Sectors with differences:"); + for(vector::iterator j = different_sectors.begin(); + j != different_sectors.end(); j++){ + int offset = (*j - start_sector_number)*af1->image_sectorsize; + char b2[16]; + printf("offset=%d\n",offset); + + memcpy(b2,data1+offset,16); + b2[15]=0; + + printf("=== sector %"I64d" (offset=%d) ===\n",*j,offset); + printf(" %s:\n",af_filename(af1)); + print_sector(af1,data1+offset); + printf("-------------------------------------\n"); + printf(" %s:\n",af_filename(af2)); + print_sector(af2,data2+offset); + printf("=====================================\n\n"); + } + ret = 1; + } + free(data1); + free(data2); + return ret; +} + +/* Compare the results of two files that were preened */ +int compare_preen(AFFILE *af1,AFFILE *af2) +{ + vector pages; + int comp_zero=0; + int comp_lzma=0; + int comp_unchanged=0; + uint64_t bytes_old = 0; + uint64_t bytes_new = 0; + + af_rewind_seg(af1); + /* Build a list of all the pages */ + char segname[AF_MAX_NAME_LEN]; + while(af_get_next_seg(af1,segname,sizeof(segname),0,0,0)==0){ + int64_t pagenumber = af_segname_page_number(segname); + if(pagenumber>=0) pages.push_back(pagenumber); + } + /* Now, compare each one */ + for(vector::const_iterator i = pages.begin(); i != pages.end(); i++){ + uint32_t arg1,arg2; + size_t len1,len2; + + if(af_get_page_raw(af1,*i,&arg1,0,&len1)){ + err(1,"Could not read page %"I64d" in file %s\n",*i,af_filename(af1)); + } + if(af_get_page_raw(af2,*i,&arg2,0,&len2)){ + err(1,"Page %"I64d" is in file %s but not in %s\n",*i,af_filename(af1), + af_filename(af2)); + } + if(arg1==arg2 && len1==len2){ + comp_unchanged++; + continue; + } + if((arg2 & AF_PAGE_COMP_ALG_MASK)==AF_PAGE_COMP_ALG_ZERO){ + comp_zero++; + continue; + } + if((arg2 & AF_PAGE_COMP_ALG_MASK)==AF_PAGE_COMP_ALG_LZMA){ + comp_lzma++; + bytes_old += len1; + bytes_new += len2; + continue; + } + } + printf("%s -> %s Nochg: %d NUL: %d LZMA: %d old: %"I64d" new: %"I64d" LZred: %6.2f%%\n", + af_filename(af1), + af_filename(af2), + comp_unchanged,comp_zero,comp_lzma,bytes_old,bytes_new,(bytes_old-bytes_new)*100.0/bytes_old); + return 0; +} + + +/* Compare two AFF files. + * Return 0 if they are equal. + */ +int compare_aff_aff(const char *file1,const char *file2) +{ + bool no_data_segments = false; + int ret = 0; + + current_source = file1; + + if(opt_all) printf("compare %s and %s:\n",file1,file2); + + AFFILE *af1 = af_open(file1,O_RDONLY,0); + if(!af1) af_err(1,"af_open(%s)",file1); + + AFFILE *af2 = af_open(file2,O_RDONLY,0); + if(!af2) af_err(1,"af_open(%s)",file2); + + af_vnode_info vni1,vni2; + + if(af_vstat(af1,&vni1) || af_vstat(af2,&vni2)){ + err(1,"af_vstat failed?"); + } + + if(af_cannot_decrypt(af1) != af_cannot_decrypt(af2)){ + printf("%s: %s decrypt\n",file1,af_cannot_decrypt(af1) ? "cannot" : "can"); + printf("%s: %s decrypt\n",file2,af_cannot_decrypt(af2) ? "cannot" : "can"); + fprintf(stderr,"affcompare must be able to decrypt both files or neither of the files.\n"); + exit(1); + } + + if(af1->image_pagesize != af2->image_pagesize){ + fprintf(stderr,"Currently, %s requires that both images have the " + "same image datsegsize.\n" + " pagesize(%s)=%"PRIu32"\n" + " pagesize(%s)=%"PRIu32"\n", + progname,file1,af1->image_pagesize, file2,af2->image_pagesize); + fprintf(stderr,"Data segments will be ignored.\n"); + no_data_segments = true; + } + + if(af1->image_sectorsize != af2->image_sectorsize){ + fprintf(stderr,"Currently, %s requires that both images have the " + "same image sectorsize.\n" + " sectorsize(%s)=%"PRIu32"\n" + " sectorsize(%s)=%"PRIu32"\n", + progname,file1,af1->image_sectorsize, file2,af2->image_sectorsize); + fprintf(stderr,"Data segments will be ignored.\n"); + no_data_segments = true; + } + + if(opt_preen){ + compare_preen(af1,af2); + af_close(af1); + af_close(af2); + return 0; + } + + if(opt_s3){ + printf("bypass\n"); + seglist list1(af1); + seglist list2(af2); + + /* Just compare the presence/absence of each segment */ + char title[1024]; + snprintf(title,sizeof(title),"\nPresent in %s but not %s:",af_filename(af1),af_filename(af2)); + for(seglist::const_iterator i=list1.begin(); i!=list1.end(); i++){ + if(find(list2.begin(),list2.end(),*i)==list2.end()){ + print_title(title); + printf(" %s\n",(*i).name.c_str()); + } + } + snprintf(title,sizeof(title),"\nPresent in %s but not %s:",af_filename(af2),af_filename(af1)); + for(seglist::const_iterator i=list2.begin(); i!=list2.end(); i++){ + if(find(list1.begin(),list1.end(),*i)==list1.end()){ + print_title(title); + printf(" %s\n",(*i).name.c_str()); + } + } + return 0; + } + + /* Compare all of the metadata segments in af1 with a2. + * Report those that are missing or different. Then report + * all of the segments in a2 but not in af1 + */ + + /* First build a list of the segments in each */ + + vector segs_with_dups; + + AFFILE *af[2] = {af1,af2}; + for(int i=0;i<2;i++){ + if(opt_verbose) printf("\n%s:\n",af_filename(af[i])); + af_rewind_seg(af[i]); + char segname[AF_MAX_NAME_LEN]; + while(af_get_next_seg(af[i],segname,sizeof(segname),0,0,0)==0){ + if(segname[0]){ + string s; + s = segname; + segs_with_dups.push_back(s); // may give duplicates + if(opt_verbose) printf(" %s\n",segname); + } + } + } + sort(segs_with_dups.begin(),segs_with_dups.end()); + vectorsegs; + + /* Make a list of segs without duplicates */ + string last; + for(vector::iterator i = segs_with_dups.begin(); + i != segs_with_dups.end(); i++){ + if(last != *i){ + segs.push_back(*i); + } + last = *i; + } + + int lowest_page = -1; + int highest_page = -1; + /* Scan for the lowest and highest numbers */ + for(vector::iterator i = segs.begin();i != segs.end(); i++){ + int64_t num = af_segname_page_number(i->c_str()); + if(num!=-1){ + if(numhighest_page||highest_page==-1) highest_page = num; + } + } + + + if(opt_page != -1){ + lowest_page = opt_page; + highest_page = opt_page; + } + + + if(opt_page == -1 + && vni1.supports_metadata + && vni2.supports_metadata + && opt_ignore_metadata==0 ){ + if(opt_all) puts("Inspecting metadata..."); + for(int mode=1;mode<=3;mode++){ + const char *title = "Metadata segments "; + char mode_title[1024]; + switch(mode){ + case 1: + snprintf(mode_title,sizeof(mode_title)," %s only in %s:\n", + title,af_filename(af1)); + break; + case 2: + snprintf(mode_title,sizeof(mode_title)," %s only in %s:\n", + title,af_filename(af2)); + break; + case 3: + snprintf(mode_title,sizeof(mode_title)," %s in both files:\n",title); + break; + } + + for(vector::iterator i = segs.begin();i != segs.end();i++){ + int64_t num = af_segname_page_number(i->c_str()); + if(num==-1){ + int r = compare_aff_metadata_segments(mode_title, af1,af2, + i->c_str(),mode); + if(r!=0) ret = r; + } + } + } + } + + if(no_data_segments==false){ + if(opt_all) puts("Inspecting data..."); + for(int mode=1;mode<=3;mode++){ + char mode_title[1024]; + switch(mode){ + case 1: snprintf(mode_title,sizeof(mode_title), + " Pages only in %s:\n", af_filename(af1));break; + case 2: snprintf(mode_title,sizeof(mode_title), + " Pages only in %s:\n", af_filename(af2));break; + case 3: snprintf(mode_title,sizeof(mode_title)," Pages in both files:\n");break; + } + + for(int i=lowest_page;i<=highest_page;i++){ + int r = compare_aff_data_segments(mode_title,af1,af2,i,mode); + if(r!=0) ret = r; + } + } + } + current_source = 0; +#ifdef HAVE_ISATTY + if(ret==0 && isatty(fileno(stdout))) printf("%s and %s: files compare okay\n",file1,file2); +#endif + return ret; +} + +int recurse(const char *dir1,const char *dir2) +{ + vector only_in_dir1; + + DIR *dirp = opendir(dir1); + struct dirent *dp; + if(!dirp) err(1,"opendir: %s",dir1); + while ((dp = readdir(dirp)) != NULL){ + + char fn1[MAXPATHLEN+1]; memset(fn1,0,sizeof(fn1)); + char fn2[MAXPATHLEN+1]; memset(fn2,0,sizeof(fn2)); + + strlcpy(fn1,dir1,sizeof(fn1)); + if(fn1[strlen(fn1)-1]!='/') strlcat(fn1,"/",sizeof(fn1)); + strlcat(fn1,dp->d_name,sizeof(fn1)); + + current_source = fn1; + if(opt_verbose) printf("%s...\n",fn1); + + switch(af_identify_file_type(fn1,1)){ + case AF_IDENTIFY_ERR: + case AF_IDENTIFY_NOEXIST: + only_in_dir1.push_back(fn1); + break; + case AF_IDENTIFY_AFF: + case AF_IDENTIFY_AFD: + case AF_IDENTIFY_AFM: + strlcpy(fn2,dir2,sizeof(fn2)); + if(fn2[strlen(fn2)-1]!='/') strlcat(fn2,"/",sizeof(fn2)); + strlcat(fn2,dp->d_name,sizeof(fn2)); + if(af_identify_file_type(fn2,1)<0){ + char buf[1024]; + snprintf(buf,sizeof(buf),"%s not in %s\n",dp->d_name,dir2); + errors.push_back(buf); + break; + } + if(opt_exist==0){ + compare_aff_aff(fn1,fn2); + } + break; + default: + break; + } + } + closedir(dirp); + printf("========================\n"); + printf("Only in %s\n",dir1); + for(vector::const_iterator i = only_in_dir1.begin(); + i != only_in_dir1.end(); + i++){ + printf("%s\n",i->c_str()); + } + return 0; +} + +int main(int argc,char **argv) +{ + int ch; + int opt_recurse=0; + +#ifdef SIGINFO + signal(SIGINFO,sig_info); +#endif + + while ((ch = getopt(argc, argv, "P:Vabcempqrsh?v")) != -1) { + switch (ch) { + case 'P': opt_page = atoi(optarg); break; + case 'V': printf("%s version %s\n",progname,PACKAGE_VERSION); exit(0); + case 'a': opt_all++; break; + case 'b': opt_print_sectors=1; break; + case 'c': opt_print_sector_contents=1; break; + case 'e': opt_exist++; break; + case 'm': opt_ignore_metadata++; break; + case 'p': opt_preen++; break; + case 'q': opt_quiet++; break; + case 'r': opt_recurse++; break; + case 's': opt_s3++;break; + case 'v': opt_verbose++;break; + case 'h': + case '?': + default: + usage(); + } + } + argc -= optind; + argv += optind; + + if(opt_recurse){ + if(argc!=2) usage(); + char *dir1 = *argv++; + char *dir2 = *argv++; + recurse(dir1,dir2); + if(errors.size()>0){ + fprintf(stderr,"================================\n"); + fprintf(stderr,"%d affcompare errors:\n",(int)errors.size()); + for(vector::const_iterator i=errors.begin(); + i!=errors.end(); + i++){ + fputs(i->c_str(),stderr); + } + exit(1); + } + exit(0); + } + + if(argc>1){ + char *last = argv[argc-1]; + if(last[strlen(last)-1]=='/'){ + while(argc>1){ + char *file1 = *argv; + char *name1 = file1; + char *cc; + + cc = strrchr(file1,'/'); + if(cc) name1 = cc+1; + + char file2[MAXPATHLEN+1]; + strlcpy(file2,last,sizeof(file2)); + strlcat(file2,name1,sizeof(file2)); + int e_code = compare_aff_aff(file1,file2); + if(e_code) exit(e_code); + argv++; + argc--; + } + } + } + + if(argc!=2) usage(); // if just 2, compare them + + char *file1 = *argv++; + char *file2 = *argv++; + + if(opt_verbose) printf("%s...\n",file1); + int e_code = compare_aff_aff(file1,file2); + exit(e_code); +} diff --git a/tools/affconvert.cpp b/tools/affconvert.cpp new file mode 100644 index 0000000..e5bd402 --- /dev/null +++ b/tools/affconvert.cpp @@ -0,0 +1,707 @@ +/* + * affconvert.cpp: + * + * Convert raw -> aff + * aff -> raw + * aff -> aff (recompressing/uncompressing) + * + * Distributed under the Berkeley 4-part license + */ + + + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" // we do enough mucking, we need the internal version +#include "utils.h" + +#include +#include + +#ifdef WIN32 +#include "unix4win32.h" +#endif + +#ifdef HAVE_CURSES_H +#include +#endif + +#ifdef HAVE_TERM_H +#include +#endif + + +#ifdef HAVE_SYS_TIME_H +#include +#endif + +#include +#include + +#ifdef HAVE_UNISTD_H +#include +#endif + +#ifdef HAVE_GETOPT_H +#include +#endif + + + +const char *progname = "affconvert"; + +int image_pagesize = 16*1024*1024; // default seg size --- 16MB +int opt_compression_alg = AF_COMPRESSION_ALG_ZLIB; +int opt_compress_level = AF_COMPRESSION_DEFAULT; +int64_t bytes_to_convert = 0; +int opt_batch = 1; +int opt_zap = 0; +int opt_quiet = 0; +int opt_write_raw = 0; // output +int opt_probe_compressed = 1; // probe for compressed files +const char *opt_write_raw_ext = "raw"; +const char *opt_outdir = 0; +const char *opt_aff_ext = "aff"; +int64_t opt_maxsize = 0; +int opt_yes = 0; +int opt_debug = 0; +std::string command_line; + + +char *append(char *base,const char *str) +{ + base = (char *)realloc(base,strlen(base)+strlen(str)+1); + strcat(base,str); // can't fail + return base; +} + + + +void usage() +{ + printf("%s version %s\n",progname,PACKAGE_VERSION); + printf("\n"); + printf("usage: %s [options] file1 [... files] \n",progname); + printf("\n"); + printf("examples:\n"); + printf(" %s file1.iso --- convert file1.iso to file1.aff\n",progname); + printf(" %s file1.iso file2.iso file3.iso... --- batch convert files\n",progname); + printf(" %s -r -e iso image.aff --- convert image.aff to image.iso\n",progname); + printf(" %s -M4g -o/media/dvd.afd bigfile.aff --- split an AFF file into 4GB chunks for archiving to DVD\n", + progname); + //printf(" %s -p image.aff --- recompress image.aff to maximum compression\n",progname); + printf("\n"); + printf("\nGeneral options:\n"); + printf(" -q -- Quiet mode. Don't ask questions, don't print status.\n"); + + printf("\nAFF output options:\n"); + printf(" -a ext -- use 'ext' for aff files (default is %s)\n",opt_aff_ext); + printf(" (use .afd for AFD files)\n"); + printf(" -Mn[kgm] -- set maximum size of output file. Suffix with g, m or k.\n"); + printf(" -sn -- set the image_pagesize (default %d)\n",image_pagesize); + printf(" -x -- don't compress AFF file.\n"); + printf(" -O dir -- use 'dir' as the output directory\n"); + printf(" -o file -- output to 'file' (can only convert one at a time)\n"); + printf(" File is AFF is file ends .aff; otherwise assumes raw.\n"); + printf(" -Xn -- Set compression to n; default is 7\n"); + printf(" -L -- Use the LZMA compression algorithm (better but slower)\n"); + + printf("\nRaw output options:\n"); + printf(" -r -- force raw output. \n"); + printf(" -e ext -- use 'ext' for the raw files (default %s)\n",opt_write_raw_ext); + printf(" (implies -r)\n"); + + printf("\nDangerous input options:\n"); + printf(" -z -- zap; delete the output file if it already exists.\n"); + printf(" -Z -- Do not automatically probe for gzip/bzip2 compression.\n"); + printf(" -y -- Always answer yes/no questions 'yes.'\n"); + printf(" -V = Just print the version number and exit.\n"); + printf("\n"); + exit(0); +} + + +/* probe_gzip(): + * Is this a gzip file? + * Right now it just looks at the file extension. + */ + +int probe_gzip(const char *infile) +{ + int len = strlen(infile); + + if(len>3 && strcmp(infile+len-3,".gz")==0){ + return 1; + } + return 0; +} + +int probe_bzip2(const char *infile) +{ + int len = strlen(infile); + + if(len>4 && strcmp(infile+len-4,".bz2")==0){ + return 1; + } + return 0; +} + +/* yesno(): + * As a yes/no question. Return 1 if yes, 0 if no. + */ + +int yesno(const char *statement,const char *question,const char *affirmative) +{ + if(opt_yes){ + if(!opt_quiet) printf("%s. %s.\n",statement,affirmative); + return 1; + } + + printf("%s. ",statement); + char buf[256]; + do { + printf("%s [y/n]: ",question); + memset(buf,0,sizeof(buf)); + if(fgets(buf,sizeof(buf)-1,stdin)==0) return 0; + if(buf[0]=='y' || buf[0]=='Y'){ + printf("%s.\n",affirmative); + return 1; + } + } while(buf[0]!='n' && buf[0]!='N'); + return 0; +} + + +/* + * Basic conversion: + * We have an input, which may be raw or aff, + * and we have an output, which may be raw or aff. + * We are going to want to read a segment at a time. + */ + + +#include +#include +#include +#include + +#ifdef HAVE_CSTRING +#include +#endif + + +using namespace std; + +/** Do the conversion. + * return 0 if success, code if fail. + */ +int convert(const char *infile,char *outfile) +{ + + if(opt_debug) fprintf(stderr,"convert(%s,%s)\n",infile,outfile); + + if(infile && outfile && strcmp(infile,outfile)==0){ + errx(1,"Can't convert a file to itself\n"); + } + + /**************************************************************** + *** Open Input + ****************************************************************/ + + AFFILE *a_in = 0; // input file, if aff + +#ifdef UNIX + /* Check to see if it is a gzip file... */ + if(opt_probe_compressed + && probe_gzip(infile) + && yesno("infile looks like a gzip file","Uncompress it","Uncompressing")){ + /* Open with a subprocess. We will need to use zlib when we move to Windows. */ + if(af_hasmeta(infile)) return -1; // don't covert with shell metacharacters + char buf[256]; + snprintf(buf,sizeof(buf),"gzcat %s",infile); + a_in = af_popen(buf,"r"); + } + + /* Check to see if it is a bzip2 file... */ + if(!a_in + && opt_probe_compressed + && probe_bzip2(infile) + && yesno("infile looks like a bzip2 file","Uncompress it","Uncompressing")){ + /* Open with a subprocess. We will need to use bzip2zlib when we move to Windows. */ + if(af_hasmeta(infile)) return -1; // don't covert with shell metacharacters + char buf[256]; + snprintf(buf,sizeof(buf),"bzcat %s",infile); + a_in = af_popen(buf,"r"); + } +#endif + + /* If the file isn't open, try to open it... */ + if(!a_in){ + a_in = af_open(infile,O_RDONLY,0); + if(!a_in) af_err(1,"%s",infile); // give up + if(af_identify(a_in)==AF_IDENTIFY_RAW){ + af_set_pagesize(a_in,image_pagesize); // match the page size we want to use + } + else { + image_pagesize = a_in->image_pagesize; // that's what we are using + } + } + + const char *ain_fn = af_filename(a_in); + struct stat si; + memset((char *)&si,0,sizeof(si)); + if(ain_fn && stat(ain_fn,&si)){ + warn("Cannot stat %s",ain_fn); + } + + + /**************************************************************** + *** Open Ouptut + ****************************************************************/ + + + if(opt_zap) unlink(outfile); // we were told to zap it + + AFFILE *a_out = 0; // output file, if aff or raw... + if(access(outfile,F_OK)==0){ + /* If outfile is a device, ask user... */ + struct stat so; + if(stat(outfile,&so)){ + err(1,"%s exists but can't be stat?",outfile); + } + if((so.st_mode & S_IFMT)==S_IFCHR || + (so.st_mode & S_IFMT)==S_IFBLK){ + char buf[1024]; + snprintf(buf,sizeof(buf),"%s is a raw device.\n",outfile); + if(yesno(buf,"Overwrite raw device?","yes")){ + goto doit; + } + } + fprintf(stderr,"%s: file exists. Delete it before converting.\n",outfile); + exit(-1); + } + /* Check for splitraw names */ + if(af_ext_is(outfile,"afm")){ + char file000[MAXPATHLEN+1]; + strlcpy(file000,outfile,sizeof(file000)); + char *cc = strrchr(file000,'.'); + if(!cc) err(1,"Cannot file '.' in %s\n",file000); + for(int i=0;i<2;i++){ + sprintf(cc,".%03d",i); + if(access(file000,F_OK)==0){ + fprintf(stderr,"%s: file exists. Delete it before converting.\n",file000); + fprintf(stderr,"NOTE: -z option will not delete %s\n",file000); + return -1; + } + } + } + + doit:; + + if(opt_write_raw){ + /* Easy way to make a raw output is to reopen an existing output file... */ + FILE *f = fopen(outfile,"w+b"); + if(!f){ + err(1,"%s",outfile); + } + a_out = af_freopen(f); + } + else { + a_out = af_open(outfile,O_RDWR|O_CREAT|O_BINARY,0777); + if(!a_out) af_err(1,"%s",outfile); + if(opt_maxsize){ + af_set_maxsize(a_out,opt_maxsize); + } + + } + if(a_out == 0) af_err(1,"af_open: %s",outfile); + + if(!opt_quiet) printf("convert %s --> %s\n",infile,outfile); + + af_update_seg(a_out,AF_ACQUISITION_COMMAND_LINE,0, + (const u_char *)command_line.c_str(), + command_line.size()); + + /**************************************************************** + *** Set up the AFF file (assuming it's an aff file) + *** stuff that we keep at the beginning of the file... + ****************************************************************/ + + MD5_CTX md5; + MD5_Init(&md5); + + SHA_CTX sha; + SHA1_Init(&sha); + + /* Setup writing */ + if(a_in->image_pagesize){ + image_pagesize = a_in->image_pagesize; + } + af_set_pagesize(a_out,image_pagesize); + af_set_sectorsize(a_out,a_in->image_sectorsize); + + struct af_vnode_info vni; + af_vstat(a_out,&vni); + if(vni.supports_compression){ + if(opt_compression_alg){ + af_enable_compression(a_out,opt_compression_alg,opt_compress_level); + } + else{ + af_enable_compression(a_out,0,0); + } + } + + /* Get a list of all the metadata segments and the pages + * (if this is a raw file, then the vnode raw driver will give us those segments) + */ + + char segname[AF_MAX_NAME_LEN]; + vector metadata_segments; + vector pages; + af_rewind_seg(a_in); // start at the beginning + int64_t highest_pagenum = 0; + while(af_get_next_seg(a_in,segname,sizeof(segname),0,0,0)==0){ + int64_t page_num = af_segname_page_number(segname); + if(page_num>=0){ + pages.push_back(page_num); + if(page_num>highest_pagenum) highest_pagenum = page_num; + } + else { + metadata_segments.push_back(segname); + } + } + + /* Copy over all of the metadata segments. + * But don't bother if we are creating raw output + */ + if(opt_write_raw==0){ + for(vector::iterator i = metadata_segments.begin(); + i != metadata_segments.end(); + i++){ + strlcpy(segname,i->c_str(),sizeof(segname)); + size_t data_len = 0; + uint32_t arg; + + /* First find out how big the segment is */ + if(af_get_seg(a_in,segname,&arg,0,&data_len)){ + warn("af_get_seg_1"); + continue; + } + /* Now get the data */ + unsigned char *data = (unsigned char *)malloc(data_len); + if(af_get_seg(a_in,segname,0,data,&data_len)){ + warn("af_get_seg_2"); + free(data); + continue; + } + /* Now put the data */ + if(af_update_seg(a_out,segname,arg,data,data_len)){ + err(1,"af_update_seg"); + } + free(data); + } + } + + /* Now sort the pages and copy them over. If there is no break, + * we can compute the hashes... + */ + sort(pages.begin(),pages.end()); + + int64_t prev_pagenum = -1; + bool hash_valid = true; + uint64_t last_byte_in_image = 0; + uint64_t total_bytes_converted = 0; + + bool copy_by_pages = af_has_pages(a_in); + + unsigned char *data = (unsigned char *)malloc(image_pagesize); + if(copy_by_pages){ + /* Copy over data one page at a time */ + for(vector::iterator i = pages.begin(); i != pages.end(); i++){ + + int64_t pagenum = *i; + + if(!opt_quiet) printf("Converting page %"I64d" of %"I64d"\r",pagenum,highest_pagenum);fflush(stdout); + + size_t data_len = image_pagesize; + if(af_get_page(a_in,pagenum,data,&data_len)){ + err(1,"af_get_page(file=%s,page=%"I64d")", + af_filename(a_in),pagenum); + } + if(af_update_page(a_out,pagenum,data,data_len)){ + err(1,"af_update_page(file=%s,page=%"I64d")", + af_filename(a_out),pagenum); + } + + if(pagenum != prev_pagenum + 1) hash_valid = false; + + if(hash_valid && vni.supports_metadata){ + MD5_Update(&md5,data,data_len); + SHA1_Update(&sha,data,data_len); + prev_pagenum = pagenum; + } + last_byte_in_image = (int64_t)image_pagesize * pagenum + (int64_t)data_len; + total_bytes_converted += data_len; + } + /* Go back and update the image size (necessary since I have been writing page-by-page) */ + if(af_update_segq(a_out,AF_IMAGESIZE,last_byte_in_image) + && errno!=ENOTSUP){ + err(1,"Could not upate AF_IMAGESIZE"); + } + } else { + /* No page support; Copy from beginning to end */ + while(!af_eof(a_in)){ + int data_len = af_read(a_in,data,image_pagesize); + if(data_len>0){ + if(!opt_quiet){ + printf("Writing to page %" I64d " with %d bytes read from input... \r", + total_bytes_converted / image_pagesize,data_len); + fflush(stdout); + } + if(af_write(a_out,data,data_len)!=data_len){ + err(1,"af_write"); + } + if(vni.supports_metadata){ + MD5_Update(&md5,data,data_len); + SHA1_Update(&sha,data,data_len); + } + } + if(data_len<0) err(1,"af_read"); + if(data_len==0){ + if(!opt_quiet) printf("af_read returned 0. Reached a sparse region or end of pipe.\n"); + break; + } + last_byte_in_image += data_len; + total_bytes_converted += data_len; + } + } + free(data); + if(!opt_quiet) printf("\n"); + + /* Write out the new hash if it is valid */ + if(hash_valid && vni.supports_metadata){ + u_char md5_buf[32],sha1_buf[40]; + char buf[256]; + MD5_Final(md5_buf,&md5); + if(af_update_seg(a_out,AF_MD5,0,md5_buf,16) && errno!=ENOTSUP){ + err(1,"Could not update AF_MD5"); + } + if(!opt_quiet) printf("md5: %s\n",af_hexbuf(buf,sizeof(buf),md5_buf,16,1)); + + SHA1_Final(sha1_buf,&sha); + if(af_update_seg(a_out,AF_SHA1,0,sha1_buf,20) && errno!=ENOTSUP){ + err(1,"Could not update AF_SHA1"); + } + if(!opt_quiet) printf("sha1: %s\n",af_hexbuf(buf,sizeof(buf),sha1_buf,20,1)); + } + + /* Finish the hash calculations and write to the db */ + if(!opt_quiet){ + printf("bytes converted: %"I64d" \n",total_bytes_converted); + /* If the vnode implementation tracked segments written, report it. */ + if(a_out->pages_written || a_out->pages_compressed){ + printf("Total pages: %"I64u" (%"I64u" compressed)\n", + a_out->pages_written,a_out->pages_compressed); + } + } + + if(vni.supports_metadata){ + /* make an AF_IMAGE_GID if it doesn't exist */ + af_make_gid(a_out); + af_set_acquisition_date(a_out,si.st_mtime); + } + + /* Make a copy of the a_out filename if we can get it */ +#ifdef HAVE_UTIMES + char *a_out_fn=0; // output filename, to remember for utimes + const char *a_ = af_filename(a_out); // remember the output filename + if(a_){ + a_out_fn = strdup(a_); // make a copy of it + } +#endif + if(af_close(a_out)) err(1,"af_close(a_out)"); + + if(!opt_quiet){ + printf("Conversion finished.\n"); + if(af_cannot_decrypt(a_in)){ + printf("*** encrypted pages are present which could not be decrypted ***\n"); + } + printf("\n\n"); + } + if(af_close(a_in)) err(1,"af_close(a_in)"); + + /* Set the utime on the resulting file if we can stat it */ + struct timeval times[2]; + + memset(times,0,sizeof(times)); + times[0].tv_sec = si.st_atime; + times[1].tv_sec = si.st_mtime; +#ifdef HAVE_UTIMES + if(a_out_fn){ + if(utimes(a_out_fn,times)) warn("utimes(%s):",outfile); + free(a_out_fn); + a_out_fn = 0; + } +#endif + return(0); +} + + +int64_t atoi64(const char *buf) +{ + int64_t r=0; + sscanf(buf,"%"I64d,&r); + return r; +} + +int64_t atoi64m(const char *optarg) +{ + int multiplier; + switch(optarg[strlen(optarg)-1]){ + case 'g': + case 'G': + multiplier=1024*1024*1024;break; + case 'm': + case 'M': + multiplier=1024*1024; break; + case 'k': + case 'K': + multiplier=1024; break; + case 'b': + case 'B': + multiplier=1;break; + default: + err(1,"Specify multiplier units of g, m, k or b in '%s'\n",optarg); + } + return atoi64(optarg) * multiplier; +} + + +int main(int argc,char **argv) +{ + char *outfile = 0; + int ch; + + command_line = aff::command_line(argc,argv); + while ((ch = getopt(argc, argv, "a:e:Lo:zqrs:xX:Zh?M:O::ydV")) != -1) { + switch (ch) { + case 'a': + opt_aff_ext = optarg; + break; + case 'e': + opt_write_raw++; + opt_write_raw_ext = optarg; + break; + case 'o': + outfile = optarg; + break; + case 'z': + opt_zap ++; + break; + case 'q': + opt_quiet++; + break; + case 'L': + opt_compression_alg = AF_COMPRESSION_ALG_LZMA; + break; + case 'r': + opt_write_raw++; + break; + case 's': + image_pagesize = atoi64m(optarg); + break; + case 'x': + opt_compression_alg=AF_COMPRESSION_ALG_NONE; + break; + case 'X': + opt_compress_level = atoi(optarg); + break; + case 'Z': + opt_probe_compressed = 0; + break; + case 'y': + opt_yes = 1; + break; + case 'M': + opt_maxsize = atoi64m(optarg); + break; + case 'O': + if(!optarg) err(1,"-O flag requires a directory"); + opt_outdir = optarg; + break; + case 'd': + opt_debug++; + break; + case 'h': + case '?': + default: + usage(); + exit(0); + case 'V': + printf("%s version %s\n",progname,PACKAGE_VERSION); + exit(0); + + } + } + argc -= optind; + argv += optind; + + if(argc<1){ + usage(); + } + + if(outfile){ + return convert(*argv,outfile); + } + + /* Check for "-o filename" at the end of the command line... */ + if(argc==3 && !strcmp(argv[1],"-o")){ + return convert(argv[0],argv[2]); + } + + /* Convert each file*/ + + while(*argv){ + char outfile[MAXPATHLEN+1]; + memset(outfile,0,sizeof(outfile)); + + const char *ext = opt_write_raw ? opt_write_raw_ext : opt_aff_ext; + char *infile = *argv; + argv++; + argc--; + + /* Copy over the filename and change the extension */ + strlcpy(outfile,infile,sizeof(outfile)); + char *cc = strrchr(outfile,'.'); // to strip off extension + if(cc){ + /* Found an extension; copy over mine. */ + strlcpy(cc+1,ext,sizeof(outfile)-(cc-outfile)); + } + else { + /* No extension; make one */ + strlcat(outfile,".",sizeof(outfile)); + strlcat(outfile,ext,sizeof(outfile)); + } + + /* The user might want us to put things + * in a different directory. Pull off the filename... + */ + if(opt_outdir){ + cc = strrchr(outfile,'/'); + char filename[PATH_MAX]; + if(cc){ + strlcpy(filename,cc+1,sizeof(filename)); // just the filename + } + else{ + strlcpy(filename,outfile,sizeof(filename)); // the outfile is the filename + } + strlcpy(outfile,opt_outdir,sizeof(outfile)); + strlcat(outfile,"/",sizeof(outfile)); + strlcat(outfile,filename,sizeof(outfile)); + } + if(convert(infile,outfile)){ + exit(1); + } + } + exit(0); +} diff --git a/tools/affcopy.cpp b/tools/affcopy.cpp new file mode 100644 index 0000000..15cce70 --- /dev/null +++ b/tools/affcopy.cpp @@ -0,0 +1,673 @@ +/* + * afcopy.cpp: + * + * Copy one AFF file to another. + * Resulting file is re-ordered and possibly re-compressed. + * Distributed under the Berkeley 4-part license + */ + + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" +#include "utils.h" +#include "base64.h" +#include "aff_bom.h" + +using namespace std; +using namespace aff; + +#ifdef HAVE_SYS_SIGNAL_H +#include +#endif + +#ifdef HAVE_TIME_H +#include +#endif + +#ifdef HAVE_SYS_TIME_H +#include +#endif + +#include +#include +#include +#include +#include + +#ifdef HAVE_UNISTD_H +#include +#endif + +#ifdef HAVE_TERM_H +#include +#endif + +#ifdef HAVE_NCURSES_TERM_H +#include +#endif + +#ifdef WIN32 +#include "unix4win32.h" +#endif + +const char *progname = "afcopy"; + +int opt_verbose = 0; +int opt_debug = 0; +int opt_x = 0; +int opt_X = AF_COMPRESSION_DEFAULT; +int opt_noverify = 0; +int opt_preen = 0; +int opt_zap =0; +int opt_missing = 0; +int opt_preen_alg_arg = 0; // algorithm for recompressing +int opt_preen_alg_flag = 0; +int opt_sign = 0; + +int opt_note = 0; +const char *opt_sign_key_file = 0; +const char *opt_sign_cert_file = 0; + +void usage() +{ + printf("%s version %s\n",progname,PACKAGE_VERSION); + printf("usage: %s [options] file1 file\n",progname); + printf(" Copies file1 to file2\n"); + printf(" %s [options] file1 file2 file3 ... dir\n",progname); + printf(" Copies file1.. into dir\n"); + printf(" %s [options] file1 file2 file3 ... dir1 dir2...\n",progname); + printf(" Copies file1.. into dirs1, dir2, ...\n"); + printf("\n"); + printf("By default, all page MACs are verified on read and all segments\n"); + printf("are verified after write.\n"); + + printf("Options:\n"); + printf(" -v = verbose: print each file as it is copied\n"); + printf(" -vv = very verbose: print each segment as it is copied\n"); + printf(" -d = print debugging information as well\n"); + printf(" -x = don't verify hashes on reads\n"); + printf(" -y = don't verify writes\n"); + printf(" -Xn = recompress pages (preen) with zlib level n\n"); + printf(" -L = recompress pages (preen) with LZMA (smaller but slower)\n"); + printf("\n"); + printf(" -h = help; print this message.\n"); + printf(" -V = print the program version and exit.\n"); + printf(" -z = zap; copy even if the destination exists.\n"); + printf(" -m = just copy the missing segments\n"); + printf("\nSignature Options:\n"); + printf(" -k filename.key = specify private key for signing\n"); + printf(" -c filename.cer = specify a X.509 certificate that matches the private key\n"); + printf(" (by default, the file is assumed to be the same one\n"); + printf(" provided with the -k option.)"); + printf(" -n = read notes to accompany the copy from standard in.\n"); + printf("\n"); + printf("\nEncryption Options:"); + printf(" Specify passphrase encryption for filename.aff with:\n"); + printf(" file://:passphrase@/filename.aff\n"); + printf("\n"); + printf("Examples:\n"); + printf(" %s file.aff file://:mypassword@/file-encrypted.aff - encrypt file.aff\n",progname); +#ifdef USE_S3 + printf(" %s -vy -X9 *.aff s3:/// Copy all files in current\n",progname); + printf(" directory to S3 default bucket with X9 compression\n"); +#endif + exit(1); +} + + +const char *current_source = 0; +const char *current_dest = 0; +const char *current_seg = 0; +void sig_info(int arg) +{ + if(current_source){ + printf("Copying %s ",current_source); + if(current_dest){ + printf("--> %s",current_dest); + if(current_seg) printf(" (%s) ",current_seg); + } + } + printf("\n"); +} + + +void unlink_outfiles(vector outfiles) +{ + int failure=0; + for(vector::const_iterator o = outfiles.begin(); + o != outfiles.end(); + o++){ + char *protocol=0; + char *path = 0; + af_parse_url(o->c_str(),&protocol,0,0,0,0,&path); + if(strcmp(protocol,"file")==0){ + unlink(path); + } + else{ + fprintf(stderr,"Cannot unlink %s\n",o->c_str()); + failure=1; + } + if(protocol) free(protocol); + if(path) free(path); + } + if(failure) exit(1); +} + +#if !defined( __BSD_VISIBLE) && !defined(isnumber) +#define isnumber(x) isdigit(x) +#endif + +#ifdef WIN32 +#include +#include +int gettimeofday (struct timeval *tv, void* tz) +{ + union { + int64_t ns100; /*time since 1 Jan 1601 in 100ns units */ + FILETIME ft; + } now; + + GetSystemTimeAsFileTime (&now.ft); + tv->tv_usec = (long) ((now.ns100 / 10LL) % 1000000LL); + tv->tv_sec = (long) ((now.ns100 - 116444736000000000LL) / 10000000LL); + return (0); +} +#endif + +void open_outfiles(AFFILE *ain,outlist &afouts,const vector &outfiles) +{ + /* Open every output file */ + for(vector::const_iterator o = outfiles.begin(); + o != outfiles.end(); o++){ + + const char *outfilename = o->c_str(); + outelement out; + + /* First see if the output file exists */ + out.af = 0; + int ident = af_identify_file_type(outfilename,1); + if(ident!=AF_IDENTIFY_NOEXIST){ + fprintf(stderr,"%s: file exists... ",outfilename); + if(opt_zap==0 && opt_missing==0){ + fprintf(stderr,"\n Will not overwrite; use -m or -z\n"); + continue; + } + if(opt_missing){ + fprintf(stderr,"Will fill in missing segments...\n"); + out.af = af_open(outfilename,O_RDWR|O_EXCL,0666); + if(!out.af) af_err(1,outfilename); + if(af_page_size(ain) != af_page_size(out.af)){ + fprintf(stderr,"%s and %s have different page sizes (%d != %d)\n", + af_filename(ain), + af_filename(out.af), + af_page_size(ain), + af_page_size(out.af)); + af_close(out.af); + out.af=0; + continue; + } + } + } + + + + if(out.af==0){ + out.af = af_open(outfilename,O_RDWR|O_EXCL|O_CREAT,0666); + if(!out.af){ + warn("%s",outfilename); + continue; + } + if(af_set_pagesize(out.af,af_page_size(ain))){ + errx(1,"%s: cannot set page size to %d\n", af_filename(out.af),af_page_size(ain)); + } + } + if(o != outfiles.begin()) printf("\t "); + if(opt_verbose){ + printf(" => %s ",outfilename); + if(opt_preen) printf(" (preening) "); + printf("\n"); + } + if(opt_missing) out.segs.get_seglist(out.af); + afouts.push_back(out); + } +} + +/* Copy pagenumber from ain to aout. + * Return 0 if success, -1 if can't do it. + * Properly handles signing and preening if requested. + */ +int copy_page(AFFILE *ain,AFFILE *aout,int64_t pagenum,uint32_t arg,u_char *seghash,u_int *seghash_len) +{ + /* If we are preening but not signing, see if we can get out fast */ + if(opt_sign==0 && opt_preen==0) return -1; // not preening and not signing + + /* If we are not signing, don't bother decompressing and recompressing*/ + if(opt_sign==0 && opt_preen){ + int alg = (arg & AF_PAGE_COMP_ALG_MASK); + if(alg==AF_PAGE_COMP_ALG_ZERO) return -1; // don't preen ZERO + if(alg==opt_preen_alg_arg) return -1; // don't decompress then re-compress with old alg + } + + /* If we get here, page must be read into memory and decompressed */ + size_t pagesize = af_page_size(ain); + if(pagesize<=0) return -1; // couldn't get pagesize + + u_char *pagebuf = (unsigned char *)malloc(pagesize); + if(!pagebuf) return -1; // couldn't allocate memory for page? + + if(af_get_page(ain,pagenum,pagebuf,&pagesize)){ // note --- this may make pagesize smaller + free(pagebuf); + return -1; + } + + if(opt_preen){ // set compression if we are preening + af_enable_compression(aout,opt_preen_alg_flag,opt_X); + } + +#ifdef USE_AFFSIGS + /* If calculating a bom, calculate the bom! */ + if(opt_sign){ + char segname[AF_MAX_NAME_LEN]; + sprintf(segname,AF_PAGE,pagenum); + aff_bom::make_hash(seghash,arg,segname,pagebuf,pagesize); + } +#endif + + /* Write out the page */ + int ret = af_update_page(aout,pagenum,pagebuf,pagesize); + free(pagebuf); + return ret; +} + +string base64(const u_char *buf,size_t buflen) +{ + size_t len = buflen*2+1; + char *str = (char *)malloc(len); + b64_ntop(buf,buflen,str,len); + string ret = string(str); + free(str); + return ret; +} + +#ifndef HAVE_ISATTY +int isatty(int fd) +{ + return 1; // have to assume it's a tty +} +#endif + +int afcopy(char *infile,vector &outfiles) +{ +#ifdef SIGINFO + signal(SIGINFO,sig_info); +#endif + hashMapT hashMap; + + /* Open the input file */ + AFFILE *ain = af_open(infile,O_RDONLY,0); + if(opt_debug) printf("af_open(%s,O_RDONLY)=%p\n",infile,ain); + if(!ain) af_err(1,"%s",infile); + seglist segments(ain); + + if(opt_zap) unlink_outfiles(outfiles); + + outlist afouts; // vector of output AFFs + vectorpreened_pages; + open_outfiles(ain,afouts,outfiles); + + /* Now, try to open the output files, to see if they exist */ + current_source = infile; + if(opt_verbose) printf("%s: ",infile); + if(opt_verbose>1) putchar('\n'); + + /* If we couldn't open any output files, return */ + if(afouts.size()==0){ + af_close(ain); // close the input file + return -1; + } + +#ifdef USE_AFFSIGS + /* If we are signing, initialize the signing machinery */ + aff_bom bom(opt_note); + if(opt_sign){ + if(bom.read_files(opt_sign_cert_file,opt_sign_key_file)){ + opt_sign = 0; // can't sign + } + } +#endif + + /* Now the files are open. For each output file: + * 1. Initialize signing if options were set and the segments aren't already signed. + * 2. Sign all of the segments that are unsigned + */ + for(outlist::iterator aout = afouts.begin(); aout != afouts.end(); aout++){ + if(opt_sign_key_file && segments.has_signed_segments()==false){ + if(af_set_sign_files(aout->af,opt_sign_key_file,opt_sign_cert_file)){ + err(1,"%s",opt_sign_key_file); + } + af_sign_all_unsigned_segments(aout->af); + opt_sign = true; + } + } + + /* Start the copying */ + struct timeval t0,t1; + gettimeofday(&t0,0); + for(seglist::const_iterator seg = segments.begin(); seg!= segments.end();seg++){ + /* For each segment, get the size of the segment */ + const char *segname = seg->name.c_str(); + current_seg = segname; // for printing + size_t seglen=0; + + if(af_get_seg(ain,segname,0,0,&seglen)){ + unlink_outfiles(outfiles); + err(1,"Cannot read length of segment '%s' on input file %s", segname,af_filename(ain)); + } + unsigned char *segbuf = (unsigned char *)malloc(seglen); + if(!segbuf){ + unlink_outfiles(outfiles); + err(1,"Cannot allocated %d bytes for segment '%s' in %s", + (int)seglen,segname,af_filename(ain)); + } + + /* Now get the raw source segment */ + uint32_t arg=0; + if(af_get_seg(ain,segname,&arg,segbuf,&seglen)){ + unlink_outfiles(outfiles); // failure; unlink the output files + err(1,"Cannot read segment '%s' in %s. Deleteing output file", segname,af_filename(ain)); + } + + /* Calculate the MD5 of this segment and remember it in the map */ + md5blob md5; + MD5(segbuf,seglen,md5.buf); + hashMap[segname] = md5; + + /* See if this is a page; if so, it is handled specially */ + int64_t pagenumber = af_segname_page_number(segname); + + /* Write the segment to each file */ + for(outlist::iterator aout = afouts.begin(); aout != afouts.end(); aout++){ + current_dest = af_filename(aout->af); + if(opt_verbose>1 || opt_debug){ + if(aout != afouts.begin()) printf("\n "); + printf(" %s -> %s:%s ...", segname,af_filename(aout->af),segname); + } + + /** COPY THE DATA **/ + + u_char seghash[32]; /* resultant message digest; could be any size */ + unsigned int seghash_len = sizeof(seghash); /* big enough to hold SHA256 */ + int sigmode = AF_SIGNATURE_MODE0; + + memset(seghash,0,sizeof(seghash)); + + bool copied = false; + + /* If we are preening, signing, or building a ToC, we need to copy the raw page */ + if(pagenumber>=0 && (opt_preen || opt_sign_key_file)){ + if(copy_page(ain,aout->af,pagenumber,arg,seghash,&seghash_len)==0){ + preened_pages.push_back(pagenumber); // preened pages won't be verified by md5 + if(opt_debug && opt_preen) printf(" (PREENED) "); + sigmode = AF_SIGNATURE_MODE1; + copied = true; + } + } + + /* Copy the page if it is not in the destination */ + if(copied==false){ + if(!aout->segs.contains(segname)){ + if(af_update_seg(aout->af,segname,arg,segbuf,seglen)){ + unlink_outfiles(outfiles); + err(1,"Cannot write segment '%s' to %s.", segname,af_filename(aout->af)); + } + +#ifdef USE_AFFSIGS + if(opt_sign){ + aff_bom::make_hash(seghash,arg,segname,segbuf,seglen); + } +#endif + } + else{ + if(opt_verbose>1 || opt_debug) printf(" [already in %s] ",af_filename(aout->af)); + } + } +#ifdef USE_AFFSIGS + if(opt_sign) bom.add(segname,sigmode,seghash,seghash_len); +#endif + } + free(segbuf); + current_dest = 0; + if(opt_verbose>1 || opt_debug) putchar('\n'); + } + current_seg = 0; + +#ifdef USE_AFFSIGS + /* For each open file, make an AF_IMAGE_GID if one doesn't exist */ + for(outlist::iterator aout = afouts.begin(); aout != afouts.end(); aout++){ + if(af_make_gid(aout->af)>0){ + if(opt_sign){ + af_sign_seg(aout->af,AF_IMAGE_GID); // make sure the GID is signed + bom.add(aout->af,AF_IMAGE_GID); + bom.add(aout->af,AF_IMAGE_GID AF_SIG256_SUFFIX); + + } + } + } + + if(opt_sign){ + bom.close(); + /* Now write to each of the output files */ + for(outlist::iterator aout = afouts.begin(); aout != afouts.end(); aout++){ + bom.write(aout->af,segments); + } + } +#endif + + gettimeofday(&t1,0); + if(afouts.size()==1){ + AFFILE *af = afouts.begin()->af; + uint64_t w = af->bytes_written; + double sec = ((t1.tv_sec-t0.tv_sec)+(t1.tv_usec-t0.tv_usec)/1000000.0); + printf("%s: %"I64d" bytes transfered in %.2f seconds. xfer rate: %.2f MBytes/sec\n", + af_filename(af),w,sec,(w/1000000.0) / sec); + } + + if(opt_noverify==0){ + current_seg = "VERIFYING"; + /* Now verify all of the hashes */ + if(opt_verbose || opt_debug) printf("\n\nFiles copied. Verifying...\n"); + for(seglist::const_iterator seg = segments.begin(); seg!= segments.end();seg++){ + + const char *segname = seg->name.c_str(); + for(outlist::iterator aout = afouts.begin(); aout != afouts.end(); aout++){ + size_t seglen=0; + char b2[1024]; + + if((aout->af)->v->flag & AF_VNODE_TYPE_RELIABLE){ + continue; // no need to verify a reliable write + } + if(opt_verbose>1 || opt_debug) printf(" verifying %s...\n",segname); + + again: + if(af_get_seg(aout->af,segname,0,0,&seglen)){ + if(segname != b2 && + segname[0]=='s' && segname[1]=='e' && segname[2]=='g' && + isnumber(segname[3])){ + /* Looks like a legacy segname name was renamed. + * Try the new name + */ + snprintf(b2,sizeof(b2),"page%s",segname+3); + if(opt_verbose) printf(" Couldn't read %s; looking for %s\n", + segname,b2); + segname = b2; + goto again; + } + unlink_outfiles(outfiles); + errx(1,"Cannot read length of segment '%s' in output file %s", + segname,af_filename(aout->af)); + } + int64_t pagenumber = af_segname_page_number(segname); + if(find(preened_pages.begin(),preened_pages.end(),pagenumber) !=preened_pages.end()){ + /* TK: page pagenumber was preened. + * It should probably be checked against the original hash... + */ + continue; + } + + unsigned char *segbuf = (unsigned char *)malloc(seglen); + if(!segbuf){ + err(1,"Cannot allocated %d bytes for segment '%s' in %s", + (int)seglen,segname,af_filename(ain)); + } + uint32_t arg; + if(af_get_seg(aout->af,segname,&arg,segbuf,&seglen)){ + err(1,"Cannot read segment '%s' in %s", + segname,af_filename(aout->af)); + } + + /* Calculate the MD5 of this segment and see if it matches the map. + * (But don't do this for preened segments. + */ + unsigned char md5_read[16]; + MD5(segbuf,seglen,md5_read); + if(memcmp(hashMap[segname].buf,md5_read,16)!=0){ + unlink_outfiles(outfiles); + errx(1,"Hash read from %s for segment %s doesn't validate.", + af_filename(aout->af),segname); + } + free(segbuf); // free the buffer + } + } + } + + /* Finally, close the output files*/ + for(outlist::iterator aout = afouts.begin(); aout != afouts.end(); aout++){ + af_close(aout->af); + } + af_close(ain); + if(opt_verbose>1 || opt_debug) printf("==============================\n"); + current_source = 0; + return 0; +} + +int main(int argc,char **argv) +{ + int ch; + + setvbuf(stdout,0,_IONBF,0); // turn off buffering on stdout + while ((ch = getopt(argc, argv, "vdVxyh?zmX:Lp:P:k:c:n")) != -1) { + switch (ch) { + case 'v': opt_verbose++; break; + case 'd': opt_debug++; break; + case 'X': + opt_preen =1; + opt_X = optarg[0] - '0'; + opt_preen_alg_arg = AF_PAGE_COMP_ALG_ZLIB; + opt_preen_alg_flag = AF_COMPRESSION_ALG_ZLIB; + if(opt_X<0 || opt_X>9) opt_X = AF_COMPRESSION_DEFAULT; + break; + case 'L': + opt_preen=1; + opt_preen_alg_arg = AF_PAGE_COMP_ALG_LZMA; + opt_preen_alg_flag = AF_COMPRESSION_ALG_LZMA; + break; + case 'x': opt_x++;break; + case 'y': opt_noverify++;break; + case 'z': opt_zap++;break; + case 'm': opt_missing++;break; + case 'n': opt_note++;break; + case 'k': + if(access(optarg,R_OK)) err(1,"%s",optarg); + opt_sign_key_file = optarg; + if(opt_sign_cert_file==0) opt_sign_cert_file=optarg; + opt_sign = true; + break; + case 'c': + if(access(optarg,R_OK)) err(1,"%s",optarg); + opt_sign_cert_file = optarg; + break; + case 'V': + printf("%s version %s\n",progname,PACKAGE_VERSION); + exit(0); + case 'h': + case '?': + default: + usage(); + break; + } + } + argc -= optind; + argv += optind; + + if(argc<2){ // at this point, we need at least two args + usage(); + } + + /* We either need both a key file and a cert file, or neither */ + if((opt_sign_key_file==0) != (opt_sign_cert_file==0)){ + errx(1,"Both a private key and a certificate must be specified."); + } + + /* Find any directories */ + vector dirlist; + for(int i=argc-1;i>0;i--){ + struct stat st; + + // s3 names that do not end with ".aff" are directories + const char *last4 = strlen(argv[i])>4 ? argv[i]+strlen(argv[i])-4 : ""; + if(strncmp(argv[i],"s3://",5)==0 && + strcmp(last4,".aff")!=0){ + dirlist.push_back(argv[i]); + argc--; + continue; + } + + if(stat(argv[i],&st)!=0) break; // out of directories + if((st.st_mode & S_IFMT)!=S_IFDIR) break; // found a non-dir + dirlist.push_back(argv[i]); + argc--; // ignore the last + } + + /* If I found no directories, then there better just be two values */ + if(dirlist.size()==0){ + if(argc!=2){ + fprintf(stderr,"Please specify a directory or just two AFF files.\n\n"); + usage(); + } + /* Must be copying from file1 to file2. Make sure file2 does not exist */ + if(access(argv[1],R_OK)==0){ + fprintf(stderr,"File exists: %s\n",argv[1]); + if(!opt_zap) exit(1); + } + + vector outfiles; + outfiles.push_back(argv[1]); + return afcopy(argv[0],outfiles); + } + + /* Loop for each file and each directory */ + + while(argc--){ + /* Open the output files */ + vector outfiles; + for(u_int i=0;i +#include +#include + +const char *progname = "afcrypto"; +#define DEFAULT_PASSPHRASE_FILE ".affpassphrase" +int opt_debug = 0; +int opt_verbose = 0; +int opt_just_print_encrypted_count = 0; +int opt_just_print_unencrypted_count = 0; +char *opt_unsealing_private_key_file= 0; +int opt_xml = 0; + +void change_passphrase(const char *fn,const char *old_passphrase,const char *new_passphrase) +{ + int fail = 0; + + AFFILE *af = af_open(fn,O_RDWR,0666); + if(!af) af_err(1,fn); + if(af_change_aes_passphrase(af,old_passphrase,new_passphrase)){ + warnx("%s: af_change_aes_passphrase failed",fn); + fail = 1; + } + af_close(af); + if(!fail) printf("%s: passphrase changed.\n",fn); +} + +void get_and_change_passphrase(const char *fn) +{ + char old_passphrase[1024]; + char new_passphrase[1024]; + + memset(old_passphrase,0,sizeof(old_passphrase)); + memset(new_passphrase,0,sizeof(new_passphrase)); + + printf("Enter old passphrase: "); + if(fgets(old_passphrase,sizeof(old_passphrase),stdin)==0) return; + char *cc = strchr(old_passphrase,'\n');if(cc) *cc='\000'; + + /* See if this passphrase works*/ + + AFFILE *af = af_open(fn,O_RDONLY,0666); + if(!af) af_err(1,fn); + if(af_use_aes_passphrase(af,old_passphrase)){ + errx(1,"passphrase incorrect"); + } + af_close(af); + + printf("Enter new passphrase: "); + if(fgets(new_passphrase,sizeof(new_passphrase),stdin)==0) return; + cc = strchr(new_passphrase,'\n');if(cc) *cc='\000'; + change_passphrase(fn,old_passphrase,new_passphrase); +} + +void usage() +{ + printf("afcrypto version %s\n",PACKAGE_VERSION); + printf("usage: afcrypto [options] filename.aff [filename2.aff ... ]\n"); + printf(" prints if each file is encrypted or not.\n"); + printf("options:\n"); + printf(" -x --- output in XML\n"); + printf(" -j --- Just print the number of encrypted segments\n"); + printf(" -J --- Just print the number of unencrypted segments\n"); + + printf("\nData conversion options:\n"); + printf(" -e --- encrypt the unencrypted non-signature segments\n"); + printf(" -d --- decrypt the encrypted non-signature segments\n"); + printf(" -r --- change passphrase (take old and new from stdin)\n"); + printf(" -O old --- specify old passphrase\n"); + printf(" -N new --- specify new passphrase\n"); + printf(" -K mykey.key -- specifies a private keyfile for unsealing (may not be repeated)\n"); + printf(" -C mycert.crt -- specifies a certificate file for sealing (may be repeated)\n"); + printf(" -S --- add symmetric encryptiong (passphrase) to AFFILE encrypted with public key\n"); + printf(" (requires a private key and a specified passphrase).\n"); + printf(" -A --- add asymmetric encryption to a AFFILE encrypted with a passphrase\n"); + printf(" (requires a certificate file spcified with the -C option\n"); + + + printf("\nPassword Cracking Options:\n"); + printf(" -p passphrase --- checks to see if passphrase is the passphrase of the file\n"); + printf(" exit code is 0 if it is, -1 if it is not\n"); + printf(" -k --- attempt to crack passwords by reading a list of passwords from ~/.affpassphrase\n"); + printf(" -f file --- Crack passwords but read them from file.\n"); + + printf("\nDebugging:\n"); + printf(" -V --- Just print the version number and exit.\n"); + printf(" -D --- debug; print out each key as it is tried\n"); + printf(" -l --- List the installed hash and encryption algorithms \n"); + printf("Note: This program ignores the environment variables:\n"); + puts(AFFLIB_PASSPHRASE); + puts(AFFLIB_PASSPHRASE_FILE); + puts(AFFLIB_PASSPHRASE_FD); + puts(AFFLIB_DECRYPTING_PRIVATE_KEYFILE); + exit(0); +} + +/* Try each of the passphrases in the file against the passphrase. If it is found, return it. */ +char *check_file(AFFILE *af,const char *passphrase_file) +{ + char *ret = 0; + FILE *f = fopen(passphrase_file,"r"); + if(!f) return 0; + + char buf[1024]; + memset(buf,0,sizeof(buf)); + while(fgets(buf,sizeof(buf)-1,f)){ + char *cc = strchr(buf,'\n'); + if(cc) *cc = 0; + if(opt_debug){ + if(opt_debug) printf("checking with '%s' ... ",buf); + fflush(stdout); + } + int r= af_use_aes_passphrase(af,buf); + if(r==0){ + if(opt_debug) printf("YES!\n"); + ret = strdup(buf); + break; + } + } + fclose(f); + return ret; +} + +/** + * This will eventually decrypt non-signature segments that are + * encrypted + * + * @param af - the AFFILE to open + * @param count - The number of pages actually encrypted + */ +int af_decrypt_encrypted_segments(AFFILE *af, int *count, int mode) +{ + af_set_option(af,AF_OPTION_AUTO_ENCRYPT,0); + af_set_option(af,AF_OPTION_AUTO_DECRYPT,0); // turn off auto decryption + aff::seglist sl(af); // get the list of the segments + af_set_option(af,AF_OPTION_AUTO_DECRYPT,1); // turn auto decryption back on + for(aff::seglist::const_iterator si = sl.begin();si!=sl.end();si++){ + if(opt_debug) printf(" checking segment %s",si->name.c_str()); + if(af_is_encrypted_segment(si->name.c_str())){ + + if(mode == O_RDONLY){ // if readonly, just tally + (*count) ++; + if(opt_debug) printf(" would decrypt segment\n"); + continue; + } + + /* Generate the name of the unencrypted segment */ + char segname[AF_MAX_NAME_LEN]; + strcpy(segname,si->name.c_str()); + char *cc = strstr(segname,AF_AES256_SUFFIX); + if(!cc){ + if(opt_debug) printf(" will not decrypt AFFKEY segments; will be deleted later.\n"); + continue; // something is wrong; can't find the /aes256 + } + *cc = '\000'; // truncate off the /aes256 + + /* Get the segment and put it, which will force the decryption to take place */ + if(opt_debug) printf(" decrypting segment\n"); + u_char *buf = (u_char *)malloc(si->len); + if(!buf) warn("malloc(%zd) failed", si->len); + else { + uint32_t arg; + size_t datalen = si->len; + if(af_get_seg(af,segname,&arg,buf,&datalen)){ + warn("Could not read segment '%s'",segname); + } + else{ + /* si->datalen >= datalen. + * si->datalen is the length of the encrypted segment. + * datalen is the length of the decrypted segment. + */ + assert(si->len >= datalen); + assert(si->arg==arg); + if(af_update_seg(af,segname,arg,buf,datalen)){ + warn("Could not decrypt segment '%s'",si->name.c_str()); + } else { + (*count) ++; + } + } + free(buf); + } + } else { + if(opt_debug) printf(" not encrypted\n"); + } + } + /* Delete the AF_AFFKEY segment */ + if(af_get_seg(af,AF_AFFKEY,0,0,0)==0) af_del_seg(af,AF_AFFKEY); + /* Delete all of the EVP segments */ + for(int i=0;;i++){ + char segname[1024]; + snprintf(segname,sizeof(segname),AF_AFFKEY_EVP,i); + if(af_get_seg(af,segname,0,0,0)!=0) break; // found the last segment + if(af_del_seg(af,segname)) warn("Cannot delete segment %s",segname); + } + return 0; +} + + +/** + * Encrypts the non-signature segments that are not encrypted. + * There is no reason to encrypt the signature segments. + * + * @param af - the AFFILE to open + * @param count - The number of pages actually encrypted + */ + +int af_encrypt_unencrypted_nonsignature_segments(AFFILE *af,int *count,int mode) +{ + af_set_option(af,AF_OPTION_AUTO_DECRYPT,0); // do not automatically decrypt + aff::seglist sl(af); + for(aff::seglist::const_iterator si = sl.begin();si!=sl.end();si++){ + if(si->name == AF_AFFKEY) continue; // don't encrypt the affkey! + if(strstr(si->name.c_str(),"affkey_evp")) continue; + if(!af_is_encrypted_segment(si->name.c_str()) && + !af_is_signature_segment(si->name.c_str())){ + + if(mode == O_RDONLY){ // if readonly, just tally + (*count) ++; + continue; + } + + /* Get the segment and put it, which will force the encryption to take place */ + if(opt_debug) printf(" encrypting segment %s\n",si->name.c_str()); + u_char *buf = (u_char *)malloc(si->len); + if(!buf) warn("Cannot encrypt segment '%s' --- too large (%zd bytes) --- malloc failed", + si->name.c_str(),si->len); + else { + uint32_t arg; + size_t datalen = si->len; + if(af_get_seg(af,si->name.c_str(),&arg,buf,&datalen)){ + warn("Could not read segment '%s'",si->name.c_str()); + } + else{ + /* make sure that what we read is what we thought we were going to read */ + assert(si->len==datalen); + assert(si->arg==arg); + if(af_update_seg(af,si->name.c_str(),arg,buf,datalen)){ + warn("Could not encrypt segment '%s'",si->name.c_str()); + } else { + (*count) ++; + } + } + free(buf); + } + } else { + if(opt_debug) printf(" already encrypted or signed: %s\n",si->name.c_str()); + } + } + af_set_option(af,AF_OPTION_AUTO_DECRYPT,1); // go back to automatically decrypting + return 0; +} + +void list_openssl_hashes() +{ + const char *digests[] = {"md5","sha1","sha256",0}; + OpenSSL_add_all_algorithms(); + for(int i=0;digests[i];i++){ + printf("OpenSSL has %s: %s\n",digests[i],EVP_get_digestbyname(digests[i]) ? "YES" : "NO"); + } + exit(0); +} + +int main(int argc,char **argv) +{ + int ch; + const char *old_passphrase=0; + const char *new_passphrase=0; + const char *check_passphrase = 0; + char *passphrase_file = 0; + const char *progname = argv[0]; + int opt_encrypt = 0; + int opt_decrypt = 0; + int opt_add_passphrase_to_public_key = 0; + int opt_add_public_key_to_passphrase = 0; + + int mode = O_RDONLY; // mode for opening AFF file + const char **certificates = (const char **)malloc(0); + int num_certificates = 0; + const char *envvars[] = {AFFLIB_PASSPHRASE,AFFLIB_PASSPHRASE_FILE,AFFLIB_PASSPHRASE_FD, + AFFLIB_DECRYPTING_PRIVATE_KEYFILE,0}; + for(int i=0;envvars[i];i++){ + /* Don't use auto-supplied passphrases */ +#ifdef HAVE_UNSETENV + unsetenv(envvars[i]); +#else + if(getenv(envvars[i])){ + fprintf(stderr,"Please unset %s and restart\n",envvars[i]); + exit(1); + } +#endif + } + + int opt_change = 0; + const char *home = getenv("HOME"); + + while ((ch = getopt(argc, argv, "zreC:SAO:N:p:f:kdDh?VK:vljJx:")) != -1) { + switch (ch) { + case 'x': opt_xml = 1; break; + case 'j': opt_just_print_encrypted_count =1;break; + case 'J': opt_just_print_unencrypted_count =1;break; + + /* These options make the mode read-write */ + case 'r': opt_change = 1; mode = O_RDWR; break; + case 'e': opt_encrypt = 1; mode = O_RDWR; break; + case 'd': opt_decrypt = 1; mode = O_RDWR; break; + case 'S': opt_add_passphrase_to_public_key = 1; mode = O_RDWR; break; + case 'A': opt_add_public_key_to_passphrase = 1; mode = O_RDWR; break; + /* These just set up variables */ + case 'C': + certificates = (const char **)realloc(certificates,sizeof(int *)*(num_certificates+1)); + certificates[num_certificates] = optarg; + num_certificates++; + break; + case 'K': opt_unsealing_private_key_file = optarg;break; + case 'O': old_passphrase = optarg;break; + case 'N': new_passphrase = optarg;break; + case 'p': check_passphrase = optarg;break; + case 'f': passphrase_file = optarg;break; + case 'k': + if(!home) home = "/"; + passphrase_file = (char *)malloc(strlen(home)+strlen(DEFAULT_PASSPHRASE_FILE)+2); + strcpy(passphrase_file,home); + strcat(passphrase_file,"/"); + strcat(passphrase_file,DEFAULT_PASSPHRASE_FILE); + break; + case 'D': opt_debug = 1;break; + case 'v': opt_verbose = 1;break; + case 'l': list_openssl_hashes(); exit(0); + case 'h': + case '?': + default: + usage(); + break; + case 'V': + printf("%s version %s\n",progname,PACKAGE_VERSION); + exit(0); + } + } + argc -= optind; + argv += optind; + if(argc<1){ + fprintf(stderr,"No image file specified\n"); + usage(); + } + + if(opt_just_print_encrypted_count && opt_just_print_unencrypted_count){ + errx(1,"Options -j and -J conflict\n"); + } + + if(num_certificates>0 && (opt_encrypt==0 && opt_decrypt==0 && opt_add_public_key_to_passphrase==0)){ + errx(1,"Encryption certificates specified but neither -e nor -d option not set. " + "What do you want me to do with these certificates? "); + } + + if((check_passphrase || passphrase_file) && opt_encrypt){ + err(1,"Sorry, can't both encrypt and password crack. Pick one.\n"); + } + + if(opt_encrypt && (new_passphrase==0 && num_certificates==0) && mode!=O_RDONLY){ + err(1,"Currently -e requires that the passphrase be specified on the command line\n" + "or that one or more encryption certificates be provided\n"); + } + + while(argc--){ + const char *fname = *argv++; + + if(opt_change){ + if(old_passphrase && new_passphrase) change_passphrase(fname,old_passphrase,new_passphrase); + else get_and_change_passphrase(fname); + } + + /* Get the information */ + AFFILE *af = af_open(fname,mode,0); + if(!af) af_err(1,"af_open(%s)",fname); + if(af_identify(af)!=AF_IDENTIFY_AFF && af_identify(af)!=AF_IDENTIFY_AFD){ + errx(1,"Cannot encrypt %s: %s only supports AFF and AFD files.",af_filename(af),progname); + } + + if(opt_encrypt && new_passphrase){ + int r = af_establish_aes_passphrase(af,new_passphrase); + switch(r){ + case AF_ERROR_NO_AES: errx(1,"AFFLIB: AES256 not available; cannot continue"); + case AF_ERROR_NO_SHA256: errx(1,"AFFLIB: SHA256 not available; cannot continue"); + default: err(1,"%s: cannot establish passphrase (error %d)",fname,r); + case 0: + case AF_ERROR_AFFKEY_EXISTS: + /* no matter if we established it or if a phrase already exists, try to use it now */ + /* File already has a passphrase; see if this is it. */ + break; + } + r = af_use_aes_passphrase(af,new_passphrase); + switch(r){ + case 0: break; // everything okay + case AF_ERROR_WRONG_PASSPHRASE: errx(1,"%s: wrong passphrase",fname); + default: errx(1,"%s: passphrase already established (error %d)",fname,r); + } + } + + if(opt_decrypt && !old_passphrase && getenv(AFFLIB_PASSPHRASE)){ + old_passphrase = getenv(AFFLIB_PASSPHRASE); + } + if(opt_decrypt && old_passphrase){ + int r = af_use_aes_passphrase(af, old_passphrase); + switch(r){ + case 0: printf("Passphrase is good!\n"); break; + case AF_ERROR_WRONG_PASSPHRASE: errx(1,"%s: wrong passphrase",fname); + } + } + + if (opt_add_public_key_to_passphrase){ + if(!num_certificates) errx(1,"You must specify a certificate with the -C option"); + if(!check_passphrase) errx(1,"You must specify a passphrase with the -p option"); + printf("Attepmting to add public key to AFFILE...\n"); + if(af->crypto->sealing_key_set) return AF_ERROR_KEY_SET; // already enabled + unsigned char affkey[32]; + int r = af_get_aes_key_from_passphrase(af,check_passphrase,affkey); + if(r) errx(1, "%s: cannot get aes key. Failed to add Public Key", fname); + af_seal_affkey_using_certificates(af, certificates, num_certificates, affkey); + printf("...Public key added successfully.\n"); + } + + if(opt_encrypt && num_certificates){ + if(af_set_seal_certificates(af,certificates,num_certificates)){ + errx(1,"%s: can't set encryption certificate%s",fname,num_certificates==1 ? "" : "s"); + } + } + if(opt_encrypt){ + int count = 0; + if(af_encrypt_unencrypted_nonsignature_segments(af,&count,mode)){ + errx(1,"%s: can't encrypt unsigned, unencrypted segments",fname); + } + if(mode==O_RDONLY){ // if it is readonly just print the number of segments that would be changed. + printf("%d\n",count); + af_close(af); + continue; + } + } + if(opt_decrypt){ + int count = 0; + if(af_decrypt_encrypted_segments(af, &count, mode)){ + } + if(mode==O_RDONLY){ + printf("%d\n",count); + af_close(af); + continue; + } + } + + if(opt_add_passphrase_to_public_key) { + if(!new_passphrase) errx(1,"You must specify a new passphrase with the -N option"); + printf("Attempting to add passphrase...\n"); + u_char affkey[32]; + if(af_get_affkey_using_keyfile(af, opt_unsealing_private_key_file,affkey)){ + errx(1,"%s: cannot unseal AFFKEY",fname); + } + if(af_save_aes_key_with_passphrase(af,new_passphrase,affkey)){ + af_err(1,"%s: could not set the passphrase",fname); + } + printf("... new passphrase established.\n"); + } + + + af_vnode_info vni; + memset(&vni,0,sizeof(vni)); + if(af_vstat(af,&vni)) err(1,"%s: af_vstat failed: ",fname); + const char *the_passphrase = 0; // the correct passphrase + + if(opt_just_print_encrypted_count){ + printf("%d\n",vni.segment_count_encrypted); + af_close(af); + continue; + } + + if(opt_just_print_unencrypted_count){ + printf("%d\n",vni.segment_count_total-vni.segment_count_encrypted); + af_close(af); + continue; + } + + + /* were we supposed to try a check_passphrase? */ + if(check_passphrase){ + if(af_use_aes_passphrase(af,check_passphrase)==0){ + the_passphrase = check_passphrase; + } + af_use_aes_passphrase(af,0); // clear the passphrase + } + + /* Is a passphrase file provided? */ + if(!the_passphrase && passphrase_file){ + the_passphrase = check_file(af,passphrase_file); + if(the_passphrase){ + af_use_aes_passphrase(af,0); // clear the passphrase + } + } + + if(opt_xml){ + /* This should be replaced with our xml.cpp object */ + printf("\n"); + printf(" %s\n",fname); + printf(" %d\n",vni.segment_count_total); + printf(" %d\n",vni.segment_count_signed); + printf(" %d\n",vni.segment_count_encrypted); + printf(" %d\n",vni.page_count_total); + printf(" %d\n",vni.page_count_encrypted); + if(the_passphrase){ + printf(" %s\n",the_passphrase); + } + printf("\n"); + } + else{ + /* re-run vstat because counts may have changed */ + if(af_vstat(af,&vni)) err(1,"%s: af_vstat failed: ",fname); + printf("%s: %5d segments; %5d signed; %5d encrypted; %5d pages; %5d encrypted pages", + fname,vni.segment_count_total,vni.segment_count_signed,vni.segment_count_encrypted, + vni.page_count_total,vni.page_count_encrypted ); + if(the_passphrase) printf("passphrase correct (\"%s\")",the_passphrase); + putchar('\n'); + } + af_close(af); + } + return(0); +} diff --git a/tools/affdiskprint.cpp b/tools/affdiskprint.cpp new file mode 100644 index 0000000..cdbeebc --- /dev/null +++ b/tools/affdiskprint.cpp @@ -0,0 +1,453 @@ +/* + * afdiskprint.cpp: + * + * Creates a diskprint AFF structure + */ + +/* + * PUBLIC DOMAIN + * By Simson L. Garfinkel + * + * The software provided here is released by the Naval Postgraduate + * School (NPS), an agency of the U.S. Department of Navy.The software + * bears no warranty, either expressed or implied. NPS does not assume + * legal liability nor responsibility for a User's use of the software + * or the results of such use. + * + * Please note that within the United States, copyright protection, + * under Section 105 of the United States Code, Title 17, is not + * available for any work of the United States Government and/or for + * any works created by United States Government employees. User + * acknowledges that this software contains work which was created by + * NPS employees and is therefore in the public domain and not + * subject to copyright. + */ + + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" +#include "base64.h" +#include "hashextent.h" + +#ifdef HAVE_LIBEXPAT + +#include + +#ifdef WIN32 +#include "unix4win32.h" +#endif + +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_CSTRING +#include +#endif + +const char *hashes[] = {"SHA256","SHA1",0}; // what should we hash? + +using namespace std; + +#if HAVE_CTYPE_H +#include +#endif + +#if !defined(HAVE_ISALPHANUM) && defined(HAVE_ISALNUM) +#define isalphanum(c) isalnum(c) +#endif + +#if !defined(HAVE_ISALPHANUM) && !defined(HAVE_ISALNUM) +#define isalphanum(c) (isalpha(c)||isdigit(c)) +#endif + +#if !defined(O_BINARY) +#define O_BINARY 0 +#endif + +const char *progname = "afdiskprint"; +const char *xml_special_chars = "<>\r\n&'\""; + +void usage() +{ + printf("%s version %s\n",progname,PACKAGE_VERSION); + printf("usage: %s [options] infile \n",progname); + printf(" -x XML = Verify the diskprint\n"); + printf(" -V = Just print the version number and exit.\n"); + printf(" -h = Print this help.\n"); + exit(0); +} + +/**************************************************************** + ** Support routines... + */ + +/** + * Return a random 64-bit number + */ +uint64_t random64() +{ + return (((uint64_t)random())<<32) | random(); +} + +uint64_t atoi64(const char *buf) +{ + uint64_t ret=0; + sscanf(buf,"%"PRIu64,&ret); + return ret; +} + + +static int *hexcharvals = 0; +static void nsrl_bloom_init() +{ + if(hexcharvals==0){ + /* Need to initialize this */ + int i; + hexcharvals = (int *)calloc(sizeof(int),256); + for(i=0;i<10;i++){ + hexcharvals['0'+i] = i; + } + for(i=10;i<16;i++){ + hexcharvals['A'+i-10] = i; + hexcharvals['a'+i-10] = i; + } + } +} + + +/** + * Convert a hex representation to binary, and return + * the number of bits converted. + * @param binbuf output buffer + * @param binbuf_size size of output buffer in bytes. + * @param hex input buffer (in hex) + */ +int nsrl_hex2bin(unsigned char *binbuf,size_t hexbuf_size,const char *hex) +{ + int bits = 0; + if(hexcharvals==0) nsrl_bloom_init(); + while(hex[0] && hex[1] && hexbuf_size>0){ + *binbuf++ = ((hexcharvals[(unsigned char)hex[0]]<<4) | + hexcharvals[(unsigned char)hex[1]]); + hex += 2; + bits += 8; + hexbuf_size -= 1; + } + if(hexbuf_size>0) binbuf[0] = 0; // might as well null-terminate if there is room + return bits; +} + + + +/** + * Strip an XML string as necessary for a tag name. + */ + +void out_xmlstr(ostream &ostr,int indent,const char *tag,const char *value) +{ + for(int i=0;i"; + for(const char *ch=value;*ch;ch++){ + if(isprint(*ch) && !strchr(xml_special_chars,*ch)){ + ostr << *ch; + } + } + ostr << ""; +} + +void out_xmlhex(ostream &ostr,int indent,const char *tag,const char **attribs, + unsigned char *md,int len) +{ + for(int i=0;i" << hashextent::bin2hex(md,len) << "\n"; +} + +/** + * Calculate the disk fingerprint for a spcific file and output it to stdout. + * Includes other named segments. + * + * @param infile the file to process. + */ + +int diskprint(const char *infile) +{ + /** segments to include in output. + */ + const char *segments[] = {AF_MD5,AF_SHA1,AF_SHA256,AF_CREATOR,AF_CASE_NUM,AF_IMAGE_GID, + AF_ACQUISITION_ISO_COUNTRY, + AF_ACQUISITION_COMMAND_LINE,AF_ACQUISITION_DATE, + AF_ACQUISITION_NOTES,AF_ACQUISITION_TECHNICIAN, + AF_BATCH_NAME,AF_BATCH_ITEM_NAME,0}; + AFFILE *af = af_open(infile,O_RDONLY,0); + if(!af){ + warn("%s",infile); + return -1; + } + + cout << "\n"; + cout << "\n"; + + /* First handle the imagesize */ + int64_t imagesize = af_get_imagesize(af); + if(imagesize>0){ + char buf[32]; + snprintf(buf,sizeof(buf),"%"PRIu64,imagesize); + out_xmlstr(cout,2,AF_IMAGESIZE,buf); + cout << "\n"; + } + /* Get sector size and number of sectors */ + uint32_t sectorsize=512; // default sectorsize + af_get_seg(af,AF_SECTORSIZE,§orsize,0,0); + if(sectorsize==0) sectorsize=512; // default sectorsize + int64_t sectors = imagesize/sectorsize; + if(sectors>0){ + char buf[32]; + snprintf(buf,sizeof(buf),"%"PRIu32"",sectorsize); + out_xmlstr(cout,2,AF_SECTORSIZE,buf); + cout << "\n"; + } + + /* Output specific named segments */ + for(int i=0;segments[i];i++){ + char buf[65536]; + size_t buflen = sizeof(buf); + if(af_get_seg(af,segments[i],0,(u_char *)buf,&buflen)==0){ + buf[buflen] = 0; // null terminate it + if(af_display_as_hex(segments[i])){ + out_xmlhex(cout,2,segments[i],0,(u_char *)buf,buflen); + } else { + out_xmlstr(cout,2,segments[i],buf); + } + } + } + + /** + * The list of segments to hash is defined by: + * 1. The first 128K sectors. + * 2. The last 128K sectors. + * 3. A random set of 64K sectors. + */ + hashvector hashextents; + + for(int i=0;i<8;i++){ + hashextents.push_back(hashextent(131072*i,131072)); + } + for(int i=0;i<8;i++){ + hashextents.push_back(hashextent(imagesize-131072*(8-i),131072)); + } + + /* Pick some random hashextents as well */ + for(int i=0;i<100;i++){ + uint64_t sector = random64() % (sectors-128); + hashextents.push_back(hashextent(sector*sectorsize,65536)); + } + + /** Sort the segments for maximal seek efficiency. + */ + sort(hashextents.begin(),hashextents.end(),hashextent::compare); + + /** Send the hashes to stdout using print_hash. + */ + cout << " \n"; + for(hashvector::iterator it=hashextents.begin(); it!=hashextents.end(); it++){ + for(int i=0;hashes[i];i++){ + if((*it).compute_digest(af,hashes[i])==0){ + cout << " " << (*it).toXML() << "\n"; + } + } + } + cout << " \n"; + cout << "\n"; + af_close(af); + return 0; +} + + +/** + * Code for reading the hashvector XML structure. + */ +#include + +class diskprintReader { +public: + /* General EXPAT stuff */ + XML_Parser parser; + bool get_cdata; // this is cdata worth getting + string cdata; // the cdata that has been gotten + diskprintReader():get_cdata(false),hash(0){ + parser = XML_ParserCreate(NULL); + XML_SetUserData(parser,this); + XML_SetElementHandler(parser,startElement,endElement); + XML_SetCharacterDataHandler(parser,cHandler); + } + int parse(const char *buf,int len) { return XML_Parse(parser, buf, len, 1);} + void clear(){ + cdata = ""; + get_cdata = false; + } + + /* Specific stuff for XML diskprint */ + hashextent *hash; // current hash + hashvector hashextents; // all discovered hashes + /* Turn the static functions into method calls */ + static void startElement(void *userData,const char *name,const char **attrs){ + ((diskprintReader *)userData)->startElement(name,attrs); + } + static void endElement(void *userData,const char *name){ + ((diskprintReader *)userData)->endElement(name); + } + static void cHandler(void *userData,const XML_Char *s,int len){ + diskprintReader *dh = (diskprintReader *)userData; + if(dh->get_cdata) dh->cdata.append(s,len); + } + void startElement(string name,const char **attrs){ + clear(); + /* If this is an element that we want, indicate such */ + if(name=="hash"){ + hash = new hashextent(); + for(int i=0;attrs[i];i+=2){ + if(strcmp(attrs[i],"coding")==0){ hash->coding = attrs[i+1]; continue;} + if(strcmp(attrs[i],"start")==0){ hash->start = atoi64(attrs[i+1]); continue;} + if(strcmp(attrs[i],"bytes")==0){ hash->bytes = atoi64(attrs[i+1]); continue;} + if(strcmp(attrs[i],"alg")==0){ hash->digest_name = attrs[i+1]; continue;} + } + get_cdata = true; + } + } + void endElement(const char *name){ + if(get_cdata==false) return; // don't care about it. + if(!strcmp(name,"hash")){ + if(hash->coding=="base16"){ + hash->hexdigest = cdata; + } + hashextents.push_back(*hash); + } + if(!strcmp(name,"diskprint")){ + XML_StopParser(parser,0); // stop the parser + return; + } + get_cdata = false; + } +}; + +void diskprint_verify(const char *filename,const char *xmlfile) +{ + AFFILE *af = af_open(filename,O_RDONLY,0); + if(!af) err(1,"af_open(%s): ",filename); + + /* Let's read the XML file */ + int fd = open(xmlfile,O_RDONLY|O_BINARY); + if(!fd) err(1,"open: %s",xmlfile); + struct stat st; + if(fstat(fd,&st)) err(1,"stat: %s",xmlfile); + char *buf = (char *)malloc(st.st_size+1); + if(!buf) err(1,"malloc"); + + if(read(fd,buf,st.st_size)!=st.st_size) err(1,"cannot read XML file"); + buf[st.st_size]=0; // terminate the buffer (not strictly needed) + + diskprintReader dp; + dp.parse(buf,st.st_size); + cout << "Number of digests read: "<< dp.hashextents.size() << "\n"; + const EVP_MD *strongest = dp.hashextents.strongest_available(); + cout << "Strongest hash available: " << EVP_MD_name(strongest) << "\n"; + /* Now verify each hash */ + int matched=0; + int notmatched=0; + for(hashvector::iterator it = dp.hashextents.begin(); it!=dp.hashextents.end() && notmatched==0;it++){ + if(EVP_MD_name(strongest) == (*it).digest_name){ + hashextent &hp = (*it); // hash print + hashextent hs(af,hp.digest_name,hp.start,hp.bytes); + //cout << "hp: " << hp << "\n"; + //cout << "hs: " << hs << "\n"; + if(hp==hs){ + matched++; + } else { + notmatched++; + } + } + } + if(notmatched){ + cout << "Diskprint does not match.\n"; + } + if(notmatched==0 && matched){ + cout << "Diskprint matches.\n"; + } + if(notmatched==0 && matched==0){ + cout << "Cannot verify Diskprint; no available hash functions.\n"; + } + exit(0); +} + +int main(int argc,char **argv) +{ + int ch; + const char *opt_x=0; + + /* Initialize */ +#ifdef HAVE_SRANDOMEDEV + srandomdev(); +#endif + OpenSSL_add_all_digests();/* Dynamically loads the digests */ + + + /* Parse arguments */ + while ((ch = getopt(argc, argv, "x:h?V")) != -1) { + switch (ch) { + case 'h': + case '?': + default: + usage(); + break; + case 'x': opt_x = optarg; break; + case 'V': + printf("%s version %s\n",progname,PACKAGE_VERSION); + exit(0); + } + } + argc -= optind; + argv += optind; + + if(argc!=1){ // currently only generates for one file + usage(); + } + + if(opt_x){ + diskprint_verify(argv[0],opt_x); + return(0); + } + + /* Loop through all of the files */ + printf("\n"); + printf("\n"); + while(*argv){ + if(opt_x){ + diskprint_verify(*argv,opt_x); + } + else { + diskprint(*argv); + } + argv++; + argc--; + } + printf("\n"); + exit(0); +} +#else +int main(int argc,char **argv) +{ + fprintf(stderr,"affdiskprint requires EXPAT. Cannot continue.\n"); + exit(-1); +} +#endif + diff --git a/tools/affinfo.cpp b/tools/affinfo.cpp new file mode 100644 index 0000000..9314177 --- /dev/null +++ b/tools/affinfo.cpp @@ -0,0 +1,828 @@ +/* + * afinfo.cpp: + * + * print information about an aff file + * Distributed under the Berkeley 4-part license + */ + + + +#include "affconfig.h" +#include "afflib.h" +#include "utils.h" +#include "afflib_i.h" + +#ifdef USE_S3 +#include "s3_glue.h" +#endif + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#ifdef HAVE_CSTRING +#include +#endif + +using namespace std; + +#ifdef HAVE_CURSES_H +#include +#endif + +#ifdef HAVE_TERM_H +#include +#endif + +#ifdef HAVE_NCURSES_TERM_H +#include +#endif + +#ifdef WIN32 +#include "unix4win32.h" +#endif + +const char *progname = "afinfo"; + +#define VALIDATE_MD5 0x01 +#define VALIDATE_SHA1 0x02 + +int opt_validate = 0; +int opt_info = 1; +int opt_all = 0; +int opt_wide = 0; +int opt_l = 0; +unsigned int cols = 80; // default +int opt_x = 0; +int opt_b = 0; +int opt_identify = 1; +int opt_verbose = 0; +int opt_y = 0; +int opt_hexbuf = AF_HEXBUF_SPACE4 | AF_HEXBUF_UPPERCASE; +int opt_page_validate = 0; +int opt_no_preview = 0; +int opt_preview_md5 = 0; +int opt_debug = 0; +int opt_figure_media = 0; +const char *opt_passphrase = 0; + +vector opt_seglist; // just info these segments +bool something_was_decrypted = false; +const char *term = 0; + + +/** + * select bold on or off + */ +void bold(int on) +{ + if(!term) return; +#ifdef HAVE_ISATTY + if(!isatty(fileno(stdout))) return; +#endif +#if defined(HAVE_TPUTS) + if(on) tputs(enter_bold_mode,1,putchar); + else tputs(exit_attribute_mode,0,putchar); +#endif +} + +/** + * select a color. + * @param num - 0 is black; 1 red; 2 green; 3 yellow; 4 blue; 5 magenta; 6 cyan; 7 white; + */ + +#define RED 1 +#define WHITE 7 + +void color(int num) +{ +#ifdef HAVE_ISATTY + if(!isatty(fileno(stdout))) return; +#endif +#if defined(HAVE_TIGETSTR) && defined(HAVE_PUTP) && defined(HAVE_TPARM) + char *setf = tigetstr((char *)"setf"); + if(!setf) setf = tigetstr((char *)"setaf"); + if(setf){ + putp(tparm(setf,num)); + } +#endif +} + + +void usage() +{ + printf("%s version %s\n",progname,PACKAGE_VERSION); + printf("usage: %s [options] infile\n",progname); + printf(" -a = print ALL segments (normally data segments are suppressed)\n"); + printf(" -b = print how many bad blocks in each segment (implies -a)\n"); + printf(" -i = identify the files, don't do info on them.\n"); + printf(" -w = wide output; print more than 1 line if necessary.\n"); + printf(" -s segment = Just print information about 'segment'.\n"); + printf(" (may be repeated)\n"); + printf(" -m = validate MD5 hash of entire image\n"); + printf(" -S = validate SHA1 hash of entire image\n"); + printf(" -v = validate the hash of each page (if present)\n"); + printf(" -y = don't print segments of lengths 16 and 20 as hex)\n"); + printf(" -p = Specify to decrypt file\n"); + printf(" -l = Just print the segment names and exit\n"); + printf(" -V = Just print the version number and exit.\n"); + + printf("\nPreview Options:\n"); + printf(" -X = no data preview; just print the segment names\n"); + printf(" -x = print binary values in hex (default is ASCII)\n"); + printf("\nMisc:\n"); + printf(" -d = debug\n"); + printf(" -A = if infile is a device, print the number of sectors\n"); + printf(" and sector size to stdout in XML. Otherwise error\n"); + printf("\nCompilation:\n"); + printf(" LZMA compression: Enabled\n"); +#ifdef USE_LIBEWF + printf(" LIBEWF enabled\n"); +#endif +#ifdef USE_QEMU + printf(" QEMU enabled\n"); +#endif +#ifdef USE_FUSE + printf(" FUSE enabled\n"); +#endif +#ifdef USE_S3 + printf(" Amazon S3 enabled\n"); +#endif +#ifdef HAVE_LIBEXPAT + printf(" HAVE_LIBEXPAT "); +#endif + printf("\n"); + + if(opt_debug){ + for(int i=0;i<9;i++){ + color(i);printf("Color %d\n",i);color(7); + } + } + + exit(0); +} + + +AFFILE *af=0; + +void sig_info(int arg) +{ + if(af==0) return; + printf("Validating %"I64d" of %"I64d"\n", af->pos,af->image_size); +} + + + + +void validate(const char *infile) +{ + af = af_open(infile,O_RDONLY,0); + if(!af) af_err(1,infile); + switch(af_identify(af)){ + case AF_IDENTIFY_AFF: + case AF_IDENTIFY_AFM: + case AF_IDENTIFY_AFD: + break; + default: + printf("%s is not an AFF file\n",infile); + af_close(af); + return; + } + + printf("\nValidating "); + if(opt_validate & VALIDATE_MD5) printf("MD5 "); + if(opt_validate == (VALIDATE_MD5|VALIDATE_SHA1)) printf("and "); + if(opt_validate & VALIDATE_SHA1) printf("SHA1 "); + printf("hash codes.\n"); + + +#ifdef SIGINFO + signal(SIGINFO,sig_info); +#endif + + /* Get a list of all the segments to see if there is a space */ + af_rewind_seg(af); + char segname[AF_MAX_NAME_LEN]; + vector pages; + memset(segname,0,sizeof(segname)); + while(af_get_next_seg(af,segname,sizeof(segname),0,0,0)==0){ + int64_t page_num = af_segname_page_number(segname); + if(page_num>=0) pages.push_back(page_num); + } + + if(pages.size()==0){ + printf("No pages to validate.\n"); + af_close(af); + } + + sort(pages.begin(),pages.end()); + vector::iterator i = pages.begin(); + int last = *i; + i++; + for(; i!= pages.end();i++){ + if(last+1 != *i){ + printf("gap in pages (%d!=%d); %s can't be validated.\n",last+1,*i,infile); + af_close(af); + return; + } + last = *i; + } + + /* Set up the hash machinery */ + MD5_CTX md5; + MD5_Init(&md5); + + SHA_CTX sha; + SHA1_Init(&sha); + + uint64_t total_bytes = 0; + while(!af_eof(af)){ + unsigned char buf[65536]; // a decent size + size_t bytes = af_read(af,buf,sizeof(buf)); + if(bytes==0) break; // reached sparse region of file + total_bytes += bytes; + if(opt_validate & VALIDATE_MD5) MD5_Update(&md5,buf,bytes); + if(opt_validate & VALIDATE_SHA1) SHA1_Update(&sha,buf,bytes); + } + + /* Finish the hash calculations and write to the db */ + + if(opt_validate & VALIDATE_MD5){ + unsigned char md5_stored[16]; + size_t md5len = sizeof(md5_stored); + unsigned char md5_computed[16]; + char buf[256]; + + MD5_Final(md5_computed,&md5); + printf("computed md5: %s\n", + af_hexbuf(buf,sizeof(buf),md5_computed,16,opt_hexbuf)); + if(af_get_seg(af,AF_MD5,0,md5_stored,&md5len)==0){ + printf(" stored md5: %s ", + af_hexbuf(buf,sizeof(buf),md5_stored,16,opt_hexbuf)); + if(md5len==16 && !memcmp((const char *)md5_stored, + (const char *)md5_computed,16)){ + printf(" MATCH\n"); + } + else { + printf(" NO MATCH!\n"); + } + } + else { + printf("(no MD5 in AFF file)\n"); + } + } + + + if(opt_validate & VALIDATE_SHA1){ + unsigned char sha1_stored[20]; + size_t sha1len = sizeof(sha1_stored); + unsigned char sha1_computed[20]; + char buf[256]; + + SHA1_Final(sha1_computed,&sha); + printf("computed sha1: %s \n",af_hexbuf(buf,sizeof(buf),sha1_computed,20,opt_hexbuf)); + if(af_get_seg(af,AF_SHA1,0,sha1_stored,&sha1len)==0){ + printf(" stored sha1: %s ",af_hexbuf(buf,sizeof(buf),sha1_stored,20,opt_hexbuf)); + if(sha1len==20 && !memcmp((const char *)sha1_stored, + (const char *)sha1_computed,20)){ + printf(" MATCH\n"); + } + else { + printf(" NO MATCH!\n"); + } + } + else { + printf("(no SHA1 in AFF file)\n"); + } + } + + af_close(af); +} + +#define OUTLINE_LEN 65536 + + +bool display_as_time(const char *segname) +{ + if(strcmp(segname,AF_ACQUISITION_SECONDS)==0) return true; + return false; +} + +bool display_as_hex(const char *segname,int data_len) +{ + if(af_display_as_hex(segname)) return true; + if(data_len==16 && strstr(segname,"md5")) return true; + if(data_len==20 && strstr(segname,"sha1")) return true; + if(opt_x) return true; + if(opt_preview_md5) return true; + return false; +} + +void badscan(AFFILE *af,int page_number,size_t data_len) +{ + size_t page_size = af->image_pagesize; + unsigned char *buf = (unsigned char *)malloc(page_size); + if(af_get_page(af,page_number,buf,&page_size)){ + err(1,"Could not read page %d",page_number); + } + printf("page_size = %d\n",(int)page_size); + int sectors = 0; + int bad_sectors = 0; + int funny_sectors = 0; + for(unsigned int offset=0;offsetimage_sectorsize){ + sectors++; + if(af_is_badsector(af,buf+offset)){ + bad_sectors ++; + continue; + } +#ifdef __FreeBSD__ + /* Look for the part of the bad flag that we know and love */ + if(strnstr((char *)buf+offset,"BAD SECTOR",af->image_sectorsize)){ + funny_sectors++; + continue; + } +#endif + } + printf(" sectors scanned: %d bad: %d ", sectors,bad_sectors); + if(funny_sectors){ + printf("suspicious: %d ",funny_sectors); + } + printf("\n"); + free(buf); +} + + +/* print_info: + * Print the info on a given segment name + */ +void print_info(AFFILE *af,const char *segname) +{ + uint32_t arg; + unsigned char *data = 0; + int dots = 0; + u_int display_len = 0; + char *cc = 0; + + /* Check to see if this is a null page. */ + if(segname[0]==0 && opt_all==0){ + return; + } + if(segname[0]=='[' && opt_all){ + puts(segname); + return; + } + + /* Check to see if this is a data page */ + int64_t page_num = af_segname_page_number(segname); + + size_t data_len = 0; + /* First find out how big the segment is, then get the data */ + if(af_get_seg(af,segname,&arg,0,&data_len)){ + printf("%-25s SEGMENT NOT FOUND\n",segname); + return; + } + + /* is this an encrypted segment that I have decrypted? + * Turn off automatic decryption and see if I can get it again... + * If we can get it again, then it wasn't decrypted. + */ + int prev = af_set_option(af,AF_OPTION_AUTO_DECRYPT,0); + bool was_decrypted = ( af_get_seg(af,segname,0,0,0)!=0) ; + af_set_option(af,AF_OPTION_AUTO_DECRYPT,prev); + + if(was_decrypted){ + bold(1); + something_was_decrypted = true; // print key at bottom + } + + /* Start the output line */ + char output_line[OUTLINE_LEN]; + memset(output_line,0,sizeof(output_line)); + + /* Now append the arg and the data len */ + sprintf(output_line,"%-24s %8"PRIu32" %8d ",segname,arg,(int)data_len); + + if(opt_no_preview){ + printf("%s\n",output_line); + goto done; + } + + data = (unsigned char *)malloc(data_len); + if(af_get_seg(af,segname,0,data,&data_len)){ + warn("af_get_seg_2 failed: segname=%s data_len=%zd",segname,data_len); + goto done; + } + + /* Special handling of values that should be displayed as time */ + if(display_as_time(segname)){ + int hours = arg / 3600; + int minutes = (arg / 60) % 60; + int seconds = arg % 60; + printf("%s= %02d:%02d:%02d (hh:mm:ss)\n",output_line,hours,minutes,seconds); + goto done; + } + + /* Special handling of quadwords that should be printed as such? */ + if(((arg == AF_SEG_QUADWORD) && (data_len==8)) || af_display_as_quad(segname)){ + /* Print it as a 64-bit value. + * The strcmp is there because early AF_IMAGESIZE segs didn't set + * AF_SEG_QUADWORD... + */ + switch(data_len){ + case 8: + printf("%s= %"I64d" (64-bit value)\n", + output_line,af_decode_q(data)); + break; + case 0: + printf("%s= 0 (0-length segment)\n",output_line); + break; + default: + printf("%s= CANNOT DECODE %d byte segment\n",output_line,(int)data_len); + } + goto done; + } + + + /* See if I need to truncate */ + display_len = data_len; + if(opt_wide==0 && data_len>32){ // don't bother showing more than first 32 bytes + dots = 1; + display_len = 32; + } + + cc = output_line + strlen(output_line); + + if(opt_preview_md5){ + u_char md5[32]; + MD5(data,data_len,md5); + memcpy(data,md5,32); + data_len = 32; + } + if(display_as_hex(segname,display_len)){ + char buf[80]; + snprintf(cc,sizeof(output_line)-strlen(output_line), + "%s%s",af_hexbuf(buf,sizeof(buf),data,display_len,opt_hexbuf), + dots ? "..." : ""); + /* Special code for SHA1 */ + if(!opt_wide && strcmp(segname,AF_SHA1)==0){ + int r = fwrite(output_line,1,82,stdout); + if(r!=82) fprintf(stderr,"fwrite(output_line,1,82,stdout) returned %d\n",r); + printf("\n%62s\n",output_line+82); + goto done; + } + } + else { + /* Fill it out with some printable data */ + unsigned int i; + if(display_len > sizeof(output_line)-strlen(output_line)){ + display_len = sizeof(output_line)-strlen(output_line); + } + for(i=0;icols){ + output_line[cols-4] = '.'; + output_line[cols-3] = '.'; + output_line[cols-2] = '.'; + output_line[cols-1] = '\000'; + } + } + fputs(output_line,stdout); + if(page_num>=0 && opt_b){ + badscan(af,page_num,data_len); + } + if(opt_page_validate && page_num>=0){ + /* Get the page again; this may involve decompression */ + unsigned char *page_data = (unsigned char *)malloc(af->image_pagesize); + size_t page_data_len = af->image_pagesize; + if(af_get_page(af,page_num,page_data,&page_data_len)){ + printf("** COULD NOT READ UNCOMPRESSED PAGE "); + goto skip1; + } + + char hash_segname[32]; + unsigned char hash_buf[16]; + unsigned char hash_calc[16]; + size_t hash_len = sizeof(hash_buf); + snprintf(hash_segname,sizeof(hash_segname),AF_PAGE_MD5,page_num); + printf(" "); + if(af_get_seg(af,hash_segname,0,hash_buf,&hash_len)){ + printf("** NO SEGMENT %s ** ",hash_segname); + goto skip1; + } + + MD5(page_data,page_data_len,hash_calc); + if(memcmp(hash_buf,hash_calc,sizeof(hash_buf))!=0){ + char hb[32]; + printf("** HASH INVALID **\n%30s Calculated %s\n","", + af_hexbuf(hb,sizeof(hb),hash_calc,16,opt_hexbuf)); + printf("%30s Wanted %s ","",af_hexbuf(hb,sizeof(hb),hash_buf,16,opt_hexbuf)); + printf("data_len=%d\n",(int)data_len); + } else{ + printf("HASH OK "); + } + free(page_data); + } + skip1:; + putchar('\n'); + done: + if(data) free(data); + bold(0); // make sure bold is off + + //color(WHITE); // make sure we are back to normal color +} + + +/* Print the information on a specific file. */ +int info_file(const char *infile) +{ + uint32_t total_segs = 0; + uint32_t total_pages = 0; + uint32_t total_hashes = 0; + uint32_t total_signatures =0; + uint32_t total_nulls = 0; + struct af_vnode_info vni; + + AFFILE *af = af_open(infile,O_RDONLY,0); + if(!af) af_err(1,"Cannot open %s",infile); + if(af_vstat(af,&vni)) err(1,"%s: af_vstat failed",infile); + + if(opt_l){ + /* Just list the segments and exit */ + aff::seglist sl; + sl.get_seglist(af); + for(aff::seglist::const_iterator i = sl.begin(); i!=sl.end(); i++){ + printf("%s\n",(*i).name.c_str()); + } + af_close(af); + return 0; + } + + if(vni.segment_count_encrypted>0 || vni.segment_count_signed>0){ + printf("%s: has %s%s%ssegments\n",infile, + (vni.segment_count_encrypted ? "encrypted " : ""), + ((vni.segment_count_encrypted && vni.segment_count_signed) ? "and ": ""), + (vni.segment_count_signed ? "signed " : "")); + } + + + if(opt_passphrase){ + if(af_use_aes_passphrase(af,opt_passphrase)){ + errx(1,"%s: cannot use passphrase",opt_passphrase); + } + } + + printf("\n%s\n",af_filename(af)); + const char *v1 = "data"; + const char *v2 = "===="; + + if(opt_all==0) printf("[skipping data segments]\n"); + if(opt_all==0 && vni.segment_count_encrypted) printf("[skipping encrypted segments]\n"); + if(opt_no_preview){ + v1 = ""; + v2 = ""; + } + if(opt_preview_md5){ + v1 = "md5"; + } + + printf(" data \n"); + printf("Segment arg length %s\n",v1); + printf("======= ========= ======== %s\n",v2); + + /* If a list of segments was specified by the user, just use that list */ + if(opt_seglist.size()>0){ + for(vector::iterator i = opt_seglist.begin(); + i != opt_seglist.end(); + i++){ + const char *segname = i->c_str(); + print_info(af,segname); + } + af_close(af); + return 0; + } + + /* Go through the whole file, get all of the segments, put them in a list */ + vector segments; + char segname[AF_MAX_NAME_LEN]; + af_rewind_seg(af); // start at the beginning + int64_t total_datalen = 0; + size_t total_segname_len = 0; + size_t datalen = 0; + int aes_segs=0; + while(af_get_next_seg(af,segname,sizeof(segname),0,0,&datalen)==0){ + total_segs ++; + total_datalen += datalen; + total_segname_len += strlen(segname); + if(segname[0]==0) total_nulls++; + + /* Check to see if this is a regular page or a hash page */ + char hash[64]; + int64_t page_num = af_segname_page_number(segname); + int64_t hash_num = af_segname_hash_page_number(segname,hash,sizeof(hash)); + if(page_num>=0) total_pages++; + if(hash_num>=0) total_hashes++; + if(strstr(segname,AF_SIG256_SUFFIX)) total_signatures++; + if(strstr(segname,AF_AES256_SUFFIX)) aes_segs++; + if(opt_all==0 && (page_num>=0||hash_num>=0)) continue; // skip + if(opt_all==0 && af_is_encrypted_segment(segname)) continue; // skip + + if(segname[0]==0 && datalen>0 && opt_all){ + snprintf(segname,sizeof(segname),"[null %zd bytes]",datalen); + } + segments.push_back(segname); + } + + /* Now process the segments */ + for(vector::const_iterator i = segments.begin(); + i != segments.end(); i++){ + print_info(af,i->c_str()); + } + + /* Print the key */ + if(something_was_decrypted){ + bold(1); + printf("Bold indicates segments that were decrypted.\n"); + bold(0); + } + + + printf("\n"); + printf("Total segments: %8"PRIu32" (%"PRIu32" real)\n", total_segs,(total_segs-total_nulls)); + if(aes_segs){ + printf(" Encrypted segments: %8u\n",aes_segs); + } + printf(" Page segments: %8"PRIu32"\n",total_pages); + printf(" Hash segments: %8"PRIu32"\n",total_hashes); + printf(" Signature segments: %8"PRIu32"\n",total_signatures); + printf(" Null segments: %8"PRIu32"\n",total_nulls); + if(opt_all){ + printf(" Empty segments: %8"PRIu32"\n",total_nulls); + printf("\n"); + printf("Total data bytes in segments: %"I64d"\n",total_datalen); + + printf("Total space in file dedicated to segment names: %zd\n", + total_segname_len); + printf("Total overhead for %"PRIu32" segments: %zd bytes (%"PRIu32"*(%zd+%zd))\n", + total_segs, + (size_t) total_segs*(sizeof(struct af_segment_head) +sizeof(struct af_segment_tail)), + total_segs, + sizeof(struct af_segment_head), + sizeof(struct af_segment_tail)); + printf("Overhead for AFF file header: %zd bytes\n",sizeof(struct af_head)); + } + + int64_t device_sectors = 0; + af_get_segq(af,AF_DEVICE_SECTORS,&device_sectors); + if(device_sectors==0){ + /* See if we can fake it */ + uint32_t cylinders=0; + uint32_t heads=0; + uint32_t sectors_per_track=0; + af_get_seg(af,AF_CYLINDERS,&cylinders,0,0); + af_get_seg(af,AF_HEADS,&heads,0,0); + af_get_seg(af,AF_SECTORS_PER_TRACK,§ors_per_track,0,0); + device_sectors = cylinders * heads * sectors_per_track; + } + //printf("device_sectors=%"I64d"\n",device_sectors); + + + int some_missing_pages = 1; + if(af->image_pagesize && af->image_sectorsize && device_sectors){ + int64_t device_bytes = (int64_t)device_sectors * af->image_sectorsize; + int64_t device_pages = (device_bytes+af->image_pagesize-1) / af->image_pagesize; + int64_t missing_pages = device_pages - total_pages; + //printf("device_bytes=%"I64d"\n",device_bytes); + //printf("device_pages=%"I64d"\n",device_pages); + if(missing_pages!=0){ + printf("Missing page segments: %8"I64u"\n",missing_pages); + } + else { + some_missing_pages=0; + } + } + if (some_missing_pages){ + if(((total_pages-1) * af->image_pagesize <= af->image_size) && + ((total_pages) * af->image_pagesize >= af->image_size)){ + some_missing_pages = 0; + } + } + + if(some_missing_pages && opt_debug){ + printf("Cannot calculate missing pages\n"); + printf(" device_sectors=%"I64d" image_pagesize=%"PRIu32" sectorsize=%"PRIu32"\n", + device_sectors,af->image_pagesize,af->image_sectorsize); + } + af_close(af); + return 0; + +} + + +void figure_media(const char *fn) +{ + int fd = open(fn,O_RDONLY,0); + if(fd<0) err(1,"open(%s)",fn); + struct af_figure_media_buf afb; + if(af_figure_media(fd,&afb)){ + err(1,"af_figure_media(%s)",fn); + } + printf("\n"); + printf("\n"); + printf("\n",fn); + printf(" %d\n",afb.sector_size); + printf(" %"PRId64"\n",afb.total_sectors); + printf(" %"PRIu64"\n",afb.max_read_blocks); + printf("\n"); + close(fd); +} + +int main(int argc,char **argv) +{ + int ch; + const char *infile; + + /* Figure out how many cols the screen has... */ +#ifdef HAVE_LIBNCURSES + term = getenv("TERM"); + if(term){ + setupterm((char *)0,1,(int *)0); + start_color(); + cols = tgetnum((char *)"co"); + } +#endif + + while ((ch = getopt(argc, argv, "abh?s:SmiIwj:p:xvVX5dAl")) != -1) { + switch (ch) { + case 'a': opt_all++; break; + case 'b': opt_all ++; opt_b ++; break; + case 'i': opt_info=0; opt_identify = 1; break; + case 'w': opt_wide++; break; + case 'X': opt_no_preview++;break; + case 'x': opt_x++; break; + case 'y': opt_y++; break; + case 'l': opt_l++;break; + case 'm': opt_validate |= VALIDATE_MD5; break; + case 'S': opt_validate |= VALIDATE_SHA1; break; + case 'v': opt_page_validate = 1;break; + case 'p': opt_passphrase = optarg; break; + case '5': opt_preview_md5 = 1;break; + case 'd': opt_debug = 1;break; + case 'A': opt_figure_media = 1 ; break; + + case 'h': + case '?': + default: + usage(); + break; + case 's': + opt_seglist.push_back(optarg); // add to the list of segments to info + break; + case 'V': + printf("%s version %s\n",progname,PACKAGE_VERSION); + exit(0); + } + } + argc -= optind; + argv += optind; + + if(argc<1){ + usage(); + } + + + /* Loop through all of the files */ + while(*argv){ + infile = *argv++; // get the file + argc--; // decrement argument counter + + const char *name = af_identify_file_name(infile,1); + if(!name) err(1,"%s",infile); + + if(opt_figure_media){ figure_media(infile);continue;} + if(opt_identify) printf("%s is a %s file\n",infile,name); + if(opt_info) info_file(infile); + if(opt_validate) validate(infile); + } +#ifdef USE_S3 + s3_audit(0); +#endif + return(0); +} + + diff --git a/tools/affix.cpp b/tools/affix.cpp new file mode 100644 index 0000000..6c5368e --- /dev/null +++ b/tools/affix.cpp @@ -0,0 +1,233 @@ +/** + * affix.cpp + * + * Fix an aff file that is corrupt. + * Current methodologies: + * - If file does not have a GUID, create one. + * Distributed under the Berkeley 4-part license + */ + + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" +#include "vnode_aff.h" // so we can use magical af_open_with... + +#include + +#include +#include +#include +#include + +#ifdef HAVE_UNISTD_H +#include +#endif + +#ifdef HAVE_TERM_H +#include +#endif + +#ifdef HAVE_NCURSES_TERM_H +#include +#endif + +#ifdef WIN32 +#include "unix4win32.h" +#include +#endif + +const char *progname = "affix"; +int opt_fix = 0; + + +void usage() +{ + printf("usage: %s [options] file1 [...]\n",progname); + printf(" -y = Actually modify the files; normally just reports the problems\n"); + printf(" -v = Just print the version number and exit.\n"); + exit(0); +} + + + +/* Returns 0 if this is a valid AFF file, code if it isn't. */ +int af_is_valid_afffile(const char *file) +{ + return 0; // I should write this +} + +int fix(const char *infile) +{ + char buf[1024]; + int flags = (opt_fix ? O_RDWR : O_RDONLY) | O_BINARY; + switch(af_identify_file_type(infile,1)){ + case AF_IDENTIFY_ERR: + perror(infile); + return 0; + default: + fprintf(stderr,"%s is not an AFF file\n",infile); + return 0; + case AF_IDENTIFY_AFF: + break; + } + + printf("%s ",infile); + int r=0; + + /* First see if the if the file begins with an AFF flag */ + int fd = open(infile,flags,0666); + if(fd<0) err(1,"fopen(%s)",infile); + if(read(fd,buf,strlen(AF_HEADER)+1)!=strlen(AF_HEADER)+1) + err(1,"can't read AFF file header. Stop."); + if(strcmp(buf,AF_HEADER)!=0) + err(1,"%s does not begin with an AF_HEADER. Stop.",infile); + if(read(fd,buf,strlen(AF_SEGHEAD)+1)!=strlen(AF_SEGHEAD)+1) + err(1,"Can't read AF_SEGHEAD after AF_HEADER. Stop."); + if(strcmp(buf,AF_SEGHEAD)!=0) + err(1,"%s does not have an AF_SEGHEAD after AF_SEGEADER. Stop.",infile); + + /* Figure out length */ + off_t len = lseek(fd,0,SEEK_END); + if(len<0) err(1,"Can't seek to end of %s. Stop.",infile); + close(fd); + + AFFILE *af = af_open_with(infile,AF_HALF_OPEN|flags,0,&vnode_aff); + printf("Scanning AFF file...\n"); + r = (*af->v->open)(af); + /* See if we can build a TOC */ + if(r<0){ + printf("AFF file corrupt at %"I64d" out of %"I64d" (%"I64d" bytes from end)\n", + ftello(af->aseg),(int64_t)len,len-ftello(af->aseg)); + if(opt_fix){ + printf("Truncating... %d \n",fileno(af->aseg)); + if(ftruncate(fileno(af->aseg),ftello(af->aseg))){ + err(1,"ftruncate"); + } + } + } + + /* See if it has a GID or an encrypted GID */ + if(af_get_seg(af,AF_IMAGE_GID,0,0,0)!=0 && + af_get_seg(af,AF_IMAGE_GID AF_AES256_SUFFIX,0,0,0)!=0){ + printf("AFF file is missing a GID. "); + if(opt_fix){ + printf("Making one..."); + if(af_make_gid(af)<0) af_err(1,"af_make_gid"); + } + putchar('\n'); + } + + af_close(af); + + return 0; +} +#if 0 + + + /* See if it ends properly */ + off_t len; + + printf("File is %"I64d" bytes long\n",len); + + if(lseek(fd,len-4,SEEK_SET)<0) + err(1,"Can't backup %d bytes. Stop.",-4); + r = read(fd,buf,4); + if(r!=4) + err(1,"Can't read last %d bytes of file. Read %d. Stop. ", strlen(AF_SEGTAIL)+1,r); + if(strcmp(buf,AF_SEGTAIL)!=0){ + printf("Does not end with an AF_SEGTAIL. Scanning backwards to find last complete AF_SEGTAIL.\n"); + for(off_t end=len-4;end>4;end--){ + if(lseek(fd,end,SEEK_SET)<0) err(1,"lseek bad=%"I64d,end); + r = read(fd,buf,4); + if(r!=4) err(1,"Can't read 4 bytes at %"I64d); + if(strcmp(buf,AF_SEGTAIL)==0){ + printf("Valid AF_SEGTAIL found at %"I64d" (%"I64d" bytes in)\n",end,len-end); + if(!opt_fix) errx(0,"Rerun with -y flag to fix"); + printf("Truncating at %"I64d"\n",end); + if(ftruncate(fd,end+4)) err(0,"ftruncate"); + break; + } + } + } + close(fd); + exit(0); + + + + AFFILE *af = af_open(infile,O_RDONLY,0); + if(!af) af_err(1,infile); + int fix = 0; + bool fix_add_gid = false; + + if(af_get_seg(af,AF_IMAGE_GID,0,0,0)){ + printf("no GID (%s)",AF_IMAGE_GID); + fix++; + fix_add_gid = true; + } + af_close(af); + if(opt_fix==0 || fix==0) return 0; + + if(fix){ + af = af_open(infile,O_RDWR,0); + if(!af){ + warn(infile); + return -1; + } + if(fix_add_gid) { + printf(" ... adding GID ",infile); + unsigned char bit128[16]; + RAND_pseudo_bytes(bit128,sizeof(bit128)); + if(af_update_seg(af,AF_IMAGE_GID,0,bit128,sizeof(bit128))){ + warn("Cannot write %s: ",AF_IMAGE_GID); + } + } + if(af_close(af)){ + warn("Cannot close %s",infile); + } + } + putchar('\n'); + return 0; +} +#endif + +int main(int argc,char **argv) +{ + int ch; + + setvbuf(stdout,0,_IONBF,0); // turn off buffering + + /* Figure out how many cols the screen has... */ + while ((ch = getopt(argc, argv, "yh?v")) != -1) { + switch (ch) { + case 'y': + opt_fix = 1; + break; + case 'h': + case '?': + default: + usage(); + break; + case 'v': + printf("%s version %s\n",progname,PACKAGE_VERSION); + exit(0); + + } + } + argc -= optind; + argv += optind; + + if(argc<1){ + usage(); + } + + + /* Loop through all of the files */ + while(*argv){ + fix(*argv++); // get the file + argc--; // decrement argument counter + } + exit(0); +} + + diff --git a/tools/affrecover.cpp b/tools/affrecover.cpp new file mode 100644 index 0000000..10ae105 --- /dev/null +++ b/tools/affrecover.cpp @@ -0,0 +1,168 @@ +/* + * afrecover.cpp + * + * Recover broken pages of an AFF file using the party bits + * This file is a work of a US government employee and as such is in the Public domain. + * Simson L. Garfinkel, March 12, 2012 + */ + + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" +#include "utils.h" + +#include + +#include +#include +#include +#include + +#ifdef HAVE_UNISTD_H +#include +#endif + +#ifdef HAVE_TERM_H +#include +#endif + +#ifdef HAVE_NCURSES_TERM_H +#include +#endif + +#ifdef WIN32 +#include "unix4win32.h" +#include +#endif + +using namespace std; +using namespace aff; + +const char *progname = "affix"; + + +int opt_b = 0; + + +void usage() +{ + printf("usage: %s filename\n",progname); + exit(0); +} + + + +int recover(const char *fname) +{ + AFFILE *af = af_open(fname,O_RDWR,0); + if(!af) af_err(1,fname); + + /* Get the parity page */ + size_t pagesize = af_page_size(af); + u_char *pagebuf = (unsigned char *)calloc(pagesize,1); + u_char *parity_buf = (unsigned char *)calloc(pagesize,1); + u_char *my_parity_buf = (unsigned char *)calloc(pagesize,1); + + + if(af_get_seg(af,AF_PARITY0,0,parity_buf,&pagesize)){ + err(1,"Cannot read %s segment; cannot continue",AF_PARITY0); + } + + /* Now, for every page: + * 1. Read the page & the signature + * 2. If the signature is good, add it into the parity buffer. + * - If not, put it on the list of bad segments. + */ + seglist segments(af); + seglist bad_sigs; + seglist good_sigs; + for(seglist::const_iterator seg = segments.begin(); + seg != segments.end(); + seg++){ + + if (seg->pagenumber()<0) continue; // only look for pages + switch(af_sig_verify_seg(af,seg->name.c_str())){ + case AF_ERROR_SIG_NO_CERT: + errx(1,"%s: no public key in AFF file\n",af_filename(af)); + case AF_ERROR_SIG_READ_ERROR: + errx(1,"no signature for segment '%s' --- recovery cannot continue",seg->name.c_str()); + case AF_ERROR_SIG_BAD: + printf("%s has a bad signature\n",af_filename(af)); + bad_sigs.push_back(*seg); + break; + case AF_SIG_GOOD: + good_sigs.push_back(*seg); + /* While the page is in the cache, make our parity buf */ + pagesize = af_page_size(af); + if(af_get_page(af,seg->pagenumber(),pagebuf,&pagesize)){ + err(1,"cannot read %s\n",seg->name.c_str()); + } + for(u_int i=0;i1) errx(1,"This program can only repair 1 bad page at the moment."); + if(bad_sigs.size()==0) errx(1,"There are no bad pages for this program to repair."); + printf("Attempting to repair %s\n",bad_sigs[0].name.c_str()); + + /* Calculate the page buf */ + for(u_int i=0;i + +#ifdef HAVE_REGEX_H +extern "C" { +#include +} +#endif + +#include +#include +#include +#include + +#ifdef HAVE_CSTRING +#include +#endif + +using namespace std; + + +const char *progname = "afsegment"; + +int opt_create = 0; +int opt_quad = 0; +int opt_arg = 0; +int opt_verbose = 0; +int filecount = 0; +int opt_debug = 0; +int opt_x = 0; + +void usage() +{ + printf("afsegment version %s\n",PACKAGE_VERSION); +#ifdef REG_EXTENDED + printf("usage: afsegment [options] file1.aff [file2.aff ...]\n"); + printf("options:\n"); + printf(" -c Create AFF files if they do not exist\n"); + printf(" -ssegval Sets the value of a segment; may be repeated\n"); + printf(" -psegname Prints the contents of the segment name for each file\n"); + printf(" -V Just print the version number and exit.\n"); + printf(" -dname Delete segment 'name'\n"); + printf(" -h, -? Print this message\n"); + printf(" -Q interpert 8-byte segments as a 64-bit value\n"); + printf(" -A Print the 32-bit arg, not the segment value\n"); + printf(" -x Print the segment as a hex string\n"); + printf("\n"); + printf("Values for segval:\n"); + printf("\n"); + printf("Setting the segment values:\n"); + printf(" -sname=- Take the new value of segment 'name' from stdin\n"); + printf(" -sname=val Sets segment 'name' to be 'val' \n"); + printf(" -sname=0){ + value = (u_char *)realloc(value,value_len+count); + memcpy(value+value_len,buf,count); + value_len += count; + } + } + int r = af_update_seg(af,segname,arg,value,value_len); + free(value); + return r; +} + + +void update_segment(AFFILE *af,const char *segname, + const char *argstr,const char *segval) +{ + uint32_t arg = 0; + + if(strlen(argstr)>1) arg = atoi(argstr+1); + + if(!strcmp(segval,"=-")){ + get_segment_from_file(af,segname,arg,stdin); + return; + } + if(!strncmp(segval,"=<",2)){ + FILE *f = fopen(segval+2,"rb"); + if(!f) err(1,"fopen(%s)",segval+2); + get_segment_from_file(af,segname,arg,f); + fclose(f); + return; + } + segval++; // skip past the "=" + int r = af_update_seg(af,segname,arg,(const u_char *)segval,strlen(segval)); + if(r) warn("af_update(%s,%s) ",af_filename(af),segname); +} + + +char *make_re_string(const char *buf,regmatch_t *match,int num) +{ + int len = match[num].rm_eo - match[num].rm_so; + char *ret = (char *)malloc(len+1); + memcpy(ret,buf+match[num].rm_so,len); + ret[len] = '\000'; + return ret; +} + + +vectordel_segs; +vectornew_segs; +vectorprint_segs; +int flags=O_RDONLY; +int openmode = 0666; +regex_t re; + +void process(const char *fn) +{ + AFFILE *af = af_open(fn,flags,openmode); + if(af){ + vector::iterator i; + + for(i=del_segs.begin();i!=del_segs.end();i++){ + if(af_del_seg(af,i->c_str())){ + warnx("af_del_seg(%s): cannot delete segment '%s' ",fn,i->c_str()); + } + else { + printf("%s: '%s' deleted\n",fn,i->c_str()); + } + } + for(i=new_segs.begin();i!=new_segs.end();i++){ + regmatch_t match[10]; + memset(match,0,sizeof(match)); + if(regexec(&re,i->c_str(),10,match,0)==0){ + char *segname = make_re_string(i->c_str(),match,1); + char *argstr = make_re_string(i->c_str(),match,2); + char *segval = make_re_string(i->c_str(),match,3); + update_segment(af,segname,argstr,segval); + free(segname); + free(argstr); + free(segval); + } + } + for(i=print_segs.begin();i!=print_segs.end();i++){ + size_t len = 0; + const char *segname = i->c_str(); + if(opt_debug) fprintf(stderr," %s: \n",segname); + unsigned char *buf=0; + if(af_get_seg(af,segname,0,0,&len)){ +#if HAVE_ISATTY + if(isatty(fileno(stdout))){ + fprintf(stderr,"%s: segment %s not found\n",fn,segname); + continue; + } +#endif + if(opt_debug) fprintf(stderr," <>\n"); + continue; + } + + buf = (u_char *)malloc(len+1); + if(!buf) err(1,"malloc"); + uint32_t arg = 0; + buf[len] = 0; + if(af_get_seg(af,segname,&arg,buf,&len)){ + af_err(1,"af_get_seg"); // this shoudln't fail here + free(buf); + continue; + } + if(opt_debug) fprintf(stderr," arg=%"PRIu32" len=%zd\n",arg,len); + int p = 1; + + if(filecount>1) printf("%s:",fn); + if(print_segs.size()>1) printf("%s=",segname); + if(opt_quad && len==8){ + uint64_t quad = af_decode_q(buf); + printf("%"I64u"\n",quad); + p = 0; + } + + if(opt_arg){ + printf("%"PRIu32"\n",arg); + p = 0; + } + + if(p){ + for(u_int i=0;i1) printf("\n"); + fflush(stdout); + if(buf) free(buf); + } + af_close(af); + if(opt_x) printf("\n"); + } + else { + af_err(1,"af_open(%s) failed:",fn); + } +} + +int main(int argc,char **argv) +{ + + int ch; + while ((ch = getopt(argc, argv, "cd:Vp:s::h?QADx")) != -1) { + switch (ch) { + case 'c': + flags |= O_CREAT; + openmode = 0666; + break; + case 'Q': opt_quad=1;break; + case 'A': opt_arg=1;break; + case 'd': + if(optarg==0) usage(); + del_segs.push_back(optarg); flags |= O_RDWR;break; + case 'D': + opt_debug=1; + break; + case 'p': + if(optarg==0) usage(); + print_segs.push_back(optarg); break; + case 's': + if(optarg==0) usage(); + if(strlen(optarg)==0) usage(); + new_segs.push_back(optarg); flags |= O_RDWR;break; + case 'x': + opt_x++; + break; + case 'h': + case '?': + default: + usage(); + exit(0); + case 'V': + printf("%s version %s\n",progname,PACKAGE_VERSION); + exit(0); + } + } + argc -= optind; + argv += optind; + + if(argc<1){ + usage(); + } + + if(regcomp(&re,"([^/=]*)(/[0-9]+)?(=.*)?",REG_EXTENDED|REG_ICASE)){ + err(1,"regcomp"); + } + + filecount = argc; + while(*argv){ + fflush(stdout); + if(opt_debug) fprintf(stderr,"%s:\n",*argv); + process(*argv); + argv++; + argc--; + } + exit(0); +} +#else +int main(int argc,char **argv) +{ + usage(); + exit(1); +} +#endif diff --git a/tools/affsign.cpp b/tools/affsign.cpp new file mode 100644 index 0000000..fd35671 --- /dev/null +++ b/tools/affsign.cpp @@ -0,0 +1,315 @@ +/* + * afsign.cpp: + * + * Sign an existing AFF file. + * This file is a work of a US government employee and as such is in the Public domain. + * Simson L. Garfinkel, March 12, 2012 + */ + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" + +#ifdef USE_AFFSIGS + +#include "utils.h" +#include "base64.h" + +#include +#include +#include +#include +#include +#include + + +#include "aff_bom.h" + +int opt_note = 0; +const char *opt_sign_key_file = 0; +const char *opt_sign_cert_file = 0; + +using namespace std; +using namespace aff; + +const char *progname = "afsign"; + +void usage() +{ + printf("%s version %s\n",progname,PACKAGE_VERSION); + printf("usage: %s [options] filename.aff\n",progname); + printf("This program will:\n"); + printf(" * Sign each segment if there are no segment signatures.\n"); + printf(" * Write signed chain-of-custody Bill of Materials segment.\n"); + printf("\nSignature Options:\n"); + printf(" -k filename.key = specify private key for signing\n"); + printf(" -c filename.cer = specify a X.509 certificate that matches the private key\n"); + printf(" (by default, the file is assumed to be the same one\n"); + printf(" provided with the -k option.)\n"); + printf(" -Z = ZAP (remove) all signature segments.\n"); + printf("options:\n"); + printf(" -n --- ask for a chain-of-custody note.\n"); + printf(" -v --- Just print the version number and exit.\n"); + exit(0); +} + + +int afsign(const char *fn) +{ + AFFILE *af = af_open(fn,O_RDWR,0); + if(!af) af_err(1,"%s",fn); + + struct af_vnode_info vni; + if(af_vstat(af,&vni)) err(1,"af_vstat"); + + if(vni.supports_metadata==0){ + /* If it is a raw file, we can create an AFM file to sign */ + if(vni.is_raw==0) errx(1,"%s: file does not support metadata. Cannot sign\n",fn); + af_close(af); // afm will open it + char afmfile[MAXPATHLEN+1]; + char file000[MAXPATHLEN+1]; + char extension[MAXPATHLEN+1]; + strcpy(afmfile,fn); + char *period = strrchr(afmfile,'.'); + if(!period) errx(1,"%s: file does not support metadata and lacks a file extension,\n" + "which is needed to create an AFM file '%s\n",afmfile,fn); + strcpy(extension,period+1); // get the extension + + /* If the file being opened is not a .000 file, and a .000 file exists, do not proceed */ + strcpy(period,".000"); + strcpy(file000,afmfile); // make the 000 file + strcpy(period,".afm"); + + if(strcmp(extension,"000")!=0){ + if(access(file000,F_OK)==0){ + errx(1,"Can't create .afm file because %s exists.\n",file000); + } + } + + strcpy(period,".afm"); // we are now going to make an afm file + af = af_open(afmfile,O_RDWR|O_CREAT,0600); + if(!af) af_err(1,"%s: file does not support metadata and cannot create AFM file '%s\n",fn,afmfile); + if(strcmp(extension,"000")!=0){ + af_update_seg(af,AF_RAW_IMAGE_FILE_EXTENSION,0,(const u_char *)extension,strlen(extension)); + af_close(af); + unlink(file000); // get rid of that .000 file + af = af_open(afmfile,O_RDWR,0600); + if(!af) af_err(1,"%s: Created AFM file but cannot re-open it\n",fn); + /* Read the first byte to force a call to afm_split_raw_setup(). + * The results of the read don't matter, but we better be able to read. + */ + u_char buf[1]; + if(af_read(af,buf,1)!=1){ + err(1,"Cannot read first byte of %s",fn); + } + af_seek(af,0L,0); + } + } + + seglist segments(af); + + if(isatty(fileno(stdout))){ + printf("Signing segments...\n"); + fflush(stdout); + } + + bool signed_unsigned_segments = false; + if(segments.has_signed_segments()==false){ + if(af_set_sign_files(af,opt_sign_key_file,opt_sign_cert_file)){ + errx(1,"key file '%s' or certificate file '%s' is invalid", + opt_sign_key_file,opt_sign_cert_file); + } + int r = af_sign_all_unsigned_segments(af); + if(r<0) af_err(1,"%s: all unsigned segments cannot be signed.",fn); + if(r>0) signed_unsigned_segments = true; + } + + aff_bom bom(opt_note); + if(bom.read_files(opt_sign_cert_file,opt_sign_key_file)) err(1,"Can't read signature files???"); + + u_char *pagebuf = (unsigned char *)calloc(af_page_size(af),1); + u_char *parity_buf = (unsigned char *)calloc(af_page_size(af),1); + bool compute_parity = true; // do we need to compute the parity? + + /* Create the parity buffer if it doesn't exist. If the parity buffer exists, we'll just trust it. + * We could do a two-pass here, one for creating the parity buffer, another for creating the BOM. + * But that would require reading the data twice; hence this extra layer of complexity. + */ + size_t parity_buf_len = af_page_size(af); + if(af_get_seg(af,AF_PARITY0,0,parity_buf,&parity_buf_len)==0){ + compute_parity = false; // no need to compute it; we read it + } + + for(seglist::const_iterator seg = segments.begin(); seg!= segments.end();seg++){ + const char *segname = seg->name.c_str(); + + if(isatty(fileno(stdout))){ + printf("\rCalculating BOM for segment %s... ",segname); + printf("\n"); + fflush(stdout); + } + + u_char seghash[32]; /* resultant message digest; could be any size */ + unsigned int seghash_len = sizeof(seghash); /* big enough to hold SHA256 */ + int sigmode = 0; + int64_t pagenumber = af_segname_page_number(segname); + if(pagenumber>=0){ + /* Page segments must run in SIGNATURE_MODE1 - the actual data in the page */ + size_t this_pagesize = af_page_size(af); + if(af_get_page(af,pagenumber,pagebuf,&this_pagesize)){ + free(pagebuf); + return -1; + } + /* Add to parity buf if we are making a parity page*/ + if(compute_parity){ + for(u_int i=0;iname.c_str()) || i->name==AF_SIGN256_CERT){ + cout << "Deleting " << i->name << "\n"; + af_del_seg(af,i->name.c_str()); + } + } + af_close(af); + return 0; +} + +int main(int argc,char **argv) +{ + int bflag, ch; + int opt_zap = 0; + + bflag = 0; + while ((ch = getopt(argc, argv, "nk:c:h?vZ")) != -1) { + switch (ch) { + case 'n': opt_note = 1;break; + case 'k': + if(access(optarg,R_OK)) err(1,"%s",optarg); + opt_sign_key_file = optarg; + break; + case 'c': + if(access(optarg,R_OK)) err(1,"%s",optarg); + opt_sign_cert_file = optarg; + break; + case 'v': + printf("%s version %s\n",progname,PACKAGE_VERSION); + exit(0); + case 'Z': + opt_zap = 1; + break; + case 'h': + case '?': + default: + usage(); + break; + } + } + argc -= optind; + argv += optind; + + if(opt_sign_cert_file==0) opt_sign_cert_file=opt_sign_key_file; // if not set, make same as key file + + + if(argc!=1){ + usage(); + } + + if(opt_zap) return remove_signatures(argv[0]); + + /* We either need both a key file and a cert file, or neither */ + if((opt_sign_key_file==0) || (opt_sign_cert_file==0)){ + errx(1,"Both a private key and a certificate must be specified."); + } + + + return afsign(argv[0]); +} +#else +int main(int argc,char **argv) +{ + fprintf(stderr,"afflib compiled without USE_AFFSIGS. afsign cannot run.\n"); + exit(-1); +} + +#endif + diff --git a/tools/affstats.cpp b/tools/affstats.cpp new file mode 100644 index 0000000..15383fd --- /dev/null +++ b/tools/affstats.cpp @@ -0,0 +1,204 @@ +/* + * afstats.cpp: + * + * print specific statistics about one or more AFF files. + * Ideally, we can get the stats from the metadata, but this program will + * calculate it if necessary. + */ + +/* + * Copyright (c) 2005-2006 + * Simson L. Garfinkel and Basis Technology, Inc. + * All rights reserved. + * + * This code is derrived from software contributed by + * Simson L. Garfinkel + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. [omitted] + * 4. Neither the name of Simson Garfinkel, Basis Technology, or other + * contributors to this program may be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY SIMSON GARFINKEL, BASIS TECHNOLOGY, + * AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL SIMSON GARFINKEL, BAIS TECHNOLOGy, + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" + +#include +#include +#include +#include + +#ifdef HAVE_ERR_H +#include +#endif + +#ifdef HAVE_UNISTD_H +#include +#endif + +#ifdef HAVE_UNISTD_H +#include +#endif + +#ifdef HAVE_TERM_H +#include +#endif + +#ifdef HAVE_NCURSES_TERM_H +#include +#endif + +#ifdef HAVE_NCURSES_H +#include +#endif + +#ifdef WIN32 +#include "unix4win32.h" +#include +#endif + +const char *progname = "afstats"; +int opt_m = 0; + +void usage() +{ + printf("%s version %s\n\n",progname,PACKAGE_VERSION); + printf("usage: %s [options] infile(s)\n",progname); + printf(" -m = print all output in megabytes\n"); + printf(" -v = Just print the version number and exit.\n"); + exit(0); +} + + +void title() +{ + printf("fname\tbytes\tcompressed\n"); +} + +void print_size(uint64_t s) +{ + if(opt_m){ + printf("%u",(unsigned int)(s/(1024*1024))); + return; + } + printf("%"I64u,s); +} + +void afstats_title() +{ + printf("Name\tAF_IMAGESIZE\tCompressed\tUncompressed\tBlank\tBad\n"); +} + +void afstats(const char *fname) +{ + AFFILE *af = af_open(fname,O_RDONLY,0); + if(!af) af_err(1,"af_open(%s)",fname); + + printf("%s\t",fname); + + uint32_t segsize=0; + + int64_t imagesize=0; + int64_t blanksectors=0; + int64_t badsectors=0; + af_get_segq(af,AF_IMAGESIZE,&imagesize); + if(af_get_seg(af,AF_PAGESIZE,&segsize,0,0)){ + af_get_seg(af,AF_SEGSIZE_D,&segsize,0,0); // check for oldstype + } + af_get_segq(af,AF_BADSECTORS,&badsectors); + af_get_segq(af,AF_BLANKSECTORS,&blanksectors); + + print_size(imagesize); + printf("\t"); + fflush(stdout); + + int64_t compressed_bytes = 0; + int64_t uncompressed_bytes = 0; + + /* Now read through all of the segments and count the number of + * data segments. We know the uncompressed size... + */ + af_rewind_seg(af); + char segname[AF_MAX_NAME_LEN+1]; + size_t datalen; + while(af_get_next_seg(af,segname,sizeof(segname),0,0,&datalen)==0){ + int64_t page_num = af_segname_page_number(segname); + if(page_num>=0){ + compressed_bytes += datalen; + uncompressed_bytes += segsize; + } + } + if(uncompressed_bytes > imagesize) uncompressed_bytes = imagesize; + + print_size(compressed_bytes); + printf("\t"); + print_size(uncompressed_bytes); + printf(" %"I64d" %"I64d,blanksectors,badsectors); + putchar('\n'); + + +} + + + + +int main(int argc,char **argv) +{ + int ch; + while ((ch = getopt(argc, argv, "mh?V")) != -1) { + switch (ch) { + case 'm': + opt_m = 1; + break; + case 'h': + case '?': + default: + usage(); + break; + case 'V': + printf("%s version %s\n",progname,PACKAGE_VERSION); + exit(0); + } + } + argc -= optind; + argv += optind; + + if(argc<1){ + usage(); + } + + /* Each argument is now a file. Process each one */ + afstats_title(); + while(*argv){ + afstats(*argv++); + argc--; + } + exit(0); +} + + diff --git a/tools/affuse.c b/tools/affuse.c new file mode 100644 index 0000000..2b3395d --- /dev/null +++ b/tools/affuse.c @@ -0,0 +1,244 @@ +/************************************************************ + * + * (c) 2007 Olivier Castan castan.o@free.fr + * Modified by Simson Garfinkel, to fit into the AFFLIB build system. + * + * License: LGP + * + * KISS: based on fuse hello.c example + * + * TODO: - use xattr to display informations from segments + * - use AF_ACQUISITION_DATE for creation date + * - option between BADFLAG and NULLs + * - ... + * + * *********************************************************/ + +#if HAVE_CONFIG_H +#include "affconfig.h" +#endif + +#include +#include +#include +#include + +#ifdef USE_FUSE + +/* bool used in afflib.h but not defined within C */ +#ifndef bool +#define bool int +#endif +#include "afflib.h" +#include +#include +#include + + +#define XCALLOC(type, num) \ + ((type *) xcalloc ((num), sizeof(type))) +#define XMALLOC(type, num) \ + ((type *) xmalloc ((num) * sizeof(type))) +#define XFREE(stale) do { \ + if (stale) { free ((void *) stale); stale = 0; } \ + } while (0) + + + +static char *raw_path = NULL; +static off_t raw_size = 0; +static AFFILE *af_image = NULL; +static const char *raw_ext = ".raw"; + +static void * +xmalloc (size_t num) +{ + void *alloc = malloc (num); + if (!alloc) { + perror ("Memory exhausted"); + exit(EXIT_FAILURE); + } + return alloc; +} + +static void * +xcalloc (size_t num, size_t size) +{ + void *alloc = xmalloc (num * size); + memset (alloc, 0, num * size); + return alloc; +} + +static char * +xstrdup(char *string) +{ + return strcpy((char *)xmalloc(strlen(string) + 1), string); +} + +static int +affuse_getattr(const char *path, struct stat *stbuf) +{ + int res = 0; + + memset(stbuf, 0, sizeof(struct stat)); + if(strcmp(path, "/") == 0) { + stbuf->st_mode = S_IFDIR | 0755; + stbuf->st_nlink = 2; + } + else if(strcmp(path, raw_path) == 0) { + stbuf->st_mode = S_IFREG | 0444; + stbuf->st_nlink = 1; + stbuf->st_size = raw_size; + } + else + res = -ENOENT; + + return res; +} + +static int +affuse_readdir(const char *path, void *buf, fuse_fill_dir_t filler, + off_t offset, struct fuse_file_info *fi) +{ + (void) offset; + (void) fi; + + if(strcmp(path, "/") != 0) + return -ENOENT; + + filler(buf, ".", NULL, 0); + filler(buf, "..", NULL, 0); + filler(buf, raw_path + 1, NULL, 0); + + return 0; +} + +static int +affuse_open(const char *path, struct fuse_file_info *fi) +{ + if(strcmp(path, raw_path) != 0) + return -ENOENT; + + if((fi->flags & 3) != O_RDONLY) + return -EACCES; + + return 0; +} + +static int +affuse_read(const char *path, char *buf, size_t size, off_t offset, + struct fuse_file_info *fi) +{ + int res = 0; + (void) fi; + if(strcmp(path, raw_path) != 0){ + return -ENOENT; + } + + /* TODO: change to sector aligned readings to write NULLs to bad + * blocks... */ + /* looks like af_seek never fails */ + af_seek(af_image, (uint64_t)offset, SEEK_SET); + errno = 0; + res = af_read(af_image, (unsigned char *)buf, (int)size); + if (res<0){ + if (errno==0) errno=-EIO; + else res = -errno; + } + return res; +} + +static void +affuse_destroy(void* param) +{ + af_close(af_image); + XFREE(raw_path); + return; +} + +static struct fuse_operations affuse_oper = { + .getattr = affuse_getattr, + .readdir = affuse_readdir, + .open = affuse_open, + .read = affuse_read, + .destroy = affuse_destroy, +}; + +static void +usage(void) +{ + char *cmdline[] = {"affuse", "-ho"}; + printf("affuse version %s\n", PACKAGE_VERSION); + printf("Usage: affuse [] af_image mount_point\n"); + /* dirty, just to get current libfuse option list */ + fuse_main(2, cmdline, &affuse_oper, NULL); + printf("\nUse fusermount -u mount_point, to unmount\n"); +} + +int main(int argc, char **argv) +{ + char *af_path = NULL, *af_basename = NULL; + size_t raw_path_len = 0; + char **fargv = NULL; + int fargc = 0; + + if (argc < 3) { + usage(); + exit(EXIT_FAILURE); + } + + /* Prepare fuse args, af_image is omitted, but "-s" is added */ + fargv = XCALLOC(char *, argc); /* usually not free'd */ + fargv[0] = argv[0]; + fargv[1] = argv[argc - 1]; + fargc = 2; + while (fargc <= (argc - 2)) { + fargv[fargc] = argv[fargc - 1]; + if (strcmp(fargv[fargc], "-h") == 0 || + strcmp(fargv[fargc], "--help") == 0 ) { + usage(); + XFREE(fargv); + exit(EXIT_SUCCESS); + } + fargc++; + } + /* disable multi-threaded operation + * (we don't know if afflib is thread safe!) + */ + fargv[fargc] = "-s"; + fargc++; + + if ((af_image = af_open(argv[argc - 2], O_RDONLY|O_EXCL, 0)) == NULL) { + perror("Can't open image file"); + XFREE(fargv); + exit(EXIT_FAILURE); + } + + af_path = xstrdup(argv[argc - 2]); + af_basename = basename(af_path); + /* "/" af_basename raw_ext "/0"*/ + raw_path_len = 1 + strlen(af_basename) + strlen(raw_ext) + 1; + raw_path = XCALLOC(char, raw_path_len); + raw_path[0] = '/'; + strcat(raw_path, af_basename); + strcat(raw_path, raw_ext); + raw_path[raw_path_len -1] = 0; + XFREE(af_path); + raw_size = af_get_imagesize(af_image); + + return fuse_main(fargc, fargv, &affuse_oper, NULL); +} +#else +int main(int argc,char **argv) +{ + fprintf(stderr,"affuse: FUSE support is disabled.\n"); +#ifndef linux + fprintf(stderr,"affuse was compiled on a platform that does not support FUSE\n"); +#else + fprintf(stderr,"affuse was compiled on a Linux system that did not\n"); + fprintf(stderr,"have the FUSE developer libraries installed\n"); + fprintf(stderr,"You need to install the fuse-devl package.\n"); +#endif + exit(1); +} +#endif diff --git a/tools/affverify.cpp b/tools/affverify.cpp new file mode 100644 index 0000000..f33336f --- /dev/null +++ b/tools/affverify.cpp @@ -0,0 +1,545 @@ +/* + * afverify.cpp: + * + * Verify the digital signature on a signed file + * This file is a work of a US government employee and as such is in the Public domain. + * Simson L. Garfinkel, March 12, 2012 + */ + +#ifdef WIN32 +# include +# include +#endif + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" + +#include "utils.h" +#include "base64.h" +#include "aff_bom.h" +#include "aftimer.h" + +#include +#include +#include +#include +#include +#include + +using namespace std; +using namespace aff; + +const char *progname = "afcrypto"; +int opt_change = 0; +int opt_verbose = 0; +int opt_all = 0; + +void usage() +{ + printf("afverify version %s\n",PACKAGE_VERSION); + printf("usage: afverify [options] filename.aff\n"); + printf("Verifies the digital signatures on a file\n"); + printf("options:\n"); + printf(" -a --- print all segments\n"); + printf(" -V --- Just print the version number and exit.\n"); + printf(" -v --- verbose\n"); + + OpenSSL_add_all_digests(); + const EVP_MD *sha256 = EVP_get_digestbyname("sha256"); + if(sha256){ + printf(" SHA256 is operational\n"); + } else { + printf("Warning: EVP_get_digestbyname(\"sha256\") fails\n"); + } + exit(0); +} + +void print_x509_info(X509 *cert) +{ + printf("SIGNING CERTIFICATE :\n"); + printf(" Subject: "); X509_NAME_print_ex_fp(stdout,X509_get_subject_name(cert),0,XN_FLAG_SEP_CPLUS_SPC); + printf("\n"); + printf(" Issuer: "); X509_NAME_print_ex_fp(stdout,X509_get_issuer_name(cert),0,XN_FLAG_SEP_CPLUS_SPC); + printf("\n"); + ASN1_INTEGER *sn = X509_get_serialNumber(cert); + if(sn){ + long num = ASN1_INTEGER_get(sn); + if(num>0) printf(" Certificate serial number: %ld\n",num); + } + printf("\n"); +} + +#ifdef USE_AFFSIGS +#include "expat.h" +void startElement(void *userData, const char *name, const char **atts); +void endElement(void *userData, const char *name); +void cHandler(void *userData,const XML_Char *s,int len); + +class segmenthash { +public: + segmenthash():total_validated(0),total_invalid(0),sigmode(0),in_cert(false), + in_seghash(false),get_cdata(false),arg(0),seglen(0), + get_cdata_segment(0),af(0),cert(0),pubkey(0) { + + parser = XML_ParserCreate(NULL); + XML_SetUserData(parser, this); + XML_SetElementHandler(parser, ::startElement, ::endElement); + XML_SetCharacterDataHandler(parser,cHandler); + }; + int parse(const char *buf,int len) { return XML_Parse(parser, buf, len, 1);} + XML_Parser parser; + int total_validated; + int total_invalid; + int sigmode; + bool in_cert; + bool in_seghash; + bool get_cdata; + string segname; + string alg; + string cdata; + int arg; + int seglen; + const char *get_cdata_segment; // just get this segment + AFFILE *af; // if set, we are parsing crypto + X509 *cert; // public key used to sign + EVP_PKEY *pubkey; + void clear(){ + segname=""; + cdata=""; + sigmode=0; + alg=""; + seglen=0; + } + ~segmenthash(){ + if(cert) X509_free(cert); + if(parser) XML_ParserFree(parser); + } + void startElement(const char *name,const char **atts); + void endElement(const char *name); +}; + +int count=0; +void startElement(void *userData, const char *name, const char **atts) +{ + segmenthash *sh = (segmenthash *)userData; + sh->startElement(name,atts); +} + +void segmenthash::startElement(const char *name,const char **atts) +{ + clear(); + if(strcmp(name,AF_XML_SEGMENT_HASH)==0){ + for(int i=0;atts[i];i+=2){ + const char *name = atts[i]; + const char *value = atts[i+1]; + if(!strcmp(name,"segname")) segname = value; + if(!strcmp(name,"sigmode")) sigmode = atoi(value); + if(!strcmp(name,"alg")) alg = value; + if(!strcmp(name,"seglen")) seglen = atoi(value); + } + in_seghash = true; + get_cdata = true; + return; + } + if(strcmp(name,"signingcertificate")==0){ + in_cert = true; + get_cdata = true; + return; + } + if(get_cdata_segment && strcmp(name,get_cdata_segment)==0){ + get_cdata = true; + return; + } +} + +void cHandler(void *userData,const XML_Char *s,int len) +{ + segmenthash *sh = (segmenthash *)userData; + if(sh->get_cdata==false) return; // don't want cdata + sh->cdata.append(s,len); +} + +void endElement(void *userData, const char *name) +{ + segmenthash *sh = (segmenthash *)userData; + sh->endElement(name); +} + + +void segmenthash::endElement(const char *name) +{ + if(get_cdata_segment && strcmp(name,get_cdata_segment)==0){ + get_cdata = false; + XML_StopParser(parser,0); + return; + } + if(in_seghash && af){ + if(segname.size()==0) return; // don't have a segment name + /* Try to validate this one */ + size_t hashbuf_len = cdata.size() + 2; + u_char *hashbuf = (u_char *)malloc(hashbuf_len); + hashbuf_len = b64_pton_slg((char *)cdata.c_str(),cdata.size(),hashbuf,hashbuf_len); + if(alg=="sha256"){ + /* TODO: Don't re-validate something that's already validated */ + int r = af_hash_verify_seg2(af,segname.c_str(),hashbuf,hashbuf_len,sigmode); + if(r==AF_HASH_VERIFIES){ + total_validated++; + } + else total_invalid++; + } + free(hashbuf); + in_seghash = false; + } + if(in_cert && af){ + BIO *cert_bio = BIO_new_mem_buf((char *)cdata.c_str(),cdata.size()); + PEM_read_bio_X509(cert_bio,&cert,0,0); + BIO_free(cert_bio); + pubkey = X509_get_pubkey(cert); + in_cert = false; + } + cdata = ""; // erase it +} + +string get_xml_field(const char *buf,const char *field) +{ + segmenthash sh; + sh.get_cdata_segment = field; + sh.parse(buf,strlen(buf)); + return sh.cdata; +} + +/* verify the chain signature; return 0 if successful, -1 if failed. + * The signature is a block of XML with a base64 encoded at the end. + */ +int verify_bom_signature(AFFILE *af,const char *buf) +{ + OpenSSL_add_all_digests(); + const EVP_MD *sha256 = EVP_get_digestbyname("sha256"); + + if(!sha256){ + fprintf(stderr,"OpenSSL does not have SHA256; signatures cannot be verified.\n"); + return -1; + } + + const char *cce = "\n"; + const char *chain_end = strstr(buf,cce); + if(!chain_end){ + warn("end of chain XML can't be found\n"); + return -1; // can't find it + } + const char *sig_start = chain_end + strlen(cce); + + BIO *seg = BIO_new_mem_buf((void *)buf,strlen(buf)); + if(BIO_seek(seg,0)!=0){ + printf("Cannot seek to beginning of BIO mem?"); + return -1; + } + X509 *cert = 0; + PEM_read_bio_X509(seg,&cert,0,0); // get the contained x509 cert + BIO_free(seg); + + /* Now get the binary signature */ + u_char sigbuf[1024]; + int sigbuf_len = b64_pton_slg(sig_start,strlen(sig_start),sigbuf,sizeof(sigbuf)); + if(sigbuf_len<80){ + warn("BOM is not signed"); + return -1; + } + + /* Try to verify it */ + EVP_MD_CTX md; + EVP_VerifyInit(&md,sha256); + EVP_VerifyUpdate(&md,buf,sig_start-buf); + int r = EVP_VerifyFinal(&md,sigbuf,sigbuf_len,X509_get_pubkey(cert)); + if(r!=1){ + printf("BAD SIGNATURE ON BOM\n"); + return -1; + } + + print_x509_info(cert); + printf("Date: %s\n",get_xml_field(buf,"date").c_str()); + printf("Notes: \n%s\n",get_xml_field(buf,"notes").c_str()); + + /* Now extract the XML block, terminating at the beginning of the XML signature */ + char *buffer_without_signature = strdup(buf); + char *sigend = strstr(buffer_without_signature,cce); + if(sigend){ + sigend[strlen(cce)] = 0;/* terminate the XML to remove the signature */ + } + + segmenthash sh; + sh.af = af; + if (!sh.parse(buffer_without_signature, strlen(buffer_without_signature))){ + fprintf(stderr, "expat error: %s at line %d\n", + XML_ErrorString(XML_GetErrorCode(sh.parser)), + (int)XML_GetCurrentLineNumber(sh.parser)); + fprintf(stderr,"buffer without signature:\n%s\n",buffer_without_signature); + return 1; + } + free(buffer_without_signature); + return 0; +} +#endif + +int crypto_verify(AFFILE *af,u_char *certbuf,size_t certbuf_len) +{ + + seglist segments(af); + seglist no_sigs; + seglist bad_sigs; + seglist good_sigs; + seglist unknown_errors; + + for(seglist::const_iterator seg = segments.begin(); + seg != segments.end(); + seg++){ + + if(parse_chain(seg->name)>=0) continue; // chain of custody segments don't need signatures + + const char *segname = seg->name.c_str(); + int i =af_sig_verify_seg(af,segname); + if(opt_verbose){ + printf("af_sig_verify_seg(af,%s)=%d\n",segname,i); + } + switch(i){ + case AF_ERROR_SIG_NO_CERT: + err(1,"%s: no public key in AFF file\n",af_filename(af)); + case AF_ERROR_SIG_BAD: + bad_sigs.push_back(*seg); + break; + case AF_ERROR_SIG_READ_ERROR: + no_sigs.push_back(*seg); + break; + case AF_SIG_GOOD: + good_sigs.push_back(*seg); + break; + case AF_ERROR_SIG_SIG_SEG: + break; // can't verify the sig on a sig seg + case AF_ERROR_SIG_NOT_COMPILED: + errx(1,"AFFLIB was compiled without signature support. Cannot continue.\n"); + default: + unknown_errors.push_back(*seg); + break; + } + } + const char *prn = ""; + /* Tell us something about the certificate */ + BIO *cert_bio = BIO_new_mem_buf(certbuf,certbuf_len); + X509 *cert = 0; + PEM_read_bio_X509(cert_bio,&cert,0,0); + if(!cert) errx(1,"Cannot decode certificate"); + printf("\n"); + printf("Filename: %s\n",af_filename(af)); + printf("# Segments signed and Verified: %d\n",(int)good_sigs.size()); + printf("# Segments unsigned: %d\n",(int)no_sigs.size()); + printf("# Segments with corrupted signatures: %d\n",(int)bad_sigs.size()); + printf("\n"); + print_x509_info(cert); + + int compromised = 0; + for(seglist::const_iterator seg = good_sigs.begin(); seg != good_sigs.end() && opt_all; + seg++){ + if(*seg==good_sigs.front()) printf("%sSegments with valid signatures:\n",prn); + printf("\t%s\n",seg->name.c_str()); + prn = "\n"; + } + for(seglist::const_iterator seg = no_sigs.begin(); + seg != no_sigs.end(); + seg++){ + if(*seg==no_sigs.front()) printf("%sUnsigned segments:\n",prn); + printf("\t%s\n",seg->name.c_str()); + prn = "\n"; + + /* Only unsigned data segments are a problem */ + if(af_segname_page_number(seg->name.c_str())>=0){ + compromised++; + } + } + for(seglist::const_iterator seg = bad_sigs.begin(); + seg != bad_sigs.end(); + seg++){ + if(*seg==bad_sigs.front()) printf("%sBad signature segments:\n",prn); + printf("\t%s\n",seg->name.c_str()); + prn = "\n"; + compromised++; + } + for(seglist::const_iterator seg = unknown_errors.begin(); + seg != unknown_errors.end(); + seg++){ + if(*seg==unknown_errors.front()) printf("%sUnknown error segments:\n",prn); + printf("\t%s\n",seg->name.c_str()); + prn = "\n"; + compromised++; + } + + int highest = highest_chain(segments); + printf("\nNumber of custody chains: %d\n",highest+1); + for(int i=0;i<=highest;i++){ + /* Now print each one */ + printf("---------------------\n"); + printf("Signed Bill of Material #%d:\n\n",i+1); + + /* Get the segment and verify */ + size_t chainbuf_len = 0; + char segname[AF_MAX_NAME_LEN]; + snprintf(segname,sizeof(segname),AF_BOM_SEG,i); + if(af_get_seg(af,segname,0,0,&chainbuf_len)){ + printf("*** BOM MISSING ***\n"); + compromised++; + } + char *chainbuf = (char *)malloc(chainbuf_len+1); + if(af_get_seg(af,segname,0,(u_char *)chainbuf,&chainbuf_len)){ + printf("*** CANNOT READ BOM ***\n"); + compromised++; + } + + chainbuf[chainbuf_len]=0; // terminate +#ifdef USE_AFFSIGS + if(verify_bom_signature(af,chainbuf)){ + printf("*** BOM SIGNATURE INVALID ***\n"); + compromised++; + } +#else + printf("BOM signature cannot be verified beause libxpat is not available.\n"); +#endif + } + printf("---------------------\n"); + af_close(af); +#ifdef USE_AFFSIGS + if(compromised){ + printf("\nEVIDENCE FILE DOES NOT VERIFY.\n"); + printf("ERRORS DETECTED: %d\n",compromised); + printf("EVIDENTUARY VALUE MAY BE COMPROMISED.\n"); + return -1; + } + printf("\nEVIDENCE FILE VERIFIES.\n"); + return 0; +#endif + printf("\n"); + return -1; +} + +int hash_verify(AFFILE *af) +{ + /* See if there is a SHA1 segment */ + unsigned char sha1_buf[20]; + unsigned char md5_buf[16]; + char hexbuf[256]; + size_t sha1_len = sizeof(sha1_buf); + size_t md5_len =sizeof(md5_buf); + const EVP_MD *md5_evp = 0; + const EVP_MD *sha1_evp = 0; + EVP_MD_CTX md5,sha1; + if(af_get_seg(af,AF_SHA1,0,sha1_buf,&sha1_len)==0){ + printf("SHA1 stored in file: %s\n",af_hexbuf(hexbuf,sizeof(hexbuf),sha1_buf,sha1_len,0)); + sha1_evp = EVP_get_digestbyname("sha1"); + EVP_DigestInit(&sha1,sha1_evp); + } + if(af_get_seg(af,AF_MD5,0,md5_buf,&md5_len)==0){ + printf("MD5 stored in file: %s\n",af_hexbuf(hexbuf,sizeof(hexbuf),md5_buf,md5_len,0)); + md5_evp = EVP_get_digestbyname("md5"); + EVP_DigestInit(&md5,md5_evp); + } + /* Might as well read this puppy */ + u_char *buf = (u_char *)malloc(af_get_pagesize(af)); + ssize_t readsize = 0; + ssize_t total_read = 0; + af_seek(af,0L,0); + aftimer t; + t.start(); + printf("\n"); + do { + double frac = (double)total_read / af_get_imagesize(af); + printf(" Read %14zd/%14"PRId64" bytes; done in %s\n", + total_read, + af_get_imagesize(af), + t.eta_text(frac).c_str()); + readsize = af_read(af,buf,af_get_pagesize(af)); + if(readsize<1) break; + if(md5_evp) EVP_DigestUpdate(&md5,buf,readsize); + if(sha1_evp) EVP_DigestUpdate(&sha1,buf,readsize); + total_read += readsize; + } while(total_read < af_get_imagesize(af)); + + printf("\n"); + + if(sha1_evp){ + unsigned char sha1_calc[32]; + unsigned int sha1_calc_len = sizeof(sha1_calc); + + EVP_DigestFinal(&sha1,sha1_calc,(unsigned int *)&sha1_calc_len); + printf("Calculated SHA1: %s ",af_hexbuf(hexbuf,sizeof(hexbuf),sha1_calc,sha1_calc_len,0)); + if(memcmp(sha1_buf,sha1_calc,sha1_len)==0){ + printf("VERIFIES\n"); + } else { + printf("INVALID\n"); + } + } + + if(md5_evp){ + unsigned char md5_calc[32]; + unsigned int md5_calc_len = sizeof(md5_calc); + + EVP_DigestFinal(&md5,md5_calc,(unsigned int *)&md5_calc_len); + printf("Calculated MD5: %s ",af_hexbuf(hexbuf,sizeof(hexbuf),md5_calc,md5_calc_len,0)); + if(memcmp(md5_buf,md5_calc,md5_len)==0){ + printf("VERIFIES\n"); + } else { + printf("INVALID\n"); + } + } + + af_close(af); + return 0; +} + +int process(const char *fn) +{ + AFFILE *af = af_open(fn,O_RDONLY,0666); + if(!af) af_err(1,fn); + + /* Get the public key */ + unsigned char certbuf[65536]; + size_t certbuf_len = sizeof(certbuf); + if(af_get_seg(af,AF_SIGN256_CERT,0,certbuf,&certbuf_len)){ + /* See if it is present, but encrypted */ + if(af_get_seg(af,AF_SIGN256_CERT AF_AES256_SUFFIX,0,0,0)==0){ + errx(1,"%s: signed file is encrypted; present decryption key to verify signature",fn); + } + printf("%s: no signing certificate present. \n\n",fn); + return hash_verify(af); + } + return crypto_verify(af,certbuf,certbuf_len); +} + + +int main(int argc,char **argv) +{ + int ch; + + while ((ch = getopt(argc, argv, "ach?vV")) != -1) { + switch (ch) { + case 'a': opt_all = 1; break; + case 'c': opt_change = 1; break; + case 'v': opt_verbose++; break; + case 'h': + case '?': + default: + usage(); + break; + case 'V': + printf("%s version %s\n",progname,PACKAGE_VERSION); + exit(0); + } + } + argc -= optind; + argv += optind; + + if(argc!=1){ + usage(); + } + + OpenSSL_add_all_digests(); + return process(argv[0]); +} diff --git a/tools/affxml.cpp b/tools/affxml.cpp new file mode 100644 index 0000000..f74839a --- /dev/null +++ b/tools/affxml.cpp @@ -0,0 +1,368 @@ +/* + * afxml.cpp: + * + * print AFF information as an XML + */ + +/* + * Copyright (c) 2005-2006 + * Simson L. Garfinkel and Basis Technology, Inc. + * All rights reserved. + * + * This code is derrived from software contributed by + * Simson L. Garfinkel + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. [omitted] + * 4. Neither the name of Simson Garfinkel, Basis Technology, or other + * contributors to this program may be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY SIMSON GARFINKEL, BASIS TECHNOLOGY, + * AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL SIMSON GARFINKEL, BAIS TECHNOLOGy, + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + + +#include "affconfig.h" +#include "afflib.h" +#include "afflib_i.h" +#include "base64.h" + +#ifdef WIN32 +#include "unix4win32.h" +#endif + +#include +#include + +#ifdef HAVE_CSTRING +#include +#endif + +using namespace std; + +#if HAVE_CTYPE_H +#include +#endif + +#if !defined(HAVE_ISALPHANUM) && defined(HAVE_ISALNUM) +#define isalphanum(c) isalnum(c) +#endif + +#if !defined(HAVE_ISALPHANUM) && !defined(HAVE_ISALNUM) +#define isalphanum(c) (isalpha(c)||isdigit(c)) +#endif + +const char *progname = "afxml"; + +int opt_x = 0; +char **opt_j = 0; +int opt_j_count = 0; +int opt_stats = 0; + +struct page_stat_block { + uint32_t long zsectors; // number of sectors that are all blank + uint32_t long badsectors; // number of bad sectors + uint32_t long zpages; // number of pages that are all blank + uint32_t long pages; // total number of pages + uint32_t long sectors; // total number of sectors +}; + + +void usage() +{ + printf("%s version %s\n",progname,PACKAGE_VERSION); + printf("usage: %s [options] infile... \n",progname); + printf(" -V = Just print the version number and exit\n"); + printf(" -x = Don't include the infile filename in output.\n"); + printf(" -j segname = Just print information about segname \n"); + printf(" (may be repeated)\n"); + printf(" -s = output 'stats' for the file data (may a long time)\n"); + exit(0); +} + + + +/* Return true if segname is in the optj list */ +bool in_opt_j_list(char *segname) +{ + for(int i=0;i=127) return false; + } + return true; +} + +bool is_blank(const u_char *buf,size_t len) +{ + for(size_t i=0;i%"I64d"\n\n",name,val,name); +} + +int xml_info(const char *infile) +{ + AFFILE *af = af_open(infile,O_RDONLY,0); + if(!af){ + warn("%s",infile); + return -1; + } + + struct page_stat_block psb; + memset(&psb,0,sizeof(psb)); + + printf("\n",PACKAGE_VERSION); + printf("\n"); + + af_rewind_seg(af); // start at the beginning + + char segname[AF_MAX_NAME_LEN]; + int pages = 0; + vector seglist; // list of segments we will get + vector pagelist; // list of segments we will get + + while(af_get_next_seg(af,segname,sizeof(segname),0,0,0)==0){ + if(segname[0]==0) continue; // segment to ignore + if(strcmp(segname,AF_DIRECTORY)==0) continue; // don't output the directories + if(strstr(segname,AF_AES256_SUFFIX)) continue; // don't output encrypted segments that won't decrypt + + /* check optj */ + if(opt_j_count > 0 && in_opt_j_list(segname)==false){ + continue; + } + + int64_t page_num = af_segname_page_number(segname); + if(page_num>=0){ + pages += 1; + pagelist.push_back(page_num); + } + else { + seglist.push_back(segname); + } + } + + printf(" %d\n",pages); // tell how many pages we have + + /* If we have been asked to create stats, create the states */ + if(opt_stats){ + unsigned char *data= (unsigned char *)malloc(af_page_size(af)); + if(!data) err(1,"Can't allocate page with %d bytes.",af_page_size(af)); + for(vector::const_iterator it = pagelist.begin(); it != pagelist.end(); it++){ + size_t pagesize = af_page_size(af); + size_t sectorsize = af_get_sectorsize(af); + if(af_get_page(af,*it,data,&pagesize)){ + err(1,"Can't read page %"PRId64,*it); + } + psb.pages++; + bool allblank = true; + for(const unsigned char *s = data; s < data+pagesize; s+=sectorsize){ + psb.sectors ++; + if(is_blank(s,sectorsize)){ + psb.zsectors++; + continue; + } + allblank = false; + if(af_is_badsector(af,s)){ + psb.badsectors++; + continue; + } + } + if(allblank) psb.zpages++; + } + free(data); + printf(" \n"); + print_xml64("pages",psb.pages); + print_xml64("zpages",psb.zpages); + print_xml64("sectors",psb.sectors); + print_xml64("zsectors",psb.zsectors); + print_xml64("badsectors",psb.badsectors); + printf(" \n"); + } + + /* Now that we have a list of segments, print them */ + for(vector::const_iterator it = seglist.begin(); + it != seglist.end(); + it++){ + + /* See how long the data is */ + size_t datalen = 0; + uint32_t arg=0; + + strcpy(segname,it->c_str()); + + if(af_get_seg(af,segname,&arg,0,&datalen)){ + err(1,"Can't read info for segment '%s'",segname); + } + + unsigned char *data= (unsigned char *)malloc(datalen); + if(data==0) err(1,"Can't allocate %zd bytes for data",datalen); + if(af_get_seg(af,segname,&arg,data,&datalen)!=0){ + err(1,"Can't read data for segment '%s'",segname); + } + + /* Change non-XML characters in segname to _ */ + for(char *cc=segname;*cc;cc++){ + if(!isalphanum(*cc)) *cc = '_'; + } + + if(datalen==8 && (arg & AF_SEG_QUADWORD || af_display_as_quad(segname))){ + /* Print it as a 64-bit value. + * The strcmp is there because early AF_IMAGESIZE segs didn't set + * AF_SEG_QUADWORD... + */ + printf(" <%s coding='base10'>%"I64d"\n",segname,af_decode_q(data),segname); + free(data); + continue; + } + + /* If datalen==0, just print the arg as an unsigned number */ + if(datalen==0){ + printf(" <%s coding='base10'>%"PRIu32"\n",segname,arg,segname); + free(data); + continue; + } + + /* Just handle it as binhex ... */ + printf(" <%s",segname); + if(datalen==0){ + printf(" arg='%"PRIu32"' />\n",arg); + free(data); + continue; + } + + /* If segname ends 'md5', code in hex */ + if(strlen(segname)>=3 && strcmp(segname+strlen(segname)-3,"md5")==0){ + char hex_buf[40]; + printf(" coding='base16'>%s\n", + af_hexbuf(hex_buf,sizeof(hex_buf),data,datalen,0), + segname); + free(data); + continue; + } + + /* If all segment contents are printable ascii with no CRs, LFs, or brackets, + * just print as-is... + */ + if(okay_to_print((const char *)data,datalen)){ + putchar('>'); + for(const char *cc=(const char *)data;cc<(const char *)data+datalen;cc++){ + switch(*cc){ + case '>': fputs("<",stdout);break; + case '<': fputs(">",stdout);break; + case '&': fputs("&",stdout);break; + case '\'': fputs("'",stdout);break; + case '"': fputs(""",stdout);break; + default: putchar(*cc); + } + } + printf("\n",segname); + free(data); + continue; + } + + /* Default coding: base64 */ + int b64size = datalen*2+2; + char *b64buf = (char *)calloc(b64size,1); + int b64size_real = b64_ntop(data,datalen,b64buf,b64size); + b64buf[b64size_real] = 0; // be sure it is null terminated + + printf(" coding='base64'>"); + fputs(b64buf,stdout); + printf("\n",segname); + free(b64buf); + free(data); + } + af_close(af); + + printf("\n"); + return 0; +} + + +int main(int argc,char **argv) +{ + int ch; + const char *infile; + + /* Figure out how many cols the screen has... */ + + while ((ch = getopt(argc, argv, "xj:h?Vs")) != -1) { + switch (ch) { + case 'j': + if(opt_j==0) opt_j = (char **)malloc(0); + opt_j_count++; + opt_j = (char **)realloc(opt_j,sizeof(char *)*opt_j_count); + opt_j[opt_j_count-1] = strdup(optarg); // make a copy + case 'x': opt_x++; break; + case 's': opt_stats++; break; + case 'h': + case '?': + default: + usage(); + break; + case 'V': + printf("%s version %s\n",progname,PACKAGE_VERSION); + exit(0); + } + } + argc -= optind; + argv += optind; + + if(argc<1){ + usage(); + } + + + /* Loop through all of the files */ + printf("\n"); + printf("\n"); + while(*argv){ + infile = *argv++; // get the file + argc--; // decrement argument counter + xml_info(infile); + } + printf("\n"); + exit(0); +} + + diff --git a/tools/conventions.txt b/tools/conventions.txt new file mode 100644 index 0000000..01fcdc3 --- /dev/null +++ b/tools/conventions.txt @@ -0,0 +1,9 @@ +conventions for options: + +-h = Help +-v = print the version number and exit + +-k = specify private key for signing +-c = specify the certificate for signing (if it is not in the keyfile) +-K = speicfy the private key for encrypting +-C = specifying the certificate for encrypting (if it is not in the keyfile) diff --git a/tools/encrypt.crt b/tools/encrypt.crt new file mode 100644 index 0000000..179d3ae --- /dev/null +++ b/tools/encrypt.crt @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICsDCCAhmgAwIBAgIJAK8x26tLzQA9MA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQwHhcNMDcwOTE5MTM0MzE2WhcNMDcxMDE5MTM0MzE2WjBF +MQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50 +ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB +gQDYSG2MFym8pHN+OKutFGOzXLvxXwbRfSs3ESeLaaFb0YXxRebEBiySh0uhrlYU +nHQfcibi+KJz0qIp/lz4EDJdifjQfA+48tqmtL3gvWI8RfSe2Du6PUPmhaT81T6G +AabP9tsHZcyMM2GqfsakPpu/JG/b0PpBTO8qJFN5/HA1qQIDAQABo4GnMIGkMB0G +A1UdDgQWBBRp/+1UvKd1vaHmA+NOPY0NuJpg/zB1BgNVHSMEbjBsgBRp/+1UvKd1 +vaHmA+NOPY0NuJpg/6FJpEcwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUt +U3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZIIJAK8x26tL +zQA9MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAE+oOXirqVgZ6oc8w +SD5T4lKKFA8Gfyk8172K7kLj8r242U1upAkqkIDlTLH7XPGbEN6F4V+KuF+nx1o5 +ATLBAit8DBZj4eZEXklwP1SWrYcg2y3vf7++B3NPL1Qpw18BJP+j87K77drPWF5D +hO9WeBNPQMCA61O2bV1DCIKrLDA= +-----END CERTIFICATE----- diff --git a/tools/encrypt.key b/tools/encrypt.key new file mode 100644 index 0000000..fdbf82a --- /dev/null +++ b/tools/encrypt.key @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQDYSG2MFym8pHN+OKutFGOzXLvxXwbRfSs3ESeLaaFb0YXxRebE +BiySh0uhrlYUnHQfcibi+KJz0qIp/lz4EDJdifjQfA+48tqmtL3gvWI8RfSe2Du6 +PUPmhaT81T6GAabP9tsHZcyMM2GqfsakPpu/JG/b0PpBTO8qJFN5/HA1qQIDAQAB +AoGBAJhF3gK8qCDtc3bRdWUlLtCwII5exhEl3BAoHrxTwUxA5tmoykEGy8jaQpBx +ip92S3d8Sg6Q2OdMy+le8Zug46OV0tNb3XAoK3WttYkgiA1kg90TLM+7mNnv785v +6DxqECHh7Nimx7fWwgQ5NVc24EK6KcYWNefNIJiPOnkJaEuFAkEA+J4rZAYSfYP5 +B46xflnjGzukoG4N5AeyOD5g5sUw8wDKH8ELxZ9GpvO//qEE9SP0xkqkvstSH79F +Y2bkc1wybwJBAN60eCv3zpt9fprDWEqgm7hW18qGx1l1m9JxWppakjl+jC8uTBlZ +BBDghogbAv4gSre8L6ReHb99v2/0ETNARWcCQQDGISbrO3uY04euYiVJUFEQ7uMv +hwVC6G87K1sGDbgIsGRkEfBTua8QdchYH/JXeiQjsnDALEByVHfz05fT53G5AkEA +n6G8gcee0NWjakubfeKhvACDrSfuvLEQgEfAKePB7HRJB4ioA4NK5s4JoIW8H2wG +Iq1BeQ/9QsVf819+9L2dAQJBAMNotSbODXYVDkpJt/hUIrG6L/YknYe3jQdSkb4b +38QvCQrYzS3bMgaQezr7Uyp7otO5n86o6ogB31w4z31hnFM= +-----END RSA PRIVATE KEY----- diff --git a/tools/hashextent.h b/tools/hashextent.h new file mode 100644 index 0000000..9d30f4c --- /dev/null +++ b/tools/hashextent.h @@ -0,0 +1,192 @@ +#ifndef HASHEXTENT_H +#define HASHEXTENT_H + +/** + * hashextent: class to track a hash request or value + * + * Simson L. Garfinkel + * 2009-09-18: SLG - Added to repository + * This file is a work of a US government employee and as such is in the Public domain. + * Simson L. Garfinkel, March 12, 2012 + */ + +#include +#include +#include +#include +#include +#include +#include + + +using std::string; +using std::ostream; +using std::vector; +using std::map; + +class hashextent { +public: + hashextent():digest(0),digest_bits_(0){} + ~hashextent(){ + if(digest) free(digest); + } + uint64_t start; + uint64_t bytes; + hashextent(uint64_t aStart,uint64_t aBytes):start(aStart),bytes(aBytes),digest(0){} + hashextent(AFFILE *af,string alg,uint64_t aStart,uint64_t aBytes):start(aStart),bytes(aBytes),digest(0){ + compute_digest(af,alg); + } + static bool compare(const hashextent &e1,const hashextent &e2){ + return e1.start < e2.start; + } + + static int hexcharval(char hex){ + if(hex>='0' && hex<='9') return hex-'0'; + if(hex>='A' && hex<='F') return hex+10-'A'; + if(hex>='a' && hex<='f') return hex+10-'a'; + return 0; + } + + static string bin2hex(unsigned char *md,int len){ + std::stringstream sbuf; + while(len>0){ + char buf[3]; + snprintf(buf,sizeof(buf),"%02x",md[0]); + sbuf << buf; + md ++; + len --; + } + return sbuf.str(); + } + + static int hex2bin(unsigned char *binbuf,size_t hexbuf_size,const char *hex){ + int bits = 0; + while(hex[0] && hex[1] && hexbuf_size>0){ + *binbuf++ = ((hexcharval(hex[0])<<4) | + hexcharval(hex[1])); + hex += 2; + bits += 8; + hexbuf_size -= 1; + } + if(hexbuf_size>0) binbuf[0] = 0; // might as well null-terminate if there is room + return bits; + } + + + u_char *get_digest(){ + if(!digest){ + int bytes = hexdigest.size()/2; + digest = (u_char *)malloc(bytes); + digest_bits_ = hex2bin(digest,bytes,hexdigest.c_str()); + } + return digest; + } + /* These parameters are for when the structure is read */ + int digest_bits() { + if(!digest) get_digest(); + return digest_bits_; + } + string digest_name; + string coding; + string hexdigest; + + /** Compute the digest from the disk and set all the fields. + * Return 0 if success, -1 if failure. + */ + int compute_digest(AFFILE *af,string digestToUse){ + const EVP_MD *md = EVP_get_digestbyname(digestToUse.c_str()); + EVP_MD_CTX ctx; + if(!md) return -1; // digest not available + EVP_DigestInit(&ctx,md); + if(af_seek(af,start,0)!=start) return -1; // can't seek + + uint64_t bytes_read = 0; + while(bytes_read < this->bytes){ + u_char buf[65536]; + int to_read = (this->bytes-bytes_read) < sizeof(buf) ? (this->bytes-bytes_read) : sizeof(buf); + if(af_read(af,buf,to_read)!=to_read) return -1; // error reading + /* compute the hash */ + EVP_DigestUpdate(&ctx,buf,to_read); + bytes_read += to_read; + } + /* Compute the results */ + if(digest!=0) free(digest); + u_int digest_bytes = 1024; + digest = (u_char *)malloc(digest_bytes); // big enough for any conceivable digest + EVP_DigestFinal(&ctx,digest,&digest_bytes); + digest_bits_ = digest_bytes*8; + digest_name = digestToUse; + hexdigest = bin2hex(digest,digest_bits_/8); + return 0; + } + /** Return XML for the digest */ + string toXML(){ + std::stringstream sstart,sbytes; + sstart << start; + sbytes << bytes; + return string(""+hexdigest+""; + } +private:; + u_char *digest; + u_int digest_bits_; +}; +ostream & operator << (ostream &os, const hashextent &he){ + os << "[" << he.digest_name << " @ " << he.start << "(" << he.bytes << " bytes) " << he.hexdigest << "]"; + return os; +} + +bool operator == (const hashextent &h1,const hashextent &h2) { + return h1.start==h2.start && h1.bytes==h2.bytes && h1.hexdigest==h2.hexdigest; +} + +class hashvector:public vector { +public: + static int ireverse(int a,int b){ + return a digests(){ + vector bits_vector; + std::set bits_set; // why isn't find working on vector? set shouldn't be needed + map bits_to_hash; + for(hashvector::iterator it = begin();it!=end();it++){ + (*it).get_digest(); // parse the digest to determine length + int bits = (*it).digest_bits(); + if(bits_set.find(bits)==bits_set.end()){ + bits_set.insert(bits); + bits_vector.push_back(bits); + bits_to_hash[bits] = (*it).digest_name; + } + } + /* Now reverse sort it */ + sort(bits_vector.begin(),bits_vector.end(),ireverse); + //sort(bits_vector.begin(),bits_vector.end()); + //reverse(bits_vector.begin(),bits_vector.end()); + /* Generate the result */ + vector ret; + for(vector::const_iterator it = bits_vector.begin();it != bits_vector.end(); it++){ + ret.push_back(bits_to_hash[*it]); + } + return ret; + } + /** + * Return the strongest digest in the hashvector that OpenSSL + * makes avilable on the runtime system. + */ + const EVP_MD *strongest_available(){ + vector algs = digests(); + for(vector::const_iterator it = algs.begin(); it!=algs.end(); it++){ + const EVP_MD *ret = EVP_get_digestbyname((*it).c_str()); + if(ret) return ret; + } + return 0; // no digest available + } +}; + + +#endif diff --git a/tools/link-info.txt b/tools/link-info.txt new file mode 100644 index 0000000..d49c024 --- /dev/null +++ b/tools/link-info.txt @@ -0,0 +1,14 @@ +---------------------------------------------------------------------- +Libraries have been installed in: + /usr/local/lib + +If you ever happen to want to link against installed libraries +in a given directory, LIBDIR, you must either use libtool, and +specify the full pathname of the library, or use the `-LLIBDIR' +flag during linking and do at least one of the following: + - add LIBDIR to the `DYLD_LIBRARY_PATH' environment variable + during execution + +See any operating system documentation about shared libraries for +more information, such as the ld(1) and ld.so(8) manual pages. +---------------------------------------------------------------------- diff --git a/tools/test_afsegment.sh b/tools/test_afsegment.sh new file mode 100755 index 0000000..ca1edb4 --- /dev/null +++ b/tools/test_afsegment.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# Test the afsegment command + +# This file is a work of a US government employee and as such is in the Public domain. +# Simson L. Garfinkel, March 12, 2012 + +export PATH=$srcdir:../tools:../../tools:.:$PATH +BLANK_BASE=`mktemp -t blankXXXXX` +BLANK_AFF=$BLANK_BASE.aff +unset AFFLIB_PASSPHRASE + +echo === Putting a new metadata segment into blank.aff === + +/bin/rm -f $BLANK_AFF +affcopy /dev/null $BLANK_AFF +affsegment -ssegname=testseg1 $BLANK_AFF +if [ x"testseg1" = x`affsegment -p segname $BLANK_AFF` ] ; then + echo affsegment worked! +else + echo affsegment does not work properly + exit 1 +fi +/bin/rm -f $BLANK_AFF + diff --git a/tools/test_crypto.sh b/tools/test_crypto.sh new file mode 100755 index 0000000..2e636ce --- /dev/null +++ b/tools/test_crypto.sh @@ -0,0 +1,80 @@ +#!/bin/sh +# +# test to make sure that the encrypted aff that is distributed with +# the AFFLIB can be decrypted using a known passphrase. +# +# This file is a work of a US government employee and as such is in the Public domain. +# Simson L. Garfinkel, March 12, 2012 + +unset AFFLIB_PASSPHRASE +export PATH=$srcdir:../tools:../../tools:.:$PATH + +BASE=`mktemp -t encryptedXXXXXX` +ENCRYPTED_AFF=$BASE.aff +ENCRYPTED_AFD=$BASE.afd +PLAINTEXT_ISO=$BASE.iso + +echo Making encrypted AFF from stored value + +openssl base64 -d > $ENCRYPTED_AFF < $PLAINTEXT_ISO </dev/null +dd if=/dev/zero bs=16777216 count=2 >> $1 2>/dev/null +for i in 1 2 3 4 5 6 7 8 9 0 ; do \ + for fn in /usr/share/dict/* ; do \ + cat $fn >> $1 ; + done ; \ +done +ls -l $1 +openssl md5 $1 +exit 0 + diff --git a/tools/test_passphrase.sh b/tools/test_passphrase.sh new file mode 100755 index 0000000..7bdebb7 --- /dev/null +++ b/tools/test_passphrase.sh @@ -0,0 +1,70 @@ +#!/bin/sh +# +# test the passphrase tools + +# This file is a work of a US government employee and as such is in the Public domain. +# Simson L. Garfinkel, March 12, 2012 +export PATH=$srcdir:../tools:../../tools:.:$PATH + +echo === testing `affcrypto -V` === +echo === MAKING THE TEST FILES == +unset AFFLIB_PASSPHRASE + +BLANK_BASE=`mktemp -t blankXXXXX` +BLANK_AFF=$BLANK_BASE.aff +BLANK_ISO=$BLANK_BASE.iso +BLANK_ENCRYPTED_AFF=${BLANK_BASE}_encrypted.aff +WORDS=`mktemp -t wordsXXXX` + +rm -f $BLANK_ISO $BLANK_AFF $BLANK_ENCRYPTED_AFF $WORDS +test_make_random_iso.sh $BLANK_ISO || (echo Cannot run test_make_random_iso.sh && exit -1) + +if [ ! -r $BLANK_ISO ]; then + echo CANNOT CREATE $BLANK_ISO + echo Permission error prevents test from continuing. + exit 0 +fi + +affconvert -o $BLANK_AFF $BLANK_ISO || exit 1 +affconvert -o file://:passphrase@/$BLANK_ENCRYPTED_AFF $BLANK_ISO || exit 1 + +if [ ! -r $BLANK_ENCRYPTED_AFF ]; then + echo CANNOT CREATE $BLANK_ENCRYPTED_AFF + echo Permission error prevents test from continuing. + exit 0 +fi + + +# Make sure affcrypto reports properly for with and with no encrypted segments +if (affcrypto $BLANK_AFF | grep " 0 encrypted" > /dev/null ) ; then + echo $BLANK_ENCRYPTED_AFF properly created +else + echo ENCRYPTED SEGMENTS IN $BLANK_ENCRYPTED_AFF --- STOP + exit 1 +fi + +# Now test affcrypto +echo Encrypted segment count: `affcrypto -j $BLANK_ENCRYPTED_AFF` +if [ `affcrypto -j $BLANK_ENCRYPTED_AFF` = "0" ]; then + echo NO ENCRYPTED SEGMENTS IN $BLANK_ENCRYPTED_AFF --- STOP + exit 1 +else + echo $BLANK_ENCRYPTED_AFF properly created +fi + +echo "sleepy" > $WORDS +echo "dopey" >> $WORDS +echo "doc" >> $WORDS +echo "passphrase" >> $WORDS +echo "foobar" >> $WORDS +if [ "`affcrypto -k -f $WORDS $BLANK_ENCRYPTED_AFF|grep correct|grep passphrase`"x = x ] ; then + echo affcrypto did not find the right passphrase + exit 1 +else + echo affcrypto found the correct pasphrase +fi + +rm $BLANK_ISO $BLANK_AFF $BLANK_ENCRYPTED_AFF $WORDS + +echo ALL TESTS PASS +exit 0 diff --git a/tools/test_recovery.sh b/tools/test_recovery.sh new file mode 100755 index 0000000..24ca517 --- /dev/null +++ b/tools/test_recovery.sh @@ -0,0 +1,61 @@ +#!/bin/sh +# +# test the signing tools +# +# This file is a work of a US government employee and as such is in the Public domain. +# Simson L. Garfinkel, March 12, 2012 + +export PATH=$srcdir:../tools:../../tools:.:$PATH + +RECOVERY_BASE=`mktemp -t recoveryXXXX` +RECOVERY_KEY=$RECOVERY_BASE.key +RECOVERY_BAK=$RECOVERY_BASE.bak +RECOVERY_ISO=$RECOVERY_BASE.iso +RECOVERY_AFM=$RECOVERY_BASE.afm +RECOVERY_PEM=$RECOVERY_BASE.pem + +/bin/rm -f $RECOVERY_KEY $RECOVERY_BAK $RECOVERY_ISO $RECOVERY_AFM + +unset AFFLIB_PASSPHRASE + +test_make_random_iso.sh $RECOVERY_ISO + +echo ==== AFRECOVERY TEST === +echo Make an X509 key + +SUBJECT="/CN=Mr. Recovery/emailAddress=recovery@investiations.com" +openssl req -x509 -newkey rsa:1024 -keyout $RECOVERY_PEM -out $RECOVERY_PEM -nodes -subj "$SUBJECT" + + +if [ ! -r $RECOVERY_ISO ]; then + echo $RECOVERY_ISO was not created. + printenv + echo current directory: `pwd` + exit 0 +fi + + +cp $RECOVERY_ISO $RECOVERY_BAK +echo =========== +echo Step 1: SIGNING $RECOVERY_ISO +if ! affsign -k $RECOVERY_PEM $RECOVERY_ISO ; then exit 1 ; fi +ls -l $RECOVERY_ISO $RECOVERY_AFM +echo =========== +echo Step 2: VERIFYING SIGNATURE +if ! affverify $RECOVERY_AFM ; then exit 1 ; fi +echo =========== +echo Step 3: CORRUPTING FILE recovery.iso +dd if=/dev/random of=$RECOVERY_ISO count=1 skip=1 conv=notrunc +echo =========== +echo Step 4: ATTEMPTING RECOVERY +if ! affrecover $RECOVERY_AFM ; then exit 1 ; fi +echo ========== +echo Step 5: MAKING SURE THAT THE MD5 HAS NOT CHANGED +if ! cmp $RECOVERY_BAK $RECOVERY_ISO ; then echo file changed ; exit 1 ; fi +echo MD5 has not changed +echo ========== +echo Step 6: See if Digital Signature is still good +if ! affverify $RECOVERY_AFM ; then echo signature no longer good ; exit 1 ; fi +echo Signature still good +echo ALL TESTS PASS +/bin/rm -f $RECOVERY_KEY $RECOVERY_BAK $RECOVERY_ISO $RECOVERY_AFM $RECOVERY_PEM diff --git a/tools/test_sealing.sh b/tools/test_sealing.sh new file mode 100755 index 0000000..d2a38b7 --- /dev/null +++ b/tools/test_sealing.sh @@ -0,0 +1,114 @@ +#!/bin/sh +# +# test the PKI sealing tools + +# This file is a work of a US government employee and as such is in the Public domain. +# Simson L. Garfinkel, March 12, 2012 + +BASE=`mktemp -t testfileXXXX` +SEALING_KEY=$BASE.sealing.key +SEALING_PEM=$BASE.sealing.pem +EVIDENCE_ISO=$BASE.evidence.iso +EVIDENCE_AFF=$BASE.evidence.aff + +/bin/rm -f $SEALING_KEY $SEALING_PEM $EVIDENCE_ISO $EVIDENCE_AFF +unset AFFLIB_PASSPHRASE +unset AFFLIB_DECRYPTING_PRIVATE_KEYFILE + +echo === MAKING THE TEST FILES === + +export PATH=$srcdir:../tools:../../tools:.:$PATH +test_make_random_iso.sh $EVIDENCE_ISO + +echo Making X.509 key + +openssl req -x509 -newkey rsa:1024 -keyout $SEALING_KEY -out $SEALING_PEM -nodes -subj "/C=US/ST=California/L=Remote/O=Country Govt./OU=Sherif Dept/CN=Mr. Agent/emailAddress=agent@investiations.com" + +# One way to do this should be to set an environment variable and do an affcopy +# Another way should be to make the AFF file and then encrypt it with affcrypto. + +# Make the aff file and encrypt it with affcrypto +if ! affconvert $EVIDENCE_ISO ; then exit 1 ; fi +if ! affcrypto -e -C $SEALING_PEM $EVIDENCE_AFF ; then exit 1 ; fi + +# Make sure we can't read it without specifying a keyfile +echo This should generate an error: +if affcompare $EVIDENCE_ISO $EVIDENCE_AFF ; then + echo ERROR - could read encrypted file without decryption key. + exit 1 +fi +echo Could not read encrypted file without setting decryption key --- CORRECT BEHAVIOR + +echo Now set the AFFLIB_DECRYPTING_PRIVATE_KEYFILE and see if we can +echo read the file... +export AFFLIB_DECRYPTING_PRIVATE_KEYFILE=$SEALING_KEY +# Make sure we can't read it without specifying a keyfile +if ! affcompare $EVIDENCE_ISO $EVIDENCE_AFF ; then + echo ERROR - could not read encrypted once decryption key was set. + exit 1 +fi +echo Could read encrypted file once decryption key was sety --- CORRECT BEHAVIOR + +#Add passphrase to aff file encrypted with $SEALING_PEM +echo Now attempting to add a passphrase to the aff file that was encrypted for a private key. + +if ! affcrypto -S -K $SEALING_KEY -N mypassword $EVIDENCE_AFF ; then + echo ERROR - could not add passphrase. + exit 1 +fi +echo Successfully added passphrase to aff file -- CORRECT BEHAVIOR + +#Make sure we cannot read the file with a wrong passphrase + +echo Verify that password was added to aff file. Should read passphrase correct. +affcrypto -p mypassword $EVIDENCE_AFF + + + +echo Recreating AFFILE to remove encryption +rm -f $EVIDENCE_AFF +if ! affconvert $EVIDENCE_ISO ; then exit 1 ; fi +unset AFFLIB_DECRYPTING_PRIVATE_KEYFILE + +echo encrypting AFFILE with passphrase +if ! affcrypto -e -N mypassword $EVIDENCE_AFF ; then + echo ERROR - could not encrypt with passphrase. + exit 1 +fi + +echo Adding public key encryption to AFFILE encrypted with passphrase +if ! affcrypto -A -p mypassword -C $SEALING_PEM $EVIDENCE_AFF ; then + echo ERROR - could not add public key +fi + +# Same tests as above now done to test the encryption +# addition that was done in reverse +# Make sure we can't read it without specifying a keyfile +echo This should generate an error: +if affcompare $EVIDENCE_ISO $EVIDENCE_AFF ; then + echo ERROR - could read encrypted file without decryption key. + exit 1 +fi +echo Could not read encrypted file without setting decryption key --- CORRECT BEHAVIOR + +echo Now set the AFFLIB_DECRYPTING_PRIVATE_KEYFILE and see if we can +echo read the file... +export AFFLIB_DECRYPTING_PRIVATE_KEYFILE=$SEALING_KEY +# Make sure we can't read it without specifying a keyfile +if ! affcompare $EVIDENCE_ISO $EVIDENCE_AFF ; then + echo ERROR - could not read encrypted once decryption key was set. + exit 1 +fi +echo Could read encrypted file once decryption key was sety --- CORRECT BEHAVIOR + +if test "x"$1 = "x--keep" ; then + echo will not erase $EVIDENCE_ISO $EVIDENCE_AFF $SEALING_KEY $SEALING_PEM +else + echo Erasing temporary files. + rm -f $EVIDENCE_ISO $EVIDENCE_AFF $SEALING_KEY $SEALING_PEM +fi + +echo $0 completed successfully. + +exit 0 + diff --git a/tools/test_signing.sh b/tools/test_signing.sh new file mode 100755 index 0000000..e7debd9 --- /dev/null +++ b/tools/test_signing.sh @@ -0,0 +1,80 @@ +#!/bin/sh +# +# test the signing tools + +# This file is a work of a US government employee and as such is in the Public domain. +# Simson L. Garfinkel, March 12, 2012 + +unset AFFLIB_PASSPHRASE + +BASE=`mktemp -t baseXXXXX` +AGENT_PEM=$BASE.agent.pem +ANALYST_PEM=$BASE.analyst.pem +ARCHIVES_PEM=$BASE.archives.pem +EVIDENCE=$BASE.evidence.aff +EVIDENCE1=$BASE.evidence1.aff +EVIDENCE2=$BASE.evidence2.aff +EVIDENCE3=$BASE.evidence3.aff + +/bin/rm -f $AGENT_PEM $ANALYST_PEM $ARCHIVES_PEM $EVIDENCE $EVIDENCE2 $EVIDENCE3 + +echo TEST $0 +echo === MAKING THE TEST FILES === + +export PATH=$srcdir:../tools:../../tools:.:$PATH +test_make_random_iso.sh rawevidence.iso + + +echo ==== AFSIGN TEST === +echo Making X.509 keys + +openssl req -x509 -newkey rsa:1024 -keyout $AGENT_PEM -out $AGENT_PEM -nodes -subj "/C=US/ST=California/L=Remote/O=Country Govt./OU=Sherif Dept/CN=Mr. Agent/emailAddress=agent@investiations.com" + + openssl req -x509 -newkey rsa:1024 -keyout $ANALYST_PEM -out $ANALYST_PEM -nodes -subj "/C=US/ST=California/L=Remote/O=State Police/OU=Forensics/CN=Ms. Analyst/emailAddress=analyst@investiations.com" +openssl req -x509 -newkey rsa:1024 -keyout $ARCHIVES_PEM -out $ARCHIVES_PEM -nodes -subj "/C=US/ST=CA/L=Remote/O=Archives/OU=Electronic/CN=Dr. Librarian/emailAddress=drbits@investiations.com" + +echo Making an AFF file to sign +rm -f $EVIDENCE evidence?.aff +affconvert -o $EVIDENCE rawevidence.iso +echo Initial AFF file +if ! affinfo -a $EVIDENCE ; then exit 1 ; fi + +echo Signing AFF file... +echo affsign -k $AGENT_PEM $EVIDENCE +if ! affsign -k $AGENT_PEM $EVIDENCE ; then echo affsign failed ; exit 1 ; fi + +echo Verifying Signature... +echo affverify $EVIDENCE +if ! affverify $EVIDENCE ; then echo affverify failed ; exit 1 ; fi ; + +echo Signature test 1 passed +echo Testing chain-of-custody signatures + +echo Step 10: Copying original raw file to evidence1.aff +if ! affcopy -k $AGENT_PEM rawevidence.iso evidence1.aff ; then exit 1; fi +echo Step 11: Running affinfo on evidence1.aff +if ! affinfo -a evidence1.aff ; then exit 1 ; fi +echo Step 12: Comparing rawevidence.iso to evidence1.aff +if ! affcompare rawevidence.iso evidence1.aff ; then exit 1 ; fi +echo Step 13: Verifying evidence1 +if ! affverify evidence1.aff ; then exit 1 ; fi + +echo +echo Making the second generation copy +echo "This copy was made by the analyst" | affcopy -z -k $ANALYST_PEM -n evidence1.aff $EVIDENCE2 +if ! affinfo -a $EVIDENCE2 ; then exit 1 ; fi +if ! affcompare rawevidence.iso $EVIDENCE2 ; then exit 1 ; fi +if ! affverify $EVIDENCE2 ; then exit 1 ; fi +echo +echo Making the third generation copy +echo "This copy was made by the archives" | affcopy -z -k $ARCHIVES_PEM -n $EVIDENCE2 $EVIDENCE3 +if ! affinfo -a $EVIDENCE3 ; then exit 1 ; fi +if ! affcompare rawevidence.iso $EVIDENCE3 ; then exit 1 ; fi +if ! affverify $EVIDENCE3 ; then exit 1 ; fi + + +echo All tests passed successfully +echo Erasing temporary files. +rm -f $AGENT_PEM $ARCHIVES_PEM $ANALYST_PEM $EVIDENCE evidence.afm rawevidence.iso cevidence.iso $EVIDENCE2 $EVIDENCE3 $EVIDENCE +exit 0 + diff --git a/tools/unix4win32.h b/tools/unix4win32.h new file mode 100644 index 0000000..fb008e8 --- /dev/null +++ b/tools/unix4win32.h @@ -0,0 +1,28 @@ +/* + * This file is a work of a US government employee and as such is in the Public domain. + * Simson L. Garfinkel, March 12, 2012 + */ + +#ifdef WIN32 +#include "getopt.h" // will pull from win32 directory +#include +#include +#include +#ifndef F_OK +#define F_OK 00 +#endif + +#ifndef R_OK +#define R_OK 04 +#endif + +#ifndef S_IFBLK +#define S_IFBLK -1 // will never be used +#endif + +#ifndef PATH_MAX +#define PATH_MAX MAXPATHLEN +#endif +#endif + + diff --git a/win32/Changes.txt b/win32/Changes.txt new file mode 100644 index 0000000..b8e1b1c --- /dev/null +++ b/win32/Changes.txt @@ -0,0 +1,14 @@ +Changes for 3.6.0: +=================== +* AFFLIB now compiles with mingw so this directory is no longer needed. Only the afflib.mak + file is provided for historical purposes (and in case somebody really, really wants to compile + udner Microsoft VC++) + + +Changes from 2.0: +================= +* AFFLIB now uses OpenSSL, rather than the Microsoft Crypto API. +* Upgrade from libewf-20070512 to libewf-20080501 +* Full support for AFFLIB 3.0 encryption +* Distribution includes executables and libraries + diff --git a/win32/README_MSVC++.txt b/win32/README_MSVC++.txt new file mode 100644 index 0000000..efc3c23 --- /dev/null +++ b/win32/README_MSVC++.txt @@ -0,0 +1,237 @@ +Information for compiling AFFLIB 3.4.2 under Windows / Microsoft +Visual C++. As of version 3.6.0 this information is obsolete since we +now compile under mingw. +================================================================ + +Introduction +============ +This directory builds the following as a single library using +Microsoft Visual C++ that contains all of the following: + + * All of LIBAFF + * LZMA compression system + * ZLIB compression system + * LIBEWF EnCase image reading system. + * AFFLIB3.0 encryption issues + +On Unix systems the ZLIB and LIBEWF libraries must be separately +installed. However, copies of these libraries are included as +subdirectories to the win32 library. These copies are included solely +to make things easier for Windows users; these copies are not used by +the Unix AFFLIB installation. + +Windows programs that are linked with this library can read files in +any of the following formats: + + * RAW & Split raw + * AFF, AFM, AFD + * EnCase / Expert Witness + +You can also use AFFLIB on Windows with Cygwin; details on that +appear in this file as well. + + +Compiling with Microsoft VC++ +============================= + +To compile this library, you need a copy of Microsoft Visual C++ 2008 Express. +(Libewf will not compile with any earlier version of Microsoft Visual C++.) + + +Installing VC++ 2008: +--------------------- +You can download a FREE copy of Visual C++ 2005 Express Edition from +Microsoft: http://www.microsoft.com/express + +1. Go to http://www.microsoft.com/express + +2. Download Visual Studio C++ 2008 Express Edition. + (You will get vcsetup.exe; save it on the desktop and run it.) + +3. Install in the default location, + C:\Program Files\Microsoft Visual Studio 9\ + +5. Follow the instructions: + - Run Visual Studio C++ 2008. + - Select "Register Product" from the help menu. + - Log into the Microsoft website with your Passport credentials. + - Get the registration key from Microsoft (after email answerback) + and paste it into the Help panel. + +6. Run Microsoft Update to install the latest service packs. + - You MUST have the most recent .NET Framework. + - You should also have the Security Updates for the VC++ 2008 + Redistributable Package + +7. Now you must download and install the Microsoft Platform SDK so + that you will have the header files for the Microsoft Crypto API. + This can be confusing. I downloaded the Windows Server 2008 + Platform SDK Web Install. I got it from this URL: + http://www.microsoft.com/downloads/details.aspx?familyid=A55B6B43-E24F-4EA3-A93E-40C0EC4F68E5 + + (Be careful not to download the x64 platform SDK unless you are + running on a 64-bit machine!) + + - Be sure that you DO NOT chose the configuration option to + Register environment variables. + + YOU CANNOT COMPILE AFFLIB UNLESS THE PLATFORM SDK IS INSTALLED. + + - If possible, install the platform SDK as + C:\Program Files\Microsoft Platform SDK\. + + If you can't do this, you will need to modify afflib.mak to + reflect the actual install location + +8. Finally, you must download and install OpenSSL: + http://www.slproweb.com/products/Win32OpenSSL.html + + If you just want to run with openSSL, use this: + http://www.slproweb.com/download/Win32OpenSSL_Light-0_9_8k.exe + (In your win32/openssl directory) + + If you want to compile, use this: + + If you wish to compile, you will need to edit the file x509.h and + add this line: + +#ifdef OPENSSL_SYS_WIN32 +/* Under Win32 these are defined in wincrypt.h */ +#undef X509_NAME +#undef X509_CERT_PAIR +#undef X509_EXTENSIONS /* added by SLG */ +#endif + + This is apparently a known bug in OpenSSL + (http://wso2.org/forum/thread/3861), but it hasn't been fixed yet. + + +Compiling AFFLIB: +----------------- +1. Unpack the afflib distribution into a directory such as c:\afflib + +2. From the Windows Start menu, run the Visual Studio 2005 Command Prompt + +3. Change into the win32 subdirectory, e.g. "chdir c:\afflib\win32" + +4. Type "make.bat" to run the makefile. + +3. The command file "make.bat" will compile AFFLIB, LIBEWF, ZLIB and LZMA, and create a single .lib + file. Compilation options are specified in afflib.mak in this directory. + +4. The following programs are ported: + + TARGETS = afcompare.exe afconvert.exe afcopy.exe afdiskprint.exe affix.exe afinfo.exe afstats.exe afxml.exe + + You can compile them all by typing: + + % make + + Alternatively, you can compile a single executable with: + + % make afcat.exe + +5. To make the library alone: + +* Open a VS2008 command prompt (run vcvars32.bat). +* Make sure you have OpenSSL and zlib installed. +* Add OpenSSL and zlib include paths to your INCLUDE path. +* Run "make.bat afflib.lib" (inside the win32 directory). +* Rename afflib.lib to afflibMT.lib + +Repeat 3 more times with different COMPILER_MODE to produce 3 more +libs: +======================================================================== +* Run "make.bat clean" before each build, just to be safe. +* COMPILER_MODE /MD /O2 /D NDEBUG => rename to afflibMD.lib +* COMPILER_MODE /MTd => rename to afflibMTd.lib +* COMPILER_MODE /MDd => rename to afflibMDd.lib +* Put libs in world/3rdparty/afflib/VER/lib/vs20XX/win32 + +Repeat 4 more times to build 64-bit libraries with same names: +============================================================== +* Run vcvars64.bat instead of vcvars32.bat to set up environment. +* May want to reset INCLUDE path for OpenSSL and zlib, just to be +safe. +* Don't forget to use /O2 /D NDEBUG when building release (MT and MD). +* Note: No need to specify /MACHINE:X64 linker option in afflib.mak +because + we're only building a static library, not linking to create any + EXEs/DLLs. +* Run "make.bat clean" before each build, just to be safe. +* Put libs in world/3rdparty/afflib/VER/lib/vs20XX/x64 + + +Once it is compiled: + +1. To open a multi-file EnCase file, just specify the first .E01 file; the AFFLIB + implementation will automatically look for all of the other EnCase files. + +2. Right now you should really use this library for READING AFF & E01 + files, rather than WRITING them. Writing should work, but it's not + very well tested on Windows. S3 is currently not supported on Windows. + +3. If you want to change the compile switches, feel free. They're in afflib.mak + + + + +================================================================ + + + +Compiling AFFLIB with Cygwin +============================ +Cygwin is a Unix emulation system that allows standard Linux/Unix open +source software to be run on top of Windows through the use of a +special "cygwin" DLL. + +To use Cygwin, follow these step-by-step instructions: + +1. Go to http://www.cygwin.com/. + +2. Click "Download Cygwin Now"; this will give you an executable file. + +3. Run the Cygwin Net Release Setup Program. + +4. Select "Install from Internet" + +5. Install into the C:\cygwin directory for All Users. Select + "Unix/binary" as the default text file type. + +6. Select a mirror site. + +7. Click on the arrows next to "Devel" to change the word "Default" to + "Install". This will cause the entire Cygwin development system to + be installed. + +8. Click "Next" and come back in an hour. + +9. When you get the "Installation Complete" message, start the cygwin + shell and type the following: + + $ mkdir afflib + $ cd afflib + $ wget http://www.afflib.org/afflib.tar.gz + $ tar xfvz afflib.tar.gz + $ cd afflib* + $ ./configure + $ make + $ make install + + NOTE: EnCase support will NOT be compiled in unless you separately + download and install LIBEWF. libewf must be downloaded from + https://www.uitwisselplatform.nl/projects/libewf/ + + +10. You should now have a working system. + + + +# +# Local Variables: +# mode: flyspell +# mode: auto-fill +# End: +# LocalWords: AFFLIB +# diff --git a/win32/afflib.mak b/win32/afflib.mak new file mode 100755 index 0000000..e5456b9 --- /dev/null +++ b/win32/afflib.mak @@ -0,0 +1,194 @@ +# +# Windows makefile for AFFLIB & libewf +# + +# What to make: +TARGETS = affcompare.exe affconvert.exe affcopy.exe affdiskprint.exe affix.exe affinfo.exe affstats.exe affxml.exe + +# These are things you may need to change: +# +# SDK_DIR is where the Windows Platform SDK is installed on your computer +# +SDK_DIR = "C:\Program Files\Microsoft SDKs\Windows\v6.1" +OPENSSL_DIR = C:\OpenSSL + +# COMPILER_MODE specifies how you want the libaries compiled: + +COMPILER_MODE = /MT /O2 /D NDEBUG + +EXPATDIR = expat-2.0.1\lib + +all: $(TARGETS) + +################################################################ + + +INCS = /I.\ + /Izlib-1.2.3\ \ + /I..\lib \ + /I..\lzma443\C \ + /I..\lzma443\C\7zip\Compress\LZMA_Alone \ + /I$(EXPATDIR) \ + /I$(SDK_DIR)/Include /I$(OPENSSL_DIR)/Include + +DEFS = /DWIN32 /DWIN32_NT /DMSC /D_CRT_SECURE_NO_DEPRECATE /DHAVE_CONFIG_WINDOWS_H /DHAVE_LIBCRYPTO /DHAVE_OPENSSL_EVP_H /DHAVE_WINDOWS_API/DHAVE_MEMMOVE + + +CC=cl + +# removed: /Gm - enable minimal rebuild; generated internal compiler error + +OTHER_FLAGS = /c /nologo /EHsc /RTC1 /RTCs /W2 $(COMPILER_MODE) + +CPPFLAGS=$(INCS) $(DEFS) $(OTHER_FLAGS) /Fp"afflib.pch" /Fo$*.obj +CFLAGS=$(INCS) $(DEFS) $(OTHER_FLAGS) /Fp"afflib.pch" /Fo$*.obj + +# Here are some useful flags: +# -CODE GENERATION- +# /W4 - warning level 4 +# /Gm - enable minimal rebuild +# /ZI - enable full Edit and Continue info (conflicts with /OPT:ICF) +# /RTC1 - Enable fast checks +# /RTCs - Stack Frame runtime checking +# +# +# -PREPROCESSOR- +# /I - specifies include directory +# /D - define a switch +# +# -OUTPUT FILES- +# /Fp - Precompiled headers +# +# -MISCELLANEOUS- +# /nologo - Disable logo +# /c - compile only, don't link +# +# -LINKING- +# /MT - Multithreaded, static link +# /MD - Multithreaded, Dynamic Link +# /MTd - Multithreaded, static w/ debugging +# /MDd - Multithreaded, Dynamic w/ debugging +# Note: "the single-threaded CRT (formerly /ML or /MLd options) +# are no longer available. Instead, use the multithreaded CRT." +# http://msdn2.microsoft.com/en-us/library/abx4dbyh(VS.80).aspx + +LZMA_OBJS = \ + ..\lzma443\C\7zip\Compress\LZMA_Alone\LzmaBench.obj \ + ..\lzma443\C\7zip\Compress\LZMA_Alone\LzmaRam.obj \ + ..\lzma443\C\7zip\Compress\LZMA_Alone\LzmaRamDecode.obj \ + ..\lzma443\C\7zip\Compress\LZMA_C\LzmaDecode.obj \ + ..\lzma443\C\7zip\Compress\Branch\BranchX86.obj \ + ..\lzma443\C\7zip\Compress\LZMA\LZMADecoder.obj \ + ..\lzma443\C\7zip\Compress\LZMA\LZMAEncoder.obj \ + ..\lzma443\C\7zip\Compress\LZ\LZInWindow.obj \ + ..\lzma443\C\7zip\Compress\LZ\LZOutWindow.obj \ + ..\lzma443\C\7zip\Compress\RangeCoder\RangeCoderBit.obj \ + ..\lzma443\C\7zip\Common\InBuffer.obj \ + ..\lzma443\C\7zip\Common\OutBuffer.obj \ + ..\lzma443\C\7zip\Common\StreamUtils.obj \ + ..\lzma443\C\Common\Alloc.obj \ + ..\lzma443\C\Common\CommandLineParser.obj \ + ..\lzma443\C\Common\CRC.obj \ + ..\lzma443\C\Common\String.obj \ + ..\lzma443\C\Common\StringConvert.obj \ + ..\lzma443\C\Common\StringToInt.obj \ + ..\lzma443\C\Common\Vector.obj + +AFF_OBJS = ..\lib\aff_db.obj \ + ..\lib\aff_toc.obj \ + ..\lib\afflib.obj \ + ..\lib\afflib_os.obj \ + ..\lib\afflib_pages.obj \ + ..\lib\afflib_stream.obj \ + ..\lib\afflib_util.obj \ + ..\lib\crypto.obj \ + ..\lib\base64.obj \ + ..\lib\lzma_glue.obj \ + ..\lib\s3_glue.obj \ + ..\lib\vnode_aff.obj \ + ..\lib\vnode_afd.obj \ + ..\lib\vnode_afm.obj \ + ..\lib\vnode_raw.obj \ + ..\lib\vnode_s3.obj \ + ..\lib\vnode_split_raw.obj \ + ..\lib\utils.obj \ + ..\lib\display.obj + + +ZLIB_OBJS = zlib-1.2.3\adler32.obj \ + zlib-1.2.3\compress.obj \ + zlib-1.2.3\crc32.obj \ + zlib-1.2.3\deflate.obj \ + zlib-1.2.3\gzio.obj \ + zlib-1.2.3\infback.obj \ + zlib-1.2.3\inffast.obj \ + zlib-1.2.3\inflate.obj \ + zlib-1.2.3\inftrees.obj \ + zlib-1.2.3\trees.obj \ + zlib-1.2.3\uncompr.obj \ + zlib-1.2.3\zutil.obj + +EXPAT_OBJS = $(EXPATDIR)\xmlparse.obj \ + $(EXPATDIR)\xmlrole.obj \ + $(EXPATDIR)\xmltok.obj \ + $(EXPATDIR)\xmltok_impl.obj \ + $(EXPATDIR)\xmltok_ns.obj + +# +# WIN32_OBJS are extra objects we need on windows because +# they aren't present +# +WIN32_OBJS = getopt.obj + + +# LIB_OBJS are all of the objects that we'll put in the library + +LIB_OBJS = $(AFF_OBJS) $(LZMA_OBJS) $(WIN32_OBJS) $(ZLIB_OBJS) + +afflib.lib: $(LIB_OBJS) + lib -out:afflib.lib $(LIB_OBJS) + +# WIN32_LIBS are the libraries that we link with on win32 +# ws2_32.lib = Winsock 2 +# advapi32.lib = CryptoAPI support DLL (LIBEWF uses crypto api) + +WIN32LIBS = ws2_32.lib advapi32.lib c:\openssl\lib\libeay32.lib + +clean: + del afflib.lib $(LIB_OBJS) $(TARGETS) + +LINK_OPTS = /libpath:$(SDK_DIR)/Lib /nodefaultlib:libc $(WIN32LIBS) + +afftest.exe: ..\lib\aftest.obj afflib.lib + link -out:afftest.exe ..\lib\afftest.obj afflib.lib $(LINK_OPTS) + +affcat.exe: ..\tools\affcat.obj afflib.lib + link -out:affcat.exe ..\tools\affcat.obj afflib.lib $(LINK_OPTS) + +affcopy.exe: ..\tools\affcopy.obj ..\tools\aff_bom.obj afflib.lib + link -out:affcopy.exe ..\tools\affcopy.obj ..\tools\aff_bom.obj afflib.lib $(LINK_OPTS) + +affcompare.exe: ..\tools\affcompare.obj afflib.lib + link -out:affcompare.exe ..\tools\affcompare.obj afflib.lib $(LINK_OPTS) + +affconvert.exe: ..\tools\affconvert.obj afflib.lib + link -out:affconvert.exe ..\tools\affconvert.obj afflib.lib $(LINK_OPTS) + +affdiskprint.exe: ..\tools\affdiskprint.obj afflib.lib $(EXPAT_OBJS) + link -out:affdiskprint.exe ..\tools\affdiskprint.obj ..\tools\aff_bom.obj afflib.lib $(EXPAT_OBJS) $(LINK_OPTS) + +affix.exe: ..\tools\affix.obj afflib.lib + link -out:affix.exe ..\tools\affix.obj afflib.lib $(LINK_OPTS) + +affinfo.exe: ..\tools\affinfo.obj afflib.lib + link -out:affinfo.exe ..\tools\affinfo.obj afflib.lib $(LINK_OPTS) + +affsegment.exe: ..\tools\affsegment.obj afflib.lib + link -out:affsegment.exe ..\tools\affsegment.obj afflib.lib $(LINK_OPTS) + +affstats.exe: ..\tools\affstats.obj afflib.lib + link -out:affstats.exe ..\tools\affstats.obj afflib.lib $(LINK_OPTS) + +affxml.exe: ..\tools\affxml.obj afflib.lib + link -out:affxml.exe ..\tools\affxml.obj afflib.lib $(LINK_OPTS) + diff --git a/win32/extra-dist.txt b/win32/extra-dist.txt new file mode 100644 index 0000000..7cc551f --- /dev/null +++ b/win32/extra-dist.txt @@ -0,0 +1,295 @@ +README.txt +affconfig.h +afflib.mak +getopt.c +getopt.h +make.bat +openssl/Win32OpenSSL-0_9_8k.exe +openssl/Win32OpenSSL_Light-0_9_8k.exe +libewf-20080501/aclocal.m4 +libewf-20080501/AUTHORS +libewf-20080501/ChangeLog +libewf-20080501/common/character_string.c +libewf-20080501/common/character_string.h +libewf-20080501/common/common.h +libewf-20080501/common/config.h +libewf-20080501/common/config.h.in +libewf-20080501/common/config_windows.h +libewf-20080501/common/date_time.c +libewf-20080501/common/date_time.h +libewf-20080501/common/endian.h +libewf-20080501/common/error_string.c +libewf-20080501/common/error_string.h +libewf-20080501/common/file_io.c +libewf-20080501/common/file_io.h +libewf-20080501/common/Makefile.am +libewf-20080501/common/Makefile.in +libewf-20080501/common/memory.h +libewf-20080501/common/notify.c +libewf-20080501/common/notify.h +libewf-20080501/common/string_conversion.c +libewf-20080501/common/string_conversion.h +libewf-20080501/common/system_string.c +libewf-20080501/common/system_string.h +libewf-20080501/common/types.h +libewf-20080501/config.guess +libewf-20080501/config.sub +libewf-20080501/configure +libewf-20080501/configure.ac +libewf-20080501/COPYING +libewf-20080501/debian/changelog +libewf-20080501/debian/changelog.in +libewf-20080501/debian/compat +libewf-20080501/debian/control +libewf-20080501/debian/copyright +libewf-20080501/debian/cron.d.ex +libewf-20080501/debian/emacsen-install.ex +libewf-20080501/debian/emacsen-remove.ex +libewf-20080501/debian/emacsen-startup.ex +libewf-20080501/debian/init.d.ex +libewf-20080501/debian/libewf-default.ex +libewf-20080501/debian/libewf-dev.dirs +libewf-20080501/debian/libewf-dev.docs +libewf-20080501/debian/libewf-dev.install +libewf-20080501/debian/libewf-tools.dirs +libewf-20080501/debian/libewf-tools.docs +libewf-20080501/debian/libewf-tools.install +libewf-20080501/debian/libewf.dirs +libewf-20080501/debian/libewf.doc-base.EX +libewf-20080501/debian/libewf.docs +libewf-20080501/debian/libewf.install +libewf-20080501/debian/libewf.postinst.debhelper +libewf-20080501/debian/libewf.postrm.debhelper +libewf-20080501/debian/manpage.1.ex +libewf-20080501/debian/manpage.sgml.ex +libewf-20080501/debian/manpage.xml.ex +libewf-20080501/debian/menu.ex +libewf-20080501/debian/postinst.ex +libewf-20080501/debian/postrm.ex +libewf-20080501/debian/preinst.ex +libewf-20080501/debian/prerm.ex +libewf-20080501/debian/rules +libewf-20080501/debian/shlibs.local.ex +libewf-20080501/debian/shlibs.local.ex.in +libewf-20080501/debian/watch.ex +libewf-20080501/depcomp +libewf-20080501/doc/header.txt +libewf-20080501/doc/header2.txt +libewf-20080501/doc/tests.txt +libewf-20080501/ewftools/ewfacquire.c +libewf-20080501/ewftools/ewfacquirestream.c +libewf-20080501/ewftools/ewfalter.c +libewf-20080501/ewftools/ewfbyte_size_string.c +libewf-20080501/ewftools/ewfbyte_size_string.h +libewf-20080501/ewftools/ewfcommon.c +libewf-20080501/ewftools/ewfcommon.h +libewf-20080501/ewftools/ewfdigest_context.c +libewf-20080501/ewftools/ewfdigest_context.h +libewf-20080501/ewftools/ewfdigest_hash.c +libewf-20080501/ewftools/ewfdigest_hash.h +libewf-20080501/ewftools/ewfexport.c +libewf-20080501/ewftools/ewfgetopt.c +libewf-20080501/ewftools/ewfgetopt.h +libewf-20080501/ewftools/ewfglob.c +libewf-20080501/ewftools/ewfglob.h +libewf-20080501/ewftools/ewfinfo.c +libewf-20080501/ewftools/ewfinput.c +libewf-20080501/ewftools/ewfinput.h +libewf-20080501/ewftools/ewfmd5.h +libewf-20080501/ewftools/ewfoutput.c +libewf-20080501/ewftools/ewfoutput.h +libewf-20080501/ewftools/ewfsha1.h +libewf-20080501/ewftools/ewfsignal.c +libewf-20080501/ewftools/ewfsignal.h +libewf-20080501/ewftools/ewfstring.c +libewf-20080501/ewftools/ewfstring.h +libewf-20080501/ewftools/ewfverify.c +libewf-20080501/ewftools/Makefile.am +libewf-20080501/ewftools/Makefile.in +libewf-20080501/include/libewf/definitions.h +libewf-20080501/include/libewf/definitions.h.in +libewf-20080501/include/libewf/extern.h +libewf-20080501/include/libewf/handle.h +libewf-20080501/include/libewf/types.h +libewf-20080501/include/libewf/types.h.in +libewf-20080501/include/libewf.h +libewf-20080501/include/libewf.h.in +libewf-20080501/INSTALL +libewf-20080501/install-sh +libewf-20080501/libewf/ewf_char.h +libewf-20080501/libewf/ewf_crc.h +libewf-20080501/libewf/ewf_data.h +libewf-20080501/libewf/ewf_definitions.h +libewf-20080501/libewf/ewf_digest_hash.h +libewf-20080501/libewf/ewf_error2.h +libewf-20080501/libewf/ewf_file_header.h +libewf-20080501/libewf/ewf_hash.h +libewf-20080501/libewf/ewf_ltree.h +libewf-20080501/libewf/ewf_section.h +libewf-20080501/libewf/ewf_session.h +libewf-20080501/libewf/ewf_string.h +libewf-20080501/libewf/ewf_table.h +libewf-20080501/libewf/ewf_volume.h +libewf-20080501/libewf/ewf_volume_smart.h +libewf-20080501/libewf/ewfx_delta_chunk.h +libewf-20080501/libewf/libewf.c +libewf-20080501/libewf/libewf_chunk_cache.c +libewf-20080501/libewf/libewf_chunk_cache.h +libewf-20080501/libewf/libewf_chunk_offset.h +libewf-20080501/libewf/libewf_compression.c +libewf-20080501/libewf/libewf_compression.h +libewf-20080501/libewf/libewf_debug.c +libewf-20080501/libewf/libewf_debug.h +libewf-20080501/libewf/libewf_file.c +libewf-20080501/libewf/libewf_file.h +libewf-20080501/libewf/libewf_filename.c +libewf-20080501/libewf/libewf_filename.h +libewf-20080501/libewf/libewf_handle.c +libewf-20080501/libewf/libewf_handle.h +libewf-20080501/libewf/libewf_hash_sections.c +libewf-20080501/libewf/libewf_hash_sections.h +libewf-20080501/libewf/libewf_hash_values.c +libewf-20080501/libewf/libewf_hash_values.h +libewf-20080501/libewf/libewf_header_sections.c +libewf-20080501/libewf/libewf_header_sections.h +libewf-20080501/libewf/libewf_header_values.c +libewf-20080501/libewf/libewf_header_values.h +libewf-20080501/libewf/libewf_interface.c +libewf-20080501/libewf/libewf_interface.h +libewf-20080501/libewf/libewf_media_values.c +libewf-20080501/libewf/libewf_media_values.h +libewf-20080501/libewf/libewf_offset_table.c +libewf-20080501/libewf/libewf_offset_table.h +libewf-20080501/libewf/libewf_read.c +libewf-20080501/libewf/libewf_read.h +libewf-20080501/libewf/libewf_section.c +libewf-20080501/libewf/libewf_section.h +libewf-20080501/libewf/libewf_section_list.c +libewf-20080501/libewf/libewf_section_list.h +libewf-20080501/libewf/libewf_sector_table.c +libewf-20080501/libewf/libewf_sector_table.h +libewf-20080501/libewf/libewf_segment_file.c +libewf-20080501/libewf/libewf_segment_file.h +libewf-20080501/libewf/libewf_segment_file_handle.c +libewf-20080501/libewf/libewf_segment_file_handle.h +libewf-20080501/libewf/libewf_segment_table.c +libewf-20080501/libewf/libewf_segment_table.h +libewf-20080501/libewf/libewf_string.c +libewf-20080501/libewf/libewf_string.h +libewf-20080501/libewf/libewf_support.c +libewf-20080501/libewf/libewf_support.h +libewf-20080501/libewf/libewf_values_table.c +libewf-20080501/libewf/libewf_values_table.h +libewf-20080501/libewf/libewf_write.c +libewf-20080501/libewf/libewf_write.h +libewf-20080501/libewf/Makefile.am +libewf-20080501/libewf/Makefile.in +libewf-20080501/libewf.pc +libewf-20080501/libewf.pc.in +libewf-20080501/libewf.spec +libewf-20080501/libewf.spec.in +libewf-20080501/ltmain.sh +libewf-20080501/macosx/Introduction.rtf +libewf-20080501/macosx/Introduction.rtf.in +libewf-20080501/macosx/libewf.pmproj +libewf-20080501/macosx/License.rtf +libewf-20080501/macosx/Readme.rtf +libewf-20080501/Makefile.am +libewf-20080501/Makefile.in +libewf-20080501/manuals/ewfacquire.1 +libewf-20080501/manuals/ewfacquirestream.1 +libewf-20080501/manuals/ewfexport.1 +libewf-20080501/manuals/ewfinfo.1 +libewf-20080501/manuals/ewfverify.1 +libewf-20080501/manuals/libewf.3 +libewf-20080501/manuals/Makefile.am +libewf-20080501/manuals/Makefile.in +libewf-20080501/missing +libewf-20080501/msvscpp/ewfacquire/ewfacquire.vcproj +libewf-20080501/msvscpp/ewfacquirestream/ewfacquirestream.vcproj +libewf-20080501/msvscpp/ewfalter/ewfalter.vcproj +libewf-20080501/msvscpp/ewfexport/ewfexport.vcproj +libewf-20080501/msvscpp/ewfinfo/ewfinfo.vcproj +libewf-20080501/msvscpp/ewfverify/ewfverify.vcproj +libewf-20080501/msvscpp/libewf.sln +libewf-20080501/msvscpp/libewf_dll/libewf_dll.vcproj +libewf-20080501/NEWS +libewf-20080501/README +zlib-1.2.3/adler32.c +zlib-1.2.3/algorithm.txt +zlib-1.2.3/as400/bndsrc +zlib-1.2.3/as400/compile.clp +zlib-1.2.3/as400/readme.txt +zlib-1.2.3/as400/zlib.inc +zlib-1.2.3/ChangeLog +zlib-1.2.3/compress.c +zlib-1.2.3/configure +zlib-1.2.3/crc32.c +zlib-1.2.3/crc32.h +zlib-1.2.3/deflate.c +zlib-1.2.3/deflate.h +zlib-1.2.3/example-res.rc +zlib-1.2.3/example-static-res.rc +zlib-1.2.3/example.c +zlib-1.2.3/examples/fitblk.c +zlib-1.2.3/examples/gun.c +zlib-1.2.3/examples/gzappend.c +zlib-1.2.3/examples/gzjoin.c +zlib-1.2.3/examples/gzlog.c +zlib-1.2.3/examples/gzlog.h +zlib-1.2.3/examples/README.examples +zlib-1.2.3/examples/zlib_how.html +zlib-1.2.3/examples/zpipe.c +zlib-1.2.3/examples/zran.c +zlib-1.2.3/FAQ +zlib-1.2.3/gzio.c +zlib-1.2.3/INDEX +zlib-1.2.3/infback.c +zlib-1.2.3/inffast.c +zlib-1.2.3/inffast.h +zlib-1.2.3/inffixed.h +zlib-1.2.3/inflate.c +zlib-1.2.3/inflate.h +zlib-1.2.3/inftrees.c +zlib-1.2.3/inftrees.h +zlib-1.2.3/make_vms.com +zlib-1.2.3/Makefile +zlib-1.2.3/Makefile.in +zlib-1.2.3/minigzip-res.rc +zlib-1.2.3/minigzip-static-res.rc +zlib-1.2.3/minigzip.c +zlib-1.2.3/patches/foo.gz +zlib-1.2.3/patches/zlib-1.2.2.diff +zlib-1.2.3/patches/zlib-1.2.3.diff +zlib-1.2.3/patches/zlib-dllversion.c +zlib-1.2.3/patches/zlib-ltversion +zlib-1.2.3/patches/zlib.ico +zlib-1.2.3/projects/README.projects +zlib-1.2.3/projects/visualc6/example.dsp +zlib-1.2.3/projects/visualc6/minigzip.dsp +zlib-1.2.3/projects/visualc6/README.txt +zlib-1.2.3/projects/visualc6/zlib.dsp +zlib-1.2.3/projects/visualc6/zlib.dsw +zlib-1.2.3/qnx/package.qpg +zlib-1.2.3/README +zlib-1.2.3/trees.c +zlib-1.2.3/trees.h +zlib-1.2.3/uncompr.c +zlib-1.2.3/win32/DLL_FAQ.txt +zlib-1.2.3/win32/Makefile.bor +zlib-1.2.3/win32/Makefile.emx +zlib-1.2.3/win32/Makefile.gcc +zlib-1.2.3/win32/Makefile.msc +zlib-1.2.3/win32/VisualC.txt +zlib-1.2.3/win32/zlib.def +zlib-1.2.3/win32/zlib1.rc +zlib-1.2.3/zconf.h +zlib-1.2.3/zconf.in.h +zlib-1.2.3/zlib-dll-res.rc +zlib-1.2.3/zlib-dllversion.c +zlib-1.2.3/zlib-ltversion +zlib-1.2.3/zlib.3 +zlib-1.2.3/zlib.h +zlib-1.2.3/zlib.ico +zlib-1.2.3/zutil.c +zlib-1.2.3/zutil.h diff --git a/win32/make.bat b/win32/make.bat new file mode 100755 index 0000000..7820b55 --- /dev/null +++ b/win32/make.bat @@ -0,0 +1,2 @@ +nmake /f afflib.mak %1 %2 %3 %4 %5 /Y +