From fc9b8055154f1b85bd566a929dabc3486ee82a5d Mon Sep 17 00:00:00 2001 From: John Lindgren Date: Mon, 25 Feb 2013 15:56:42 -0500 Subject: [PATCH 01/14] Remove trailing whitespace. --- lib/aff_db.cpp | 14 ++++----- lib/aff_toc.cpp | 4 +-- lib/afflib.cpp | 34 ++++++++++----------- lib/afflib.h | 42 +++++++++++++------------- lib/afflib_i.h | 30 +++++++++---------- lib/afflib_os.cpp | 14 ++++----- lib/afflib_pages.cpp | 36 +++++++++++----------- lib/afflib_stream.cpp | 22 +++++++------- lib/afflib_util.cpp | 6 ++-- lib/aftest.cpp | 38 ++++++++++++------------ lib/aftimer.h | 14 ++++----- lib/base64.cpp | 12 ++++---- lib/crypto.cpp | 66 ++++++++++++++++++++--------------------- lib/lzma_glue.cpp | 8 ++--- lib/s3.cpp | 16 +++++----- lib/s3_glue.cpp | 28 ++++++++--------- lib/s3_glue.h | 4 +-- lib/utils.cpp | 6 ++-- lib/vnode_afd.cpp | 30 +++++++++---------- lib/vnode_aff.cpp | 42 +++++++++++++------------- lib/vnode_afm.cpp | 20 ++++++------- lib/vnode_qemu.cpp | 6 ++-- lib/vnode_raw.cpp | 10 +++---- lib/vnode_s3.cpp | 18 +++++------ lib/vnode_split_raw.cpp | 30 +++++++++---------- 25 files changed, 275 insertions(+), 275 deletions(-) diff --git a/lib/aff_db.cpp b/lib/aff_db.cpp index 0279b0e..1bc82d8 100644 --- a/lib/aff_db.cpp +++ b/lib/aff_db.cpp @@ -1,8 +1,8 @@ -/* Distributed under the 4-part Berkeley License */ +/* Distributed under the 4-part Berkeley License */ /* * afflib_db.cpp: - * + * * Functions for the manipulation of the AFF database. */ @@ -18,7 +18,7 @@ ****************************************************************/ /**************************************************************** - *** Probe Functions + *** Probe Functions ****************************************************************/ int af_probe_next_seg(AFFILE *af, @@ -29,7 +29,7 @@ int af_probe_next_seg(AFFILE *af, size_t *segsize_, // optional get size of entire segment int do_rewind) // optional rewind af->aseg, otherwise leave at start of segment data { - if(!af->aseg)(*af->error_reporter)("af_probe_next_segment only works with aff files"); + if(!af->aseg)(*af->error_reporter)("af_probe_next_segment only works with aff files"); struct af_segment_head segh; memset(&segh,0,sizeof(segh)); @@ -37,7 +37,7 @@ int af_probe_next_seg(AFFILE *af, uint64_t start = ftello(af->aseg); #ifdef __BORLANDC__ fseeko(af->aseg, start, SEEK_SET) ; // Windows is dumb :-( -#endif +#endif if(fread(&segh,sizeof(segh),1,af->aseg)!=1){ return AF_ERROR_EOF; @@ -152,7 +152,7 @@ int aff_find_seg(AFFILE *af,const char *segname, } return -1; // couldn't find segment } - + int af_get_segq(AFFILE *af,const char *name,int64_t *aff_quad) { unsigned char buf[8]; @@ -164,7 +164,7 @@ int af_get_segq(AFFILE *af,const char *name,int64_t *aff_quad) if(bufsize!=sizeof(struct aff_quad)){ // make sure size is good. return -1; } - + *aff_quad = af_decode_q(buf); return 0; } diff --git a/lib/aff_toc.cpp b/lib/aff_toc.cpp index 5bd067b..76006e6 100644 --- a/lib/aff_toc.cpp +++ b/lib/aff_toc.cpp @@ -2,7 +2,7 @@ /* * afflib_dir.cpp: - * + * * Functions for the manipulation of the AFF directories */ @@ -87,7 +87,7 @@ int aff_toc_build(AFFILE *af) // build the dir if we couldn't find it aff_toc_free(af); // clear the old one af_rewind_seg(af); // start at the beginning - // note: was malloc(0), but that causes problems under Borland + // note: was malloc(0), but that causes problems under Borland af->toc = (aff_toc_mem *)malloc(sizeof(aff_toc_mem)); while(1){ char segname[AF_MAX_NAME_LEN]; diff --git a/lib/afflib.cpp b/lib/afflib.cpp index 45eda59..c607a8b 100644 --- a/lib/afflib.cpp +++ b/lib/afflib.cpp @@ -40,15 +40,15 @@ struct af_vnode *af_vnode_array[] = { #ifdef USE_S3 &vnode_s3, // must be first for s3:// interpertation #endif - &vnode_afd, + &vnode_afd, &vnode_afm, // must be before aff &vnode_aff, #ifdef USE_QEMU &vnode_vmdk, &vnode_dmg, #endif -#ifdef USE_SPARSEIMAGE - &vnode_sparseimage, +#ifdef USE_SPARSEIMAGE + &vnode_sparseimage, #endif &vnode_split_raw, // must be before raw &vnode_raw, // greedy; must be last @@ -134,7 +134,7 @@ void af_perror(const char *str) fprintf(stderr,"%s: %s: %s\n",getprogname(),str,af_error_str); #else fprintf(stderr,"%s: %s\n",str,af_error_str); -#endif +#endif } @@ -284,7 +284,7 @@ AFFILE *af_open_with(const char *url,int flags,int mode, struct af_vnode *v) af->password[buflen] = '\000'; } } - + /* TK: If no password was set and the AFFLIB_ASK_PASS is set, ask for a passphrase */ /* Note things for hard files */ @@ -304,7 +304,7 @@ AFFILE *af_open_with(const char *url,int flags,int mode, struct af_vnode *v) if(flags & AF_HALF_OPEN) return af; // for low-level tools /* Try opening it! */ - if((*af->v->open)(af)){ + if((*af->v->open)(af)){ strlcpy(af_error_str,af->error_str,sizeof(af_error_str)); // make a copy of the error string af_deallocate(af); return 0; @@ -337,7 +337,7 @@ AFFILE *af_open_with(const char *url,int flags,int mode, struct af_vnode *v) af_sanitize_password(af); } } - + /* Try public key... */ if(can_decrypt==false){ const char *kf = getenv(AFFLIB_DECRYPTING_PRIVATE_KEYFILE); @@ -346,7 +346,7 @@ AFFILE *af_open_with(const char *url,int flags,int mode, struct af_vnode *v) } } } - + af_read_sizes(af); // set up the metadata if(af_trace) fprintf(af_trace,"af_open_with(%s,%o,%o,%s)\n",url,flags,mode,v->name); return af; @@ -537,7 +537,7 @@ void af_set_callback(AFFILE *af,void (*wcb)(struct affcallback_info *)) void af_enable_compression(AFFILE *af,int type,int level) { AF_WRLOCK(af); - af->compression_type = type; + af->compression_type = type; af->compression_level = level; AF_UNLOCK(af); } @@ -588,7 +588,7 @@ int64_t af_get_imagesize(AFFILE *af) return ret; } -/* +/* * af_make_badflag: * Create a randomized bag flag and * leave an empty segment of how many badsectors there are @@ -655,14 +655,14 @@ void af_aes_decrypt(AFFILE *af,const char *segname,unsigned char *data,size_t *d /* An encrypted segment was retrieved; decrypt and trunc the length as necessary */ uint32_t extra = (*datalen) % AES_BLOCK_SIZE; uint32_t pad = (AES_BLOCK_SIZE - extra) % AES_BLOCK_SIZE; - + if(data==0){ // just wants to find out new length if(extra>0){ *datalen -= AES_BLOCK_SIZE; } return; } - + if(extra!=0 && *datalen < AES_BLOCK_SIZE){ *datalen = 0; // something is wrong return; @@ -678,7 +678,7 @@ void af_aes_decrypt(AFFILE *af,const char *segname,unsigned char *data,size_t *d unsigned char iv[AES_BLOCK_SIZE]; memset(iv,0,sizeof(iv)); strlcpy((char *)iv,segname,sizeof(iv)); - + /* Decrypt! */ AF_READLOCK(af); AES_cbc_encrypt(data,data,*datalen,&af->crypto->dkey,iv,AES_DECRYPT); @@ -728,7 +728,7 @@ int af_get_seg(AFFILE *af,const char *segname,uint32_t *arg,unsigned char *data, return 0; // finally it fits } } -#endif +#endif /* Try for the unencrypted segment */ int ret = (*af->v->get_seg)(af,segname,arg,data,datalen); AF_UNLOCK(af); @@ -747,7 +747,7 @@ int af_get_next_seg(AFFILE *af,char *segname,size_t segname_len,uint32_t *arg, } int r = (*af->v->get_next_seg)(af,segname,segname_len,arg,data,datalen); #ifdef HAVE_AES_ENCRYPT - if(AF_SEALING_VNODE(af) + if(AF_SEALING_VNODE(af) && ends_with(segname,AF_AES256_SUFFIX) && af->crypto->auto_decrypt){ segname[strlen(segname)-strlen(AF_AES256_SUFFIX)] = 0; @@ -771,7 +771,7 @@ int af_get_next_seg(AFFILE *af,char *segname,size_t segname_len,uint32_t *arg, AF_UNLOCK(af); return r; // not sure why we got this error } -#endif +#endif AF_UNLOCK(af); return r; } @@ -827,7 +827,7 @@ int af_update_segf(AFFILE *af, const char *segname, oldname = segname; segname = aesname; - /* Figure out if we need to padd out for encryption. Allocate space and + /* Figure out if we need to padd out for encryption. Allocate space and */ uint32_t extra = (datalen) % AES_BLOCK_SIZE; uint32_t pad = (AES_BLOCK_SIZE - extra) % AES_BLOCK_SIZE; diff --git a/lib/afflib.h b/lib/afflib.h index ab2d005..5a97f22 100644 --- a/lib/afflib.h +++ b/lib/afflib.h @@ -1,14 +1,14 @@ #ifndef _AFFLIB_H_ -#define _AFFLIB_H_ +#define _AFFLIB_H_ /* * afflib.h: - * + * * This file describes the public AFFLIB interface. * The interface to reading AFF files and Raw files. * * Copyright (c) 2005-2006 - * Simson L. Garfinkel and Basis Technology, Inc. + * Simson L. Garfinkel and Basis Technology, Inc. * All rights reserved. * * This code is derrived from software contributed by @@ -43,7 +43,7 @@ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. + * SUCH DAMAGE. */ /* Figure out what kind of OS we are running on */ @@ -86,7 +86,7 @@ typedef unsigned int uint32_t; #endif /** WIN32 is defined by the NMAKE makefile for Visual C++ under Windows and by mingw **/ -#ifdef WIN32 +#ifdef WIN32 #include #include // gets isatty @@ -115,7 +115,7 @@ typedef __int64 int64_t; #define PRIu64 "I64u" #endif -#endif +#endif /** END OF WIN32 DEFINES **/ #define I64d PRIi64 @@ -126,7 +126,7 @@ typedef __int64 int64_t; #define USE_LZMA struct affcallback_info; struct aff_pagebuf { - int64_t pagenum; // -1 means no page loaded + int64_t pagenum; // -1 means no page loaded unsigned char *pagebuf; // where the data is; size is image_pagesize size_t pagebuf_bytes; // number of bytes in the pagebuf that are valid. uint32_t pagenum_valid:1; // buffer contains data @@ -152,7 +152,7 @@ struct af_vnode_info { uint32_t segment_count_signed; uint32_t segment_count_encrypted; uint32_t page_count_encrypted; -}; // +}; // /* All of this stuff should be hidden inside a single private structure... */ @@ -161,7 +161,7 @@ typedef struct _AFFILE AFFILE; /* The information that is provided in the aff callback */ struct affcallback_info { int info_version; // version number for this segment - AFFILE *af; // v1: the AFFILE responsibile for the callback + AFFILE *af; // v1: the AFFILE responsibile for the callback int phase; // v1: 1 = before compress; 2 = after compressing; // 3 = before writing; 4 = after writing int64_t pagenum; // v1: page number being written @@ -210,7 +210,7 @@ int af_set_option(AFFILE *af,int option,int value); #define AF_OPTION_AUTO_ENCRYPT 1 // 1 = auto-encrypt #define AF_OPTION_AUTO_DECRYPT 2 // 1 = auto-decrypt -// The following are not implemented yet +// The following are not implemented yet #define AF_OPTION_PIECEWISE_MD5 3 // 1 = automatically write pagen_md5 segments #define AF_OPTION_PIECEWISE_SHA1 4 // 1 = automatically write pagen_md5 segments #define AF_OPTION_PIECEWISE_SHA256 5 // 1 = automatically write pagen_md5 segments @@ -220,7 +220,7 @@ int af_set_option(AFFILE *af,int option,int value); /* Special AFOPEN flags for af_open_with */ #define AF_OPEN_PRIMITIVE (1<<31) // only open primtive, not compound files #define AF_BADBLOCK_FILL (1<<30) // fill unallocated (sparse) with BADBLOCK flag -#define AF_HALF_OPEN (1<<29) // return af before calling af->v->open; +#define AF_HALF_OPEN (1<<29) // return af before calling af->v->open; #define AF_NO_CRYPTO (1<<28) // disable encryption layer /* navigating within the data segments as if they were a single file */ @@ -234,7 +234,7 @@ uint64_t af_tell(AFFILE *af); int af_eof(AFFILE *af); // is the virtual file at the end? /* Additional routines for writing */ -void af_set_callback(AFFILE *af, void (*cb)(struct affcallback_info *acbi)); +void af_set_callback(AFFILE *af, void (*cb)(struct affcallback_info *acbi)); void af_enable_compression(AFFILE *af,int type,int level); // set/gunset compression for writing int af_compression_type(AFFILE *af); int af_write(AFFILE *af,unsigned char *buf,size_t count); @@ -254,7 +254,7 @@ int64_t af_get_imagesize(AFFILE *af); // byte # of last mapped byte in image, or int af_get_pagesize(AFFILE *af); // returns page size, or -1 int af_set_acquisition_date(AFFILE *af,time_t t); // sets AF_ACQUISITION_DATE -#define af_imagesize(af) af_get_imagesize(af) // backwards compatiability +#define af_imagesize(af) af_get_imagesize(af) // backwards compatiability int af_get_segq(AFFILE *af,const char *name,int64_t *quad);/* Get/set 8-byte values */ int af_update_segq(AFFILE *af,const char *name,int64_t quad); @@ -268,7 +268,7 @@ int af_update_segq(AFFILE *af,const char *name,int64_t quad); * If arg!=0, set *arg to be the segment's flag. * if data==0, don't return it. * if datalen && *datalen==0, return the size of the data segment. - *** Returns 0 on success, + *** Returns 0 on success, *** -1 on end of file. (AF_ERROR_EOF) *** -2 if *data is not large enough to hold the segment (AF_ERROR_DATASMALL) *** -3 file is corrupt or other internal error. (AF_ERROR_CORRUPT) @@ -336,7 +336,7 @@ int af_set_sign_files(AFFILE *af,const char *keyfile,const char *certfile); int af_sign_seg3(AFFILE *af,const char *segname, uint32_t arg, const unsigned char *data,uint32_t datalen,uint32_t signmode); int af_sign_seg(AFFILE *af,const char *segname); -int af_sign_all_unsigned_segments(AFFILE *af); // +int af_sign_all_unsigned_segments(AFFILE *af); // int af_sig_verify_seg(AFFILE *af,const char *segname); // see below for return codes int af_is_signature_segment(const char *segname); @@ -421,7 +421,7 @@ int af_hash_verify_seg2(AFFILE *af,const char *segname,unsigned char *sigbuf_,si /* AFF Flags */ /* Flags for 8-byte segments */ -#define AF_SEG_QUADWORD 0x0002 +#define AF_SEG_QUADWORD 0x0002 /* Flags for selecting compression algorithm to try */ #define AF_COMPRESSION_ALG_NONE 0 // don't compress @@ -432,10 +432,10 @@ int af_hash_verify_seg2(AFFILE *af,const char *segname,unsigned char *sigbuf_,si #define AF_PAGE_COMPRESSED 0x0001 #define AF_PAGE_COMP_MAX 0x0002 // compressed at maximum; nice to know #define AF_PAGE_COMP_ALG_MASK 0x00F0 // up to 16 compression algorithms may be used -#define AF_PAGE_COMP_ALG_ZLIB 0x0000 +#define AF_PAGE_COMP_ALG_ZLIB 0x0000 #define AF_PAGE_COMP_ALG_BZIP 0x0010 // not implemented; why bother? #define AF_PAGE_COMP_ALG_LZMA 0x0020 // high compression but pretty slow -#define AF_PAGE_COMP_ALG_ZERO 0x0030 // Data segment is a 4-byte value of # of NULLs. +#define AF_PAGE_COMP_ALG_ZERO 0x0030 // Data segment is a 4-byte value of # of NULLs. #define AF_MD5 "md5" // stores image md5 #define AF_SHA1 "sha1" // stores image sha1 @@ -452,13 +452,13 @@ int af_hash_verify_seg2(AFFILE *af,const char *segname,unsigned char *sigbuf_,si #define AF_ACQUISITION_NOTES "acquisition_notes" // notes made while imaging #define AF_ACQUISITION_DEVICE "acquisition_device" // device used to do the imaging #define AF_ACQUISITION_SECONDS "acquisition_seconds" // stored in arg -#define AF_ACQUISITION_TECHNICIAN "acquisition_tecnician" -#define AF_ACQUISITION_MACADDR "acquisition_macaddr" +#define AF_ACQUISITION_TECHNICIAN "acquisition_tecnician" +#define AF_ACQUISITION_MACADDR "acquisition_macaddr" #define AF_ACQUISITION_DMESG "acquisition_dmesg" // mac addresses are store in ASCII as a list of lines that end with \n, -// for example, "00:03:93:14:c5:04\n" +// for example, "00:03:93:14:c5:04\n" // It is all the mac addresses that were on the acquisition system // DMESG is the output from the "dmesg" command at the time of acquisition diff --git a/lib/afflib_i.h b/lib/afflib_i.h index 6208116..de58aa2 100644 --- a/lib/afflib_i.h +++ b/lib/afflib_i.h @@ -1,12 +1,12 @@ /* * afflib_i.h: * The "master include file" of the AFF Library. - * Includes many fucntions that are not designed + * Includes many fucntions that are not designed * to be used by application programmers. * * * Copyright (c) 2005-2006 - * Simson L. Garfinkel and Basis Technology, Inc. + * Simson L. Garfinkel and Basis Technology, Inc. * All rights reserved. * * This code is derrived from software contributed by @@ -41,7 +41,7 @@ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. + * SUCH DAMAGE. */ #ifndef AFFLIB_I_H @@ -381,7 +381,7 @@ struct _AFFILE { /* additional support for writing. */ unsigned int compression_type; // preferred compression type int compression_level; // 0 is no compression - + /* w_callback: * A callback that is called before and after each segment is written. @@ -410,7 +410,7 @@ struct _AFFILE { void *vnodeprivate; // private storage for the vnode void (*error_reporter)(const char *fmt, ...); - struct af_crypto *crypto; + struct af_crypto *crypto; #ifdef HAVE_PTHREAD pthread_rwlock_t rwlock; // automatically created and destroyed if pthread exists #endif @@ -432,7 +432,7 @@ struct af_crypto { #ifdef AES_BLOCK_SIZE AES_KEY ekey; // encrypt key AES_KEY dkey; // decrypt key -#endif +#endif #ifdef HAVE_OPENSSL_EVP_H EVP_PKEY *sign_privkey; // signing private key (to write signatures) EVP_PKEY *sign_pubkey; // signing public key (to verify signatures) @@ -440,14 +440,14 @@ struct af_crypto { /* Sealing is kept locally and immediately turned into a dkey & ekey */ #endif }; - + /* The AFF STREAM VNODE */ struct af_vnode { int type; // numeric vnode type int flag; // file system flag type const char *name; - int (*identify)(const char *fname,int exists); // returns 1 if file system is identified by implementation; + int (*identify)(const char *fname,int exists); // returns 1 if file system is identified by implementation; int (*open)(AFFILE *af); int (*close)(AFFILE *af); int (*vstat)(AFFILE *af,struct af_vnode_info *); // returns info about the vnode image file @@ -463,8 +463,8 @@ struct af_vnode { }; /* VNODE Flags */ -#define AF_VNODE_TYPE_PRIMITIVE 0x01 // single-file implementation -#define AF_VNODE_TYPE_COMPOUND 0x02 // multi-file implementation +#define AF_VNODE_TYPE_PRIMITIVE 0x01 // single-file implementation +#define AF_VNODE_TYPE_COMPOUND 0x02 // multi-file implementation #define AF_VNODE_TYPE_RELIABLE 0x04 // writes are reliable; no need to verify them. #define AF_VNODE_MAXSIZE_MULTIPLE 0x08 // maxsize must be multiple of pagesize (for AFM and splitraw) #define AF_VNODE_NO_SIGNING 0x10 // vnode does not support signing (like raw) @@ -513,7 +513,7 @@ struct aff_quad { /* As it is kept in memory */ struct aff_toc_mem { char *name; // name of this directory entry - uint64_t offset; // offset from beginning of file. + uint64_t offset; // offset from beginning of file. uint64_t segment_len; // includes head, tail, name & length }; @@ -532,7 +532,7 @@ void af_initialize(); // initialize the AFFLIB /* Internal identification routines */ int af_identify_file_type(const char *filename,int exists); // returns type of a file; if exists=1, file must exist -const char *af_identify_file_name(const char *filename,int exists); // returns name of a file type; +const char *af_identify_file_name(const char *filename,int exists); // returns name of a file type; int split_raw_increment_fname (char *fn); /* exposed for testing in aftest */ /* AFF implementation types returned by af_identify_type() and af_identify_name()*/ @@ -595,7 +595,7 @@ int af_figure_media(int fd,struct af_figure_media_buf *); * -2 if segname_len was not large enough to hold segname * - segname - the name of the next segment. * - segsize - number of bytes the entire segment is. - * + * * doesn't change af->aseg pointer if do_rewind is true, otherwise leaves stream * positioned ready to read the data */ @@ -613,7 +613,7 @@ int af_backspace(AFFILE *af); // back up one segment /* Support for data pages. This is what the stream system is built upon. * Note: pagename to string translation happens inside afflib.cpp, not inside - * the vnode driver. + * the vnode driver. */ int af_page_size(AFFILE *af); // legacy (now is af_get_pagesize) void af_read_sizes(AFFILE *af); // sets up values if we can get them. @@ -679,7 +679,7 @@ size_t strlcat(char *dest,const char *src,size_t dest_size); * This is all experimental right now. */ -int aff_segment_overhead(const char *segname); +int aff_segment_overhead(const char *segname); int aff_toc_free(AFFILE *af); void aff_toc_print(AFFILE *af); int aff_toc_build(AFFILE *af); // build by scanning the AFFILE diff --git a/lib/afflib_os.cpp b/lib/afflib_os.cpp index ecd2ab9..918cb64 100644 --- a/lib/afflib_os.cpp +++ b/lib/afflib_os.cpp @@ -2,7 +2,7 @@ * afflib_os.cpp: * * The OS-specific features of AFFLIB - * + * * This file is a work of a US government employee and as such is in the Public domain. * Simson L. Garfinkel, March 12, 2012 */ @@ -19,7 +19,7 @@ #undef HAVE_SYS_DISK_H #endif -#if defined(HAVE_SYS_DISK_H) +#if defined(HAVE_SYS_DISK_H) #include #endif @@ -36,7 +36,7 @@ ****************************************************************/ /* No longer needed with VC2008 */ -#if 0 +#if 0 #if defined(WIN32) and !defined(__MINGW_H) #pragma warning(disable: 4996) int64 ftello(FILE *stream) @@ -88,7 +88,7 @@ int af_figure_media(int fd,struct af_figure_media_buf *afb) { memset(afb,0,sizeof(*afb)); afb->version = 1; - + #ifdef __APPLE__ #define MEDIA_FIGURED if(ioctl(fd,DKIOCGETBLOCKSIZE,&afb->sector_size)){ @@ -125,7 +125,7 @@ int af_figure_media(int fd,struct af_figure_media_buf *afb) * from a #define */ - afb->sector_size = BLOCK_SIZE; + afb->sector_size = BLOCK_SIZE; #ifdef BLKGETSIZE64 uint64_t total_bytes=0; if(ioctl(fd,BLKGETSIZE64,&total_bytes)){ @@ -150,10 +150,10 @@ int af_figure_media(int fd,struct af_figure_media_buf *afb) afb->sector_size = BLOCK_SIZE; // it's a good guess #else afb->sector_size = 512; // it's a good guess -#endif +#endif /* Try seeking to the end of fd and ask where we are! */ - + off_t start_pos = lseek(fd,0,SEEK_CUR); // find where we are off_t end_of_file = lseek(fd,0,SEEK_END); diff --git a/lib/afflib_pages.cpp b/lib/afflib_pages.cpp index 362f9a5..f539ea6 100644 --- a/lib/afflib_pages.cpp +++ b/lib/afflib_pages.cpp @@ -27,7 +27,7 @@ void af_read_sizes(AFFILE *af) af->badflag_set = 1; } - /* Read the image file segment if it is present. + /* Read the image file segment if it is present. * If it isn't, scan through the disk image to figure out the size of the disk image. */ @@ -134,7 +134,7 @@ int af_get_page_raw(AFFILE *af,int64_t pagenum,uint32_t *arg, unsigned char *data,size_t *bytes) { char segname[AF_MAX_NAME_LEN]; - + memset(segname,0,sizeof(segname)); sprintf(segname,AF_PAGE,pagenum); int r = af_get_seg(af,segname,arg,data,bytes); @@ -185,7 +185,7 @@ int af_get_page(AFFILE *af,int64_t pagenum,unsigned char *data,size_t *bytes) } return r; // segment doesn't exist } - + /* If the segment isn't compressed, just get it*/ uint32_t pageflag = 0; @@ -270,7 +270,7 @@ int af_get_page(AFFILE *af,int64_t pagenum,unsigned char *data,size_t *bytes) } break; #endif - + default: (*af->error_reporter)("Unknown compression algorithm 0x%d", pageflag & AF_PAGE_COMP_ALG_MASK); @@ -297,7 +297,7 @@ int af_get_page(AFFILE *af,int64_t pagenum,unsigned char *data,size_t *bytes) data[*bytes + i] = 0; } size_t end_of_data = *bytes + bytes_left_in_sector; - + /* Now fill to the end of the page... */ for(size_t i = end_of_data; i <= af->image_pagesize-SECTOR_SIZE; i+=SECTOR_SIZE){ memcpy(data+i,af->badflag,SECTOR_SIZE); @@ -390,14 +390,14 @@ int af_update_page(AFFILE *af,int64_t pagenum,unsigned char *data,int datalen) /* Try zero compression first; it's the best algorithm we have */ if(is_buffer_zero(data,datalen)){ - acbi.compression_alg = AF_PAGE_COMP_ALG_ZERO; + acbi.compression_alg = AF_PAGE_COMP_ALG_ZERO; acbi.compression_level = AF_COMPRESSION_MAX; if(af->w_callback) { acbi.phase = 1; (*af->w_callback)(&acbi); } *ldata = htonl(datalen); // store the data length destLen = 4; // 4 bytes - flag = AF_PAGE_COMPRESSED | AF_PAGE_COMP_ALG_ZERO | AF_PAGE_COMP_MAX; + flag = AF_PAGE_COMPRESSED | AF_PAGE_COMP_ALG_ZERO | AF_PAGE_COMP_MAX; cres = 0; acbi.compressed = 1; // it was compressed @@ -468,13 +468,13 @@ int af_update_page(AFFILE *af,int64_t pagenum,unsigned char *data,int datalen) cdata = 0; } } - + /* If a compressed segment was not written, write it uncompressed */ - if(af->pages_written == starting_pages_written){ + if(af->pages_written == starting_pages_written){ if(af->w_callback) {acbi.phase = 3;(*af->w_callback)(&acbi);} ret = af_update_segf(af,segname_buf,0,data,datalen,AF_SIGFLAG_NOSIG); acbi.bytes_written = datalen; - if(af->w_callback) {acbi.phase = 4;(*af->w_callback)(&acbi);} + if(af->w_callback) {acbi.phase = 4;(*af->w_callback)(&acbi);} if(ret==0){ acbi.bytes_written = datalen; // because that is how much we wrote af->pages_written++; @@ -488,15 +488,15 @@ int af_update_page(AFFILE *af,int64_t pagenum,unsigned char *data,int datalen) ****************************************************************/ /* The page cache is a read/write cache. - * + * * Pages that are read are cached after they are decompressed. - * When new pages are fetched, we check the cache first to see if they are there; + * When new pages are fetched, we check the cache first to see if they are there; * if so, they are satsfied by the cache. - * - * Modifications are written to the cache, then dumped to the disk. - * + * + * Modifications are written to the cache, then dumped to the disk. + * * The cache is managed by two functions: - * af_cache_flush(af) - (prevously af_purge) + * af_cache_flush(af) - (prevously af_purge) * - Makes sure that all dirty buffers are written. * - Sets af->pb=NULL (no current page) * - (returns 0 if success, -1 if failure.) @@ -549,7 +549,7 @@ void af_cache_writethrough(AFFILE *af,int64_t pagenum,const unsigned char *buf,i } } } - + #ifdef HAVE_MALLOC_H #include #endif @@ -584,7 +584,7 @@ struct aff_pagebuf *af_cache_alloc(AFFILE *af,int64_t pagenum) break; } } - if(slot==-1){ + if(slot==-1){ /* Find the oldest cache entry */ int oldest_i = 0; int oldest_t = af->pbcache[0].last; diff --git a/lib/afflib_stream.cpp b/lib/afflib_stream.cpp index e1e12d8..7f5143e 100644 --- a/lib/afflib_stream.cpp +++ b/lib/afflib_stream.cpp @@ -16,7 +16,7 @@ #ifdef _WIN32 #define ASIZE SSIZE_T #else -#define ASIZE ssize_t +#define ASIZE ssize_t #endif @@ -110,7 +110,7 @@ extern "C" ASIZE af_read(AFFILE *af,unsigned char *buf,ASIZE count) if(af->pb==0){ int64_t pagenum = offset / af->image_pagesize; af->pb = af_cache_alloc(af,pagenum); - if(af->pb->pagebuf_valid==0){ + if(af->pb->pagebuf_valid==0){ /* page buffer isn't valid; need to get it */ af->pb->pagebuf_bytes = af->image_pagesize; // we can hold this much if(af_get_page(af,af->pb->pagenum,af->pb->pagebuf, &af->pb->pagebuf_bytes)){ @@ -124,7 +124,7 @@ extern "C" ASIZE af_read(AFFILE *af,unsigned char *buf,ASIZE count) } // Compute how many bytes can be copied... // where we were reading from - u_int page_offset = (u_int)(offset - af->pb->pagenum * af->image_pagesize); + u_int page_offset = (u_int)(offset - af->pb->pagenum * af->image_pagesize); if(page_offset > af->pb->pagebuf_bytes){ /* Page is short. */ @@ -138,7 +138,7 @@ extern "C" ASIZE af_read(AFFILE *af,unsigned char *buf,ASIZE count) if(bytes_to_read > page_left) bytes_to_read = page_left; if(bytes_to_read > af->image_size - offset) bytes_to_read = (u_int)(af->image_size - offset); - assert(bytes_to_read >= 0); // + assert(bytes_to_read >= 0); // if(bytes_to_read==0) break; // that's all we could get /* Copy out the bytes for the user */ @@ -173,9 +173,9 @@ int af_write(AFFILE *af,unsigned char *buf,size_t count) af_invalidate_vni_cache(af); /* vnode write bypass: - * If a write function is defined, use it and avoid the page and cache business. + * If a write function is defined, use it and avoid the page and cache business. */ - if (af->v->write){ + if (af->v->write){ int r = (af->v->write)(af, buf, af->pos, count); if(r>0){ af->pos += r; @@ -224,7 +224,7 @@ int af_write(AFFILE *af,unsigned char *buf,size_t count) AF_UNLOCK(af); return -1; // error } - + /* Can't use high-speed optimization; write through the cache */ int total = 0; @@ -245,15 +245,15 @@ int af_write(AFFILE *af,unsigned char *buf,size_t count) } } // where writing to - u_int seg_offset = (u_int)(offset - af->pb->pagenum * af->image_pagesize); + u_int seg_offset = (u_int)(offset - af->pb->pagenum * af->image_pagesize); // number we can write into - u_int seg_left = af->image_pagesize - seg_offset; + u_int seg_left = af->image_pagesize - seg_offset; u_int bytes_to_write = count; if(bytes_to_write > seg_left) bytes_to_write = seg_left; - assert(bytes_to_write >= 0); // + assert(bytes_to_write >= 0); // if(bytes_to_write==0) break; // that's all we could get /* Copy out the bytes for the user */ @@ -275,7 +275,7 @@ int af_write(AFFILE *af,unsigned char *buf,size_t count) /* If we wrote out all of the bytes that were left in the segment, * then we are at the end of the segment, write it back... */ - if(seg_left == bytes_to_write){ + if(seg_left == bytes_to_write){ if(af_cache_flush(af)){ AF_UNLOCK(af); return -1; diff --git a/lib/afflib_util.cpp b/lib/afflib_util.cpp index 9e32405..74687d9 100644 --- a/lib/afflib_util.cpp +++ b/lib/afflib_util.cpp @@ -72,7 +72,7 @@ void warnx(const char *fmt,...) -/* +/* * af_hexbuf: * Turn a binay string into a hex string, optionally with spaces. */ @@ -91,7 +91,7 @@ const char *af_hexbuf(char *dst,int dst_len,const unsigned char *bin,int bytes,i dst_len -= 2; bytes--; charcount++; // how many characters - + if((flag & AF_HEXBUF_SPACE4) && charcount%2==0){ *dst++ = ' '; *dst = '\000'; @@ -314,5 +314,5 @@ void af_parse_url(const char *url,char **protocol,char **hostname,char **usernam free(scratch); url = slash+1; } - if(path) *path = strdup(url); // remember file name + if(path) *path = strdup(url); // remember file name } diff --git a/lib/aftest.cpp b/lib/aftest.cpp index 5be9b4e..3ec5695 100644 --- a/lib/aftest.cpp +++ b/lib/aftest.cpp @@ -28,7 +28,7 @@ const char *progname = 0; const char *opt_protocol = "file:///"; const char *opt_ext = "aff"; int opt_compression_level = AF_COMPRESSION_DEFAULT;// default compression level -int opt_compression_type = AF_COMPRESSION_ALG_ZLIB; // +int opt_compression_type = AF_COMPRESSION_ALG_ZLIB; // const char *tempdir = "/tmp/"; @@ -72,7 +72,7 @@ int sequential_test() const char *fmt = "this is line %d\n"; printf("Sequential test...\n"); - + AFFILE *af = open_testfile("test_sequential",1); for(int i=0;i #include - + void rsatest() { const EVP_MD *sha256 = EVP_get_digestbyname("sha256"); @@ -779,9 +779,9 @@ void rsatest() X509 *x = 0; PEM_read_bio_X509(bp,&x,0,0); EVP_PKEY *pubkey = X509_get_pubkey(x); - + printf("pubkey=%p\n",pubkey); - + EVP_VerifyInit(&md,sha256); EVP_VerifyUpdate(&md,ptext,sizeof(ptext)); int r = EVP_VerifyFinal(&md,sig,siglen,pubkey); @@ -963,7 +963,7 @@ int main(int argc,char **argv) case 'f': figure(optarg); break; case 'e': opt_ext = optarg; break; case 'c': compress(optarg); break; - + case 'p': opt_protocol = optarg; break; case 'x': xmltest(optarg);break; case 'C': aestest(); break; @@ -975,7 +975,7 @@ int main(int argc,char **argv) usage(); } } - + if(do_bugs || do_all) bugs_test(); if(do_sequential || do_all) sequential_test(); if(do_reverse || do_all ) reverse_test(); @@ -984,7 +984,7 @@ int main(int argc,char **argv) if(do_image_test || do_all) image_test(); for(int i=0;i -# include +# include # ifndef DELTA_EPOCH_IN_MICROSECS # if defined(_MSC_VER) || defined(_MSC_EXTENSIONS) # define DELTA_EPOCH_IN_MICROSECS 11644473600000000Ui64 @@ -66,15 +66,15 @@ inline void timestamp(struct timeval *t) tmpres |= ft.dwHighDateTime; tmpres <<= 32; tmpres |= ft.dwLowDateTime; - + /*converting file time to unix epoch*/ - tmpres -= DELTA_EPOCH_IN_MICROSECS; + tmpres -= DELTA_EPOCH_IN_MICROSECS; tmpres /= 10; /*convert into microseconds*/ t->tv_sec = (long)(tmpres / 1000000UL); t->tv_usec = (long)(tmpres % 1000000UL); #else gettimeofday(t,NULL); -#endif +#endif } inline void aftimer::start() @@ -115,7 +115,7 @@ inline std::string aftimer::hms(long t) { char buf[64]; int days = t / (60*60*24); - + t = t % (60*60*24); /* what's left */ int h = t / 3600; @@ -173,7 +173,7 @@ inline std::string aftimer::eta_time(double fraction_done) #else tm = *localtime(&when); #endif - + char buf[64]; snprintf(buf,sizeof(buf),"%02d:%02d:%02d",tm.tm_hour,tm.tm_min,tm.tm_sec); return std::string(buf); diff --git a/lib/base64.cpp b/lib/base64.cpp index fcbb67b..1de680e 100755 --- a/lib/base64.cpp +++ b/lib/base64.cpp @@ -108,9 +108,9 @@ static const char Pad64 = '='; end of the data is performed using the '=' character. Since all base64 input is an integral number of octets, only the - ------------------------------------------------- + ------------------------------------------------- following cases can arise: - + (1) the final quantum of encoding input is an integral multiple of 24 bits; here, the final unit of encoded output will be an integral multiple of 4 characters @@ -123,7 +123,7 @@ static const char Pad64 = '='; characters followed by one "=" padding character. */ -extern "C" +extern "C" int b64_ntop(u_char const *src, size_t srclength, char *target, size_t targsize) { size_t datalength = 0; @@ -153,14 +153,14 @@ b64_ntop(u_char const *src, size_t srclength, char *target, size_t targsize) { target[datalength++] = Base64[output[2]]; target[datalength++] = Base64[output[3]]; } - + /* Now we worry about padding. */ if (0 != srclength) { /* Get what's left. */ input[0] = input[1] = input[2] = '\0'; for (i = 0; i < srclength; i++) input[i] = *src++; - + output[0] = input[0] >> 2; output[1] = ((input[0] & 0x03) << 4) + (input[1] >> 4); output[2] = ((input[1] & 0x0f) << 2) + (input[2] >> 6); @@ -275,7 +275,7 @@ b64_pton_slg(char const *src, int srclen, u_char *target, size_t targsize) if (ch == Pad64) { /* We got a pad char. */ ch = *src++; /* Skip it, get next. */ srclen--; - + switch (state) { case 0: /* Invalid = in first position */ case 1: /* Invalid = in second position */ diff --git a/lib/crypto.cpp b/lib/crypto.cpp index 75c0f6b..ee2ad49 100644 --- a/lib/crypto.cpp +++ b/lib/crypto.cpp @@ -62,7 +62,7 @@ int af_is_signature_segment(const char *segname){ /**************************************************************** *** AES ENCRYPTION LAYER ****************************************************************/ - + static const char *aff_cannot_sign = "AFFLIB: OpenSSL does not have SHA256! "\ "AFF segments cannot be signed. "\ "See http://www.afflib.org/requirements.php for additional information."; @@ -94,7 +94,7 @@ void af_crypto_deallocate(AFFILE *af) #ifdef AES_BLOCK_SIZE memset(&af->crypto->ekey,0,sizeof(af->crypto->ekey)); memset(&af->crypto->dkey,0,sizeof(af->crypto->dkey)); -#endif +#endif #ifdef HAVE_PEM_READ_BIO_RSA_PUBKEY if(af->crypto->sign_privkey){ EVP_PKEY_free(af->crypto->sign_privkey); @@ -143,7 +143,7 @@ int af_set_aes_key(AFFILE *af,const unsigned char *userKey,const int bits) -/** +/** * Take an unencrypted AFFKEY, encrypt it with the SHA256 of the passphrase, * and save it in the appropriate segment. */ @@ -160,7 +160,7 @@ int af_save_aes_key_with_passphrase(AFFILE *af,const char *passphrase, const u_c struct affkey affkey_seg; assert(sizeof(affkey_seg)==AFFKEY_SIZE); memset((unsigned char *)&affkey_seg,0,sizeof(affkey_seg)); - + uint32_t version_number = htonl(1); // version 1 memcpy(affkey_seg.version,(u_char *)&version_number,4); memcpy(affkey_seg.affkey_aes256,affkey,32); @@ -187,7 +187,7 @@ int af_save_aes_key_with_passphrase(AFFILE *af,const char *passphrase, const u_c /** MacOS 10.5 with GCC 4.0.1 packed affkey at 52 bytes. ** Linux GCC 4.1.2 packed affkey at 56 bytes. It should be 52 bytes ** --- 4 bytes for the version number, 32 bytes for the affkey, 16 bytes for encryption of zeros. - ** original code specified the version as uint32_t version:32, for which the + ** original code specified the version as uint32_t version:32, for which the ** compiler allocated 64 bits... ** So this code needs to be willing to accept a 52-byte or 56-byte affkey. **/ @@ -223,7 +223,7 @@ int af_get_aes_key_from_passphrase(AFFILE *af,const char *passphrase, if(sizeof(affkey_seg)==klen){ // On-disk structure is correct; copy it over - memcpy(&affkey_seg,kbuf,klen); + memcpy(&affkey_seg,kbuf,klen); memcpy((char *)&version,affkey_seg.version,4); kversion = ntohl(version); } else { @@ -233,7 +233,7 @@ int af_get_aes_key_from_passphrase(AFFILE *af,const char *passphrase, memcpy(affkey_seg.affkey_aes256,kbuf+4,sizeof(affkey_seg.affkey_aes256)); memcpy(affkey_seg.zeros_aes256,kbuf+36,sizeof(affkey_seg.zeros_aes256)); } - + /* make sure version is correct */ if(kversion != 1){ errno = EINVAL; @@ -333,7 +333,7 @@ int af_change_aes_passphrase(AFFILE *af,const char *oldphrase,const char *newph unsigned char affkey[32]; int r = af_get_aes_key_from_passphrase(af,oldphrase,affkey); - + if(r) return r; r = af_save_aes_key_with_passphrase(af,newphrase,affkey); memset(affkey,0,sizeof(affkey)); // erase the temp data @@ -427,7 +427,7 @@ int af_set_sign_files(AFFILE *af,const char *keyfile,const char *certfile) (*af->error_reporter)(aff_cannot_sign); return AF_ERROR_NO_SHA256; // } - + BIO *bp = BIO_new_file(keyfile,"r"); if(!bp) return -1; af->crypto->sign_privkey = PEM_read_bio_PrivateKey(bp,0,0,NULL); @@ -507,7 +507,7 @@ int af_sign_seg3(AFFILE *af,const char *segname, int af_sign_seg(AFFILE *af,const char *segname) { size_t datalen = 0; - + /* Now get the data to verify */ if(af_get_seg(af,segname,0,0,&datalen)){ return AF_ERROR_SIG_DATAREAD_ERROR; // can't read the segment length @@ -603,7 +603,7 @@ int af_hash_verify_seg2(AFFILE *af,const char *segname,u_char *sigbuf_,size_t si if(af_get_seg(af,segname,0,0,&seglen)){ return AF_ERROR_SIG_DATAREAD_ERROR; // can't read the segment length } - + /* Now read the segment */ segbuf=(unsigned char *)malloc(seglen); if(segbuf==0) return AF_ERROR_SIG_MALLOC; @@ -664,7 +664,7 @@ int af_sig_verify_seg2(AFFILE *af,const char *segname,EVP_PKEY *pubkey,u_char *s if(af_get_seg(af,segname,0,0,&seglen)){ return AF_ERROR_SIG_DATAREAD_ERROR; // can't read the segment length } - + /* Now read the segment */ segbuf=(unsigned char *)malloc(seglen); if(segbuf==0) return AF_ERROR_SIG_MALLOC; @@ -711,12 +711,12 @@ int af_sig_verify_seg(AFFILE *af,const char *segname) BIO_free(cert_bio); af->crypto->sign_pubkey = X509_get_pubkey(af->crypto->sign_cert); } - + /* Figure out the signature segment name */ char sigseg[AF_MAX_NAME_LEN + 1 + sizeof(AF_SIG256_SUFFIX)]; strlcpy(sigseg,segname,sizeof(sigseg)); strlcat(sigseg,AF_SIG256_SUFFIX,sizeof(sigseg)); - + /* Get the signature (it says how we need to handle the data) */ unsigned char sigbuf[2048]; // big enough to hold any conceivable signature size_t sigbuf_len=sizeof(sigbuf); @@ -768,7 +768,7 @@ int af_set_seal_certificates(AFFILE *af,const char *certfiles[],int numcertfile int r = RAND_bytes(affkey,sizeof(affkey)); if(r!=1) r = RAND_pseudo_bytes(affkey,sizeof(affkey)); // true random not supported if(r!=1) return AF_ERROR_RNG_FAIL; // pretty bad... - + af_seal_affkey_using_certificates(af, certfiles, numcertfiles, affkey); return 0; } @@ -786,10 +786,10 @@ int af_seal_affkey_using_certificates(AFFILE *af,const char *certfiles[],int nu /* Repeat for each public key.. */ int r; for(int i=0;i buflen) return E_FAIL; memcpy(buf+ptr,data,size); @@ -88,7 +88,7 @@ int lzma_compress(unsigned char *dest,size_t *destLen,const unsigned char *data, }; const int nprops = sizeof(propIDs) / sizeof(propIDs[0]); PROPVARIANT p[nprops]; - + p[0].vt = VT_UI4; p[0].ulVal = UInt32(1 << 24); p[1].vt = VT_UI4; p[1].ulVal = UInt32(2); // posBits p[2].vt = VT_UI4; p[2].ulVal = UInt32(3); // literal context bits @@ -116,7 +116,7 @@ int lzma_compress(unsigned char *dest,size_t *destLen,const unsigned char *data, COutMemoryStream *outStream = new COutMemoryStream(dest,*destLen,destLen); outStream->AddRef(); - encoder->WriteCoderProperties(outStream); + encoder->WriteCoderProperties(outStream); for (int i = 0; i < 8; i++) { Byte b = Byte(fileSize >> (8 * i)); diff --git a/lib/s3.cpp b/lib/s3.cpp index 0e42834..e9e03c2 100644 --- a/lib/s3.cpp +++ b/lib/s3.cpp @@ -1,11 +1,11 @@ -/* +/* * s3.cpp: * The stand-alone S3 program. * * These features would be nice: * have "ls" use Delimiter option to just list the drives * Give "ls" an option to list just the AFF files. - * Have implementation read a list of all the segments on open, and cache this. + * Have implementation read a list of all the segments on open, and cache this. * * Distributed under the Berkeley 4-part license. * Simson L. Garfinkel, March 12, 2012 @@ -119,7 +119,7 @@ void s3_ls(FILE *out,const char *prefix,cvector *cv) strlcpy(tstamp,(*i)->LastModified.c_str(),sizeof(tstamp)); tstamp[10] = ' '; tstamp[19] = '\000'; - + if(out){ fprintf(out,"%s ",(*i)->OwnerDisplayName.c_str()); fprintf(out,"%8d ",(int)(*i)->Size); @@ -191,7 +191,7 @@ void s3_delp(int argc,char **argv) argv++; argc--; } - + if(argc!=1) errx(1,"delp requires a single argument"); s3_ls(f,*argv,&cv); @@ -314,7 +314,7 @@ int regress() snprintf(value,sizeof(value),"This is the contents of bucket %d\n",i); object_put(opt_bucket,name,value,strlen(value),0); } - + for(int i=0;i<100;i++){ if(i%10==0) printf("\n"); class s3_result *e = list_bucket(opt_bucket,"","",0); @@ -375,7 +375,7 @@ void s3_bandwidth(int argc,char **argv) char randp[1024]; snprintf(base,sizeof(base),"%s.%d",BANDWIDTH_PREFIX,size); snprintf(randp,sizeof(randp),"%s.%d.%d",BANDWIDTH_PREFIX,size,random() % 1000); - + aftimer twrite; @@ -392,7 +392,7 @@ void s3_bandwidth(int argc,char **argv) write_err += s3_object_put_retry_count; twrite.stop(); - + if(opt_make) exit(0); // file made /* Delete the writes */ @@ -410,7 +410,7 @@ void s3_bandwidth(int argc,char **argv) char rkey[1024]; snprintf(rkey,sizeof(rkey),"%s.%d",base,bandwidth_offset); response_buffer *r=0; - + if(opt_url==0){ r = object_get(opt_bucket,rkey,0); if(!r || r->result==404){ diff --git a/lib/s3_glue.cpp b/lib/s3_glue.cpp index 71adab5..c77b78e 100644 --- a/lib/s3_glue.cpp +++ b/lib/s3_glue.cpp @@ -104,7 +104,7 @@ size_t buffer::read(char *b,size_t count){ } return 0; } - + void buffer::print() { fwrite(base,1,len,stdout); } @@ -187,7 +187,7 @@ static void endElement(void *userData, const char *name) } #ifdef BAD_STL einfo->cbuf = ""; -#else +#else einfo->cbuf.clear(); #endif einfo->depth--; @@ -200,7 +200,7 @@ static void characterDataHandler(void *userData,const XML_Char *s,int len) } -static class s3_result *xml_extract_response(const class buffer *buf) +static class s3_result *xml_extract_response(const class buffer *buf) { class s3_result *e = new s3_result(); @@ -230,7 +230,7 @@ static string canonical_string(string method,string path,curl_slist *headers, ti { /* Iterate through the headers a line at a time */ map interesting_headers; - + for(;headers;headers = headers->next){ char *line = strdup(headers->data); char *word; @@ -360,11 +360,11 @@ static int hexval(int ch) { return (isdigit(ch) ? ch-'0' : ch-'a'+10);} * sendbuf - if we are sending something ,this is what is being sent. * sendbuflen - how long that buffer is * extraheaders - any additional headers that should be sent; useful for metadata - * + * * Returns a response buffer */ -/* CURLINFO_RESPONSE_CODE is the new name for the option previously known as +/* CURLINFO_RESPONSE_CODE is the new name for the option previously known as * CURLINFO_HTTP_CODE. */ @@ -380,7 +380,7 @@ class response_buffer *request(string method,string path,string query,time_t exp /* Note: this function is not threadsafe */ static bool curl_initted = false; if(!curl_initted){ - curl_global_init(CURL_GLOBAL_ALL); + curl_global_init(CURL_GLOBAL_ALL); curl_initted=true; } @@ -391,7 +391,7 @@ class response_buffer *request(string method,string path,string query,time_t exp if(s3_debug>1) printf("==================================================\n"); if(s3_debug && retry_count>0) printf("=== S3 RETRY %d ===\n",retry_count); - + CURL *c = curl_easy_init(); struct curl_slist *headers=NULL; @@ -421,7 +421,7 @@ class response_buffer *request(string method,string path,string query,time_t exp /* Create an Authorization header */ char authorization[96]; - + snprintf(authorization,sizeof(authorization),"Authorization: AWS %s:%s", aws_access_key_id,encoded_canonical.c_str()); headers = curl_slist_append(headers, authorization); @@ -492,7 +492,7 @@ class response_buffer *request(string method,string path,string query,time_t exp s3_bytes_read += b->len; // CURL API says do not assume NULL terminate, so terminate it - h->write("\000",1); + h->write("\000",1); curl_easy_getinfo(c,CURLINFO_RESPONSE_CODE,&b->result); /* Now clean up */ @@ -596,7 +596,7 @@ class s3_result *list_bucket(string bucket,string prefix,string marker,int max_k return r; } -/* +/* * af_hexbuf: * Turn a binay string into a hex string, optionally with spaces. */ @@ -619,7 +619,7 @@ static const char *hexbuf(char *dst,int dst_len,const unsigned char *bin,int byt dst_len -= 2; bytes--; charcount++; // how many characters - + bool add_spaces = false; if(flag & HEXBUF_SPACE2) add_spaces = true; if((flag & HEXBUF_SPACE4) && charcount%2==0){ @@ -635,7 +635,7 @@ static const char *hexbuf(char *dst,int dst_len,const unsigned char *bin,int byt /* object_put: * Put an object. Make sure that the MD5 of the response matches.. * Makes a few retry attempts - * Return 0 if success, -1 if failure. + * Return 0 if success, -1 if failure. */ int object_put(string bucket,string path, const char *buf,size_t buflen, @@ -668,7 +668,7 @@ int object_put(string bucket,string path, } /* Write failed. Delete the written object and return */ response_buffer *res = request("DELETE",bucket + "/" + path,"",0,0,0,0); - if(res) delete res; + if(res) delete res; errno = EIO; return -1; } diff --git a/lib/s3_glue.h b/lib/s3_glue.h index d59dedb..5bc5294 100644 --- a/lib/s3_glue.h +++ b/lib/s3_glue.h @@ -25,7 +25,7 @@ #define S3_DEFAULT_BUCKET "S3_DEFAULT_BUCKET" #define S3_DEBUG "S3_DEBUG" #define AWS_ACCESS_KEY_ID "AWS_ACCESS_KEY_ID" -#define AWS_SECRET_ACCESS_KEY "AWS_SECRET_ACCESS_KEY" +#define AWS_SECRET_ACCESS_KEY "AWS_SECRET_ACCESS_KEY" extern int s3_debug; extern int s3_retry_max; // default 5; you can set however you wish @@ -52,7 +52,7 @@ namespace s3 { class buffer { public: - char *base; // array + char *base; // array size_t len; // length int ptr; // for reading bool writable; diff --git a/lib/utils.cpp b/lib/utils.cpp index 0843f21..777a22f 100644 --- a/lib/utils.cpp +++ b/lib/utils.cpp @@ -64,7 +64,7 @@ namespace aff { while(af_get_next_seg(af,name_,sizeof(name_),&arg_,0,&len_)==0){ // We shouldn't have 0-len segment names, but we do in some files. // Don't copy these segments. - if(strlen(name_)>0){ + if(strlen(name_)>0){ seginfo si(name_,len_,arg_); push_back(si); } @@ -84,7 +84,7 @@ namespace aff { bool seglist::contains(std::string segname) - { + { for(std::vector::const_iterator i = begin(); i!=end(); i++){ if(i->name == segname) return true; } @@ -93,4 +93,4 @@ namespace aff { #endif } - + diff --git a/lib/vnode_afd.cpp b/lib/vnode_afd.cpp index 70decd2..011386d 100644 --- a/lib/vnode_afd.cpp +++ b/lib/vnode_afd.cpp @@ -1,6 +1,6 @@ /* * vnode_aff.cpp: - * + * * Functions for the manipulation of AFF files... * Distributed under the Berkeley 4-part license */ @@ -292,7 +292,7 @@ static int afd_add_file(AFFILE *af,const char *fname_) int new_file = access(fname,F_OK)!=0; // Is this a new file? - AFFILE *af2 = af_open(fname,af->openflags|AF_NO_CRYPTO,af->openmode); + AFFILE *af2 = af_open(fname,af->openflags|AF_NO_CRYPTO,af->openmode); if(af2==0){ (*af->error_reporter)("open(%s,%d,%d) failed: %s\n", fname,af->openflags,af->openmode,strerror(errno)); @@ -309,18 +309,18 @@ static int afd_add_file(AFFILE *af,const char *fname_) af_set_pagesize(af2,af->image_pagesize); // af_set_sectorsize(af2,af->image_sectorsize); af_update_seg(af,AF_AFF_FILE_TYPE,0,(const u_char *)"AFD",3); - + /* If this is the second file, copy over additional metadata from first... */ if(ap->num_afs>1){ AFFILE *af0 = ap->afs[0]; memcpy(af2->badflag,af0->badflag,af->image_sectorsize); af2->bytes_memcpy += af->image_sectorsize; - + for(const char **segname=segs_to_copy;*segname;segname++){ unsigned char data[65536]; // big enough for most metadata size_t datalen = sizeof(data); uint32_t arg=0; - + if(af_get_seg(af0,*segname,&arg,data,&datalen)==0){ int r = af_update_seg(af2,*segname,arg,data,datalen); if(r!=0){ @@ -331,12 +331,12 @@ static int afd_add_file(AFFILE *af,const char *fname_) } } } - + return 0; } - - - + + + /**************************************************************** *** User-visible functions. ****************************************************************/ @@ -348,7 +348,7 @@ static int afd_open(AFFILE *af) /* If the name ends with a '/', remove it */ char *lastc = af->fname + strlen(af->fname) - 1; if(*lastc=='/') *lastc = '\000'; - + /* If the directory doesn't exist, make it (if we are O_CREAT) */ struct stat sb; @@ -370,7 +370,7 @@ static int afd_open(AFFILE *af) errno = ENOTDIR; // needs to be a directory return -1; } - + af->maxsize = AFD_DEFAULT_MAXSIZE; af->vnodeprivate = (void *)calloc(1,sizeof(struct afd_private)); @@ -486,7 +486,7 @@ static int afd_rewind_seg(AFFILE *af) for(int i=0;inum_afs;i++){ af_rewind_seg(ap->afs[i]); } - return 0; + return 0; } @@ -498,7 +498,7 @@ static int afd_rewind_seg(AFFILE *af) */ static int afd_update_seg(AFFILE *af, const char *name, uint32_t arg,const u_char *value,uint32_t vallen) - + { struct afd_private *ap = AFD_PRIVATE(af); AFFILE *af2 = afd_file_with_seg(af,name); @@ -540,8 +540,8 @@ int afd_del_seg(AFFILE *af,const char *segname) struct af_vnode vnode_afd = { - AF_IDENTIFY_AFD, // - AF_VNODE_TYPE_COMPOUND|AF_VNODE_TYPE_RELIABLE, // + AF_IDENTIFY_AFD, // + AF_VNODE_TYPE_COMPOUND|AF_VNODE_TYPE_RELIABLE, // "AFF Directory", afd_identify_file, afd_open, // open diff --git a/lib/vnode_aff.cpp b/lib/vnode_aff.cpp index 9d51ce7..499ca4c 100644 --- a/lib/vnode_aff.cpp +++ b/lib/vnode_aff.cpp @@ -1,6 +1,6 @@ /* * vnode_aff.cpp: - * + * * Functions for the manipulation of AFF files... * Distributed under the Berkeley 4-part license */ @@ -22,11 +22,11 @@ static int aff_write_ignore(AFFILE *af,size_t bytes); static int aff_write_seg(AFFILE *af,const char *name,uint32_t arg, - const u_char *value,size_t vallen); + const u_char *value,size_t vallen); static int aff_get_seg(AFFILE *af,const char *name,uint32_t *arg, unsigned char *data,size_t *datalen); #ifdef KERNEL_LIBRARY -static int aff_write_seg_no_data(AFFILE *af,const char *name,uint32_t arg, size_t vallen); +static int aff_write_seg_no_data(AFFILE *af,const char *name,uint32_t arg, size_t vallen); #endif static int aff_get_next_seg(AFFILE *af,char *segname,size_t segname_len, uint32_t *arg, unsigned char *data, size_t *datalen); @@ -140,7 +140,7 @@ int aff_write_seg(AFFILE *af, const char *segname,uint32_t arg,const u_char *dat segt.segment_len = htonl(sizeof(segh)+segname_len + datalen + sizeof(segt)); aff_toc_update(af,segname,ftello(af->aseg),datalen); - + if(af_trace) fprintf(af_trace,"aff_write_seg: putting segment %s (datalen=%d) offset=%"PRId64"\n", segname,(int)datalen,ftello(af->aseg)); @@ -183,7 +183,7 @@ int aff_write_seg_no_data(AFFILE *af, const char *segname,uint32_t arg,size_t da segt.segment_len = htonl(sizeof(segh)+segname_len + datalen + sizeof(segt)); aff_toc_update(af,segname,ftello(af->aseg),datalen); - + if(af_trace) fprintf(af_trace,"aff_write_seg: putting segment %s (datalen=%zd) offset=%"PRId64"\n", segname,datalen,ftello(af->aseg)); @@ -199,7 +199,7 @@ int aff_write_seg_no_data(AFFILE *af, const char *segname,uint32_t arg,size_t da /**************************************************************** - *** low-level routines for reading + *** low-level routines for reading ****************************************************************/ /* aff_get_segment: @@ -236,7 +236,7 @@ static int aff_get_seg(AFFILE *af,const char *name, * @param arg - pointer to the arg * @param data - pointer to the data * @param datalen_ - length of the data_ array. If *datalen_==0, set to the length of the data. - * + * * @return * 0 = success. * -1 = end of file. (AF_ERROR_EOF) @@ -338,7 +338,7 @@ int af_truncate_blank(AFFILE *af) } } fseeko(af->aseg,last_loc,SEEK_SET); // return to where we were - return -1; // say that we couldn't do it. + return -1; // say that we couldn't do it. } @@ -368,7 +368,7 @@ static int aff_update_seg(AFFILE *af, const char *name, size_t size_closest = 0; uint64_t loc_closest = 0; struct aff_toc_mem *adm = aff_toc(af,name); - + if(af_trace) fprintf(af_trace,"aff_update_seg(name=%s,arg=%"PRIu32",vallen=%u)\n",name,arg,vallen); @@ -412,7 +412,7 @@ static int aff_update_seg(AFFILE *af, const char *name, */ fseeko(af->aseg,(uint64_t)0,SEEK_END); // go to the end of the file break; // and exit this loop - + } if((next_segment_name[0]==0) && (next_datasize>=size_needed)){ @@ -439,7 +439,7 @@ static int aff_update_seg(AFFILE *af, const char *name, fseeko(af->aseg,loc_closest,SEEK_SET); // move to the location aff_write_seg(af,name,arg,value,vallen); // write the new segment - + size_t newsize = size_closest - vallen - aff_segment_overhead(0) - strlen(name); aff_write_ignore(af,newsize); // write the smaller ignore return 0; @@ -450,7 +450,7 @@ static int aff_update_seg(AFFILE *af, const char *name, /* Keep truncating until there is nothing left */ } //printf("*** appending '%s' bytes=%d to the end\n",name,vallen); - fseeko(af->aseg,0L,SEEK_END); // move back to the end of the file + fseeko(af->aseg,0L,SEEK_END); // move back to the end of the file return aff_write_seg(af,name,arg,value,vallen); // just write at the end } @@ -505,13 +505,13 @@ static int aff_del_seg(AFFILE *af,const char *segname) */ static int aff_create(AFFILE *af) { - fwrite(AF_HEADER,1,8,af->aseg); // writes the header + fwrite(AF_HEADER,1,8,af->aseg); // writes the header aff_toc_build(af); // build the toc (will be pretty small) af_make_badflag(af); // writes the flag for bad blocks - + const char *version = xstr(PACKAGE_VERSION); aff_update_seg(af,AF_AFFLIB_VERSION,0,(const u_char *)version,strlen(version)); - + #ifdef HAVE_GETPROGNAME const char *progname = getprogname(); if(aff_update_seg(af,AF_CREATOR,0,(const u_char *)progname,strlen(progname))) return -1; @@ -553,9 +553,9 @@ static int aff_identify_file(const char *filename,int exists) if(af_ext_is(filename,"aff")) return 1; return 0; } - + if(fd>0){ - int len = strlen(AF_HEADER)+1; + int len = strlen(AF_HEADER)+1; char buf[64]; int r = read(fd,buf,len); close(fd); @@ -581,7 +581,7 @@ static int aff_open(AFFILE *af) /* Open the raw file */ int fd = open(af->fname,af->openflags | O_BINARY,af->openmode); if(fd<0){ // couldn't open - return -1; + return -1; } /* Lock the file if writing */ @@ -597,13 +597,13 @@ static int aff_open(AFFILE *af) /* Set defaults */ - af->compression_type = AF_COMPRESSION_ALG_ZLIB; + af->compression_type = AF_COMPRESSION_ALG_ZLIB; af->compression_level = Z_DEFAULT_COMPRESSION; /* Open the FILE for the AFFILE */ char strflag[8]; strcpy(strflag,"rb"); // we have to be able to read - if(af->openflags & O_RDWR) strcpy(strflag,"w+b"); + if(af->openflags & O_RDWR) strcpy(strflag,"w+b"); af->aseg = fdopen(fd,strflag); if(!af->aseg){ @@ -622,7 +622,7 @@ static int aff_open(AFFILE *af) if(sb.st_size==0){ return aff_create(af); } - + /* We are opening an existing file. Verify once more than it is an AFF file * and skip past the header... */ diff --git a/lib/vnode_afm.cpp b/lib/vnode_afm.cpp index b65c051..a037a50 100644 --- a/lib/vnode_afm.cpp +++ b/lib/vnode_afm.cpp @@ -64,7 +64,7 @@ static int invalid_extension_char(const char *extension,int ext_len) static int afm_create(AFFILE *af) { - if (af_update_seg (af, AF_RAW_IMAGE_FILE_EXTENSION, 0, (const u_char *)SPLITRAW_DEFAULT_EXTENSION, + if (af_update_seg (af, AF_RAW_IMAGE_FILE_EXTENSION, 0, (const u_char *)SPLITRAW_DEFAULT_EXTENSION, strlen(SPLITRAW_DEFAULT_EXTENSION))) { (*af->error_reporter)("split_raw_read_write_setup: %s: failed to write %s\n", af->fname, AF_RAW_IMAGE_FILE_EXTENSION); @@ -81,7 +81,7 @@ static int afm_create(AFFILE *af) * sub-implementations in their own files: an AFF file * and a split_raw file. */ - + static int afm_open(AFFILE *af) { af->vnodeprivate = (void *)calloc(sizeof(struct afm_private),1); @@ -107,7 +107,7 @@ static int afm_open(AFFILE *af) /* Read the split raw extension */ char raw_file_extension[4]; size_t len=3; // don't overwrite the NUL - + memset(raw_file_extension,0,sizeof(raw_file_extension)); if (af_get_seg(ap->aff,AF_RAW_IMAGE_FILE_EXTENSION,0,(unsigned char *)raw_file_extension,&len)) { (*af->error_reporter)("afm_open: %s: %s segment missing or too large\n", @@ -121,7 +121,7 @@ static int afm_open(AFFILE *af) afm_close(af); return -1; } - + /* Now open the splitraw file */ char *sr_filename = strdup(af_filename(af)); char *ext = strrchr(sr_filename,'.'); @@ -156,7 +156,7 @@ static int afm_open(AFFILE *af) * the parameters set with af_update_seg() calls, yet the split_raw * implementation gets the proper settings */ - return 0; + return 0; } @@ -233,8 +233,8 @@ static int afm_split_raw_setup(AFFILE *af) /* Push down the image_pagesize from the AFM to the split_raw */ uint32_t image_pagesize = af->image_pagesize; // default to what's in memory af_get_seg(af,AF_PAGESIZE,&image_pagesize,0,0); // get from the AFF file if possible - ap->sr->image_pagesize = af->image_pagesize; // overwrite the default with what the AFM file - + ap->sr->image_pagesize = af->image_pagesize; // overwrite the default with what the AFM file + ap->sr_initialized = 1; return 0; } @@ -265,7 +265,7 @@ static int afm_get_seg(AFFILE *af,const char *name,uint32_t *arg,unsigned char * int64_t page_num = af_segname_page_number(name); if(page_num>=0) return af_get_seg(ap->sr,name,arg,data,datalen); return af_get_seg(ap->aff,name,arg,data,datalen); - + } @@ -317,7 +317,7 @@ static int afm_rewind_seg(AFFILE *af) */ static int afm_update_seg(AFFILE *af, const char *name, uint32_t arg,const u_char *value,uint32_t vallen) - + { struct afm_private *ap = AFM_PRIVATE(af); int64_t page_num = af_segname_page_number(name); // <0 means update metadata @@ -356,7 +356,7 @@ struct af_vnode vnode_afm = { afm_identify_file, afm_open, afm_close, - afm_raw_vstat, // nothing aff specific here. + afm_raw_vstat, // nothing aff specific here. afm_get_seg, // get seg afm_get_next_seg, // get_next_seg afm_rewind_seg, // rewind_seg diff --git a/lib/vnode_qemu.cpp b/lib/vnode_qemu.cpp index d40b231..57b4063 100644 --- a/lib/vnode_qemu.cpp +++ b/lib/vnode_qemu.cpp @@ -124,7 +124,7 @@ static int qemu_get_seg(AFFILE *af,const char *name, uint32_t *arg, return 0; } if(*datalen<8) return -2; - + struct aff_quad q; q.low = htonl((uint32_t)(af->image_size & 0xffffffff)); q.high = htonl((uint32_t)(af->image_size >> 32)); @@ -187,11 +187,11 @@ static int qemu_get_next_seg(AFFILE *af,char *segname,size_t segname_len,uint32_ char pagename[AF_MAX_NAME_LEN]; // memset(pagename,0,sizeof(pagename)); snprintf(pagename,sizeof(pagename),AF_PAGE,af->cur_page++); - + int r = 0; /* Get the segment, if it is wanted */ if(data) r = qemu_get_seg(af,pagename,arg,data,datalen); - + /* If r==0 and there is room for copying in the segment name, return it */ if(r==0){ if(strlen(pagename)+1 < segname_len){ diff --git a/lib/vnode_raw.cpp b/lib/vnode_raw.cpp index fab306e..7c7badb 100644 --- a/lib/vnode_raw.cpp +++ b/lib/vnode_raw.cpp @@ -8,7 +8,7 @@ */ /* the RAW_PAGESIZE is visible outside the module, but it's kind of irrevellant */ -#define RAW_PAGESIZE 16*1024*1024 +#define RAW_PAGESIZE 16*1024*1024 /* raw file implementation */ struct raw_private { @@ -176,7 +176,7 @@ static int raw_get_seg(AFFILE *af,const char *name, int bytes_to_read = af->image_pagesize; // copy this many bytes, unless if(bytes_to_read > bytes_left) bytes_to_read = bytes_left; // only this much is left - + if(arg) *arg = 0; // arg is always 0 if(datalen){ if(data==0){ // asked for 0 bytes, so give the actual size @@ -189,7 +189,7 @@ static int raw_get_seg(AFFILE *af,const char *name, } } if(data){ - fseeko(rp->raw,pos,SEEK_SET); + fseeko(rp->raw,pos,SEEK_SET); int bytes_read = fread(data,1,bytes_to_read,rp->raw); if(bytes_read==bytes_to_read){ if(datalen) *datalen = bytes_read; @@ -237,7 +237,7 @@ static int raw_vstat(AFFILE *af,struct af_vnode_info *vni) fflush(rp->raw); vni->imagesize = raw_filesize(af); vni->supports_compression = 0; - vni->has_pages = 1; + vni->has_pages = 1; if(rp->raw_popen){ /* popen files require special handling */ @@ -258,7 +258,7 @@ static int raw_rewind_seg(AFFILE *af) static int raw_get_next_seg(AFFILE *af,char *segname,size_t segname_len,uint32_t *arg, unsigned char *data,size_t *datalen) { - + /* See if we are at the end of the "virtual" segment list */ if((uint64_t)af->cur_page * af->image_pagesize >= af->image_size) return -1; diff --git a/lib/vnode_s3.cpp b/lib/vnode_s3.cpp index b7bf23c..5e3ddec 100644 --- a/lib/vnode_s3.cpp +++ b/lib/vnode_s3.cpp @@ -1,6 +1,6 @@ /* * vnode_aff.cpp: - * + * * Functions for the manipulation of AFF files... * * Distributed under the Berkeley 4-part license @@ -77,7 +77,7 @@ static int s3_open(AFFILE *af) /* Create the bucket if it doesn't exist */ aws_access_key_id = getenv(AWS_ACCESS_KEY_ID); aws_secret_access_key = getenv(AWS_SECRET_ACCESS_KEY); - + if(!aws_access_key_id) fprintf(stderr,"s3: AWS_ACCESS_KEY_ID not defined\n"); if(!aws_secret_access_key) fprintf(stderr,"s3: AWS_SECRET_ACCESS_KEY not defined\n"); if(!aws_access_key_id || !aws_secret_access_key) return -1; /* can't open */ @@ -108,7 +108,7 @@ static int s3_open(AFFILE *af) * file was already created. Return an error. */ bool exists = af_get_seg(af,AF_PAGESIZE,0,0,0)==0; - + if((af->openflags & O_CREAT) && (af->openflags & O_EXCL) && exists){ errno = EEXIST; return -1; @@ -252,7 +252,7 @@ static int s3_rewind_seg(AFFILE *af) delete sp->lbr; sp->lbr = 0; } - return 0; + return 0; } @@ -262,7 +262,7 @@ static int s3_rewind_seg(AFFILE *af) */ static int s3_update_seg(AFFILE *af, const char *name, uint32_t arg,const u_char *value,uint32_t vallen) - + { struct s3_private *sp =S3_PRIVATE(af); char metabuf[64]; @@ -285,8 +285,8 @@ int s3_del_seg(AFFILE *af,const char *segname) struct af_vnode vnode_s3 = { - AF_IDENTIFY_S3, // - AF_VNODE_TYPE_RELIABLE, // + AF_IDENTIFY_S3, // + AF_VNODE_TYPE_RELIABLE, // "s3.amazonaws.com", s3_identify_file, s3_open, // open @@ -308,8 +308,8 @@ static int s3_cantopen(AFFILE *af) } struct af_vnode vnode_s3 = { - AF_IDENTIFY_S3, // - AF_VNODE_TYPE_RELIABLE, // + AF_IDENTIFY_S3, // + AF_VNODE_TYPE_RELIABLE, // "s3.amazonaws.com", s3_identify_file, s3_cantopen, // open diff --git a/lib/vnode_split_raw.cpp b/lib/vnode_split_raw.cpp index 05d8d16..df9c208 100644 --- a/lib/vnode_split_raw.cpp +++ b/lib/vnode_split_raw.cpp @@ -132,12 +132,12 @@ int split_raw_increment_fname (char *fn) /* Get the case */ int lower = islower(ext[0]); - + /* Convert to all uppercase */ for(int i=0;i<3;i++){ if(isalpha(ext[i])) ext[i] = toupper(ext[i]); } - + /* Increment */ if(incval(ext[2],10)){ if(incval(ext[1],36)){ @@ -161,7 +161,7 @@ void srp_validate(AFFILE *af) for(uint32_t i=0;inum_raw_files;i++){ assert(srp->fds[i]!=0); } -} +} /** Debugging routine. */ @@ -173,7 +173,7 @@ void srp_dump(AFFILE *af) } srp_validate(af); fprintf(stderr,"===================\n"); -} +} static void srp_add_fd(AFFILE *af,int fd) { @@ -217,7 +217,7 @@ static int split_raw_open_internal(AFFILE *af, uint64_t *image_size) (*image_size) = sb.st_size; return 0; } - + /* This gets set to 1 the first time we find a file whose size doesn't match the size of the first file. If we successfully open a file when this flag is already 1, then our sanity checks fail. */ @@ -322,7 +322,7 @@ static int split_raw_read(AFFILE *af, unsigned char *buf, uint64_t pos,size_t co while (count > 0) { int filenum = -1; off_t file_offset = 0; - + if (af->maxsize) { // if we do file segments filenum = (int)(pos / af->maxsize); file_offset = (off_t)(pos % af->maxsize); @@ -439,7 +439,7 @@ int split_raw_write_internal2(AFFILE *af, unsigned char *buf, uint64_t pos,size_ /* DONE! */ acbi.bytes_written = c4; - if(af->w_callback) {acbi.phase = 4;(*af->w_callback)(&acbi);} + if(af->w_callback) {acbi.phase = 4;(*af->w_callback)(&acbi);} if (c4 <= 0) { // some error writing? if (ret) return ret; @@ -452,7 +452,7 @@ int split_raw_write_internal2(AFFILE *af, unsigned char *buf, uint64_t pos,size_ pos += c4; srp->pos[i] += c4; if (af->image_size < pos) af->image_size = pos; // image was extended - if (c3 != c4){ // amount written doesn't equal request; return + if (c3 != c4){ // amount written doesn't equal request; return return ret; } } @@ -467,13 +467,13 @@ int split_raw_write(AFFILE *af, unsigned char *buf, uint64_t pos,size_t count) if (af->maxsize) { if (pos > af->image_size) { // writing beyond the end... - while(pos > af->image_size){ + while(pos > af->image_size){ /* repeat until file is as big as where we should be writing */ int64_t bytes_left = pos - af->image_size; - int bytes_to_write = (int)(af->maxsize - (af->image_size % af->maxsize)); + int bytes_to_write = (int)(af->maxsize - (af->image_size % af->maxsize)); if(bytes_to_write > bytes_left) bytes_to_write = (int)bytes_left; - int bytes_written = split_raw_write_internal2(af,0,af->image_size,bytes_to_write); + int bytes_written = split_raw_write_internal2(af,0,af->image_size,bytes_to_write); if(bytes_to_write != bytes_written){ return -1; // some kind of internal error } @@ -530,13 +530,13 @@ static int split_raw_get_seg(AFFILE *af,const char *name,uint32_t *arg,unsigned errno = ENOTSUP; // sorry! We don't store metadata return -1; } - + uint64_t pos = page_num * af->image_pagesize; // where we are to start reading uint64_t bytes_left = af->image_size - pos; // how many bytes left in the file uint32_t bytes_to_read = af->image_pagesize; // copy this many bytes, unless if(bytes_to_read > bytes_left) bytes_to_read = (uint32_t)bytes_left; // only this much is left - + if(arg) *arg = 0; // arg is always 0 if(datalen){ if(data==0){ // asked for 0 bytes, so give the actual size @@ -604,14 +604,14 @@ static int split_raw_rewind_seg(AFFILE *af) static int split_raw_update_seg(AFFILE *af, const char *name, uint32_t arg,const u_char *value,uint32_t vallen) - + { int64_t page_num = af_segname_page_number(name); if(page_num<0){ errno = ENOTSUP; // sorry! We don't store metadata return -1; } - + uint64_t pos = page_num * af->image_pagesize; // where we are to start reading int written = split_raw_write(af, (unsigned char *)value, pos,vallen); if(written==(int)vallen) return 0; // success From 8cdef829ef84fa45c40c2370210cb87ff01f84c9 Mon Sep 17 00:00:00 2001 From: John Lindgren Date: Mon, 25 Feb 2013 15:59:42 -0500 Subject: [PATCH 02/14] Clear execute bit on base64.{cpp,h}. --- lib/base64.cpp | 0 lib/base64.h | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 lib/base64.cpp mode change 100755 => 100644 lib/base64.h diff --git a/lib/base64.cpp b/lib/base64.cpp old mode 100755 new mode 100644 diff --git a/lib/base64.h b/lib/base64.h old mode 100755 new mode 100644 From aea62bc0b642e5dc8e125cd96d848d2d5d7325a9 Mon Sep 17 00:00:00 2001 From: John Lindgren Date: Mon, 25 Feb 2013 16:16:27 -0500 Subject: [PATCH 03/14] Remove leftover config.h.in. --- config.h.in | 446 ---------------------------------------------------- 1 file changed, 446 deletions(-) delete mode 100644 config.h.in diff --git a/config.h.in b/config.h.in deleted file mode 100644 index 29d8294..0000000 --- a/config.h.in +++ /dev/null @@ -1,446 +0,0 @@ -/* config.h.in. Generated from configure.ac by autoheader. */ - -/* Define to one of `_getb67', `GETB67', `getb67' for Cray-2 and Cray-YMP - systems. This function is required for `alloca.c' support on those systems. - */ -#undef CRAY_STACKSEG_END - -/* Define to 1 if using `alloca.c'. */ -#undef C_ALLOCA - -/* Define to 1 if you have the `AES_encrypt' function. */ -#undef HAVE_AES_ENCRYPT - -/* Define to 1 if you have `alloca', as a function or macro. */ -#undef HAVE_ALLOCA - -/* Define to 1 if you have and it should be used (not on Ultrix). - */ -#undef HAVE_ALLOCA_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_ARPA_INET_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_ASSERT_H - -/* Define to 1 if you have the `beep' function. */ -#undef HAVE_BEEP - -/* Define to 1 if you have the `CC_SHA256_Init' function. */ -#undef HAVE_CC_SHA256_INIT - -/* Define to 1 if you have the header file. */ -#undef HAVE_COMMONCRYPTO_COMMONDIGEST_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_CTYPE_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_CURSES_H - -/* Define to 1 if you have the `des_read_pw_string' function. */ -#undef HAVE_DES_READ_PW_STRING - -/* Define to 1 if you have the header file, and it defines `DIR'. - */ -#undef HAVE_DIRENT_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_DLFCN_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_DMALLOC_H - -/* Define to 1 if you have the `endwin' function. */ -#undef HAVE_ENDWIN - -/* Define to 1 if you have the `err' function. */ -#undef HAVE_ERR - -/* Define to 1 if you have the header file. */ -#undef HAVE_ERRNO_H - -/* Define to 1 if you have the `errx' function. */ -#undef HAVE_ERRX - -/* Define to 1 if you have the header file. */ -#undef HAVE_ERR_H - -/* Define to 1 if you have the `err_set_exit' function. */ -#undef HAVE_ERR_SET_EXIT - -/* Define to 1 if you have the `EVP_read_pw_string' function. */ -#undef HAVE_EVP_READ_PW_STRING - -/* Define to 1 if you have the `EVP_sha256' function. */ -#undef HAVE_EVP_SHA256 - -/* Define to 1 if you have the header file. */ -#undef HAVE_FCNTL_H - -/* Define to 1 if fseeko (and presumably ftello) exists and is declared. */ -#undef HAVE_FSEEKO - -/* Define to 1 if you have the `fstatfs' function. */ -#undef HAVE_FSTATFS - -/* Define to 1 if you have the `ftruncate' function. */ -#undef HAVE_FTRUNCATE - -/* Define to 1 if you have the header file. */ -#undef HAVE_GETOPT_H - -/* Define to 1 if you have the `getprogname' function. */ -#undef HAVE_GETPROGNAME - -/* Define to 1 if you have the `gotorc' function. */ -#undef HAVE_GOTORC - -/* Define to 1 if you have the header file. */ -#undef HAVE_INTTYPES_H - -/* Define to 1 if you have the `isalnum' function. */ -#undef HAVE_ISALNUM - -/* Define to 1 if you have the `isalphanum' function. */ -#undef HAVE_ISALPHANUM - -/* Define to 1 if you have the `isatty' function. */ -#undef HAVE_ISATTY - -/* Define to 1 if you have the `isdigit' function. */ -#undef HAVE_ISDIGIT - -/* Define to 1 if you have the `crypto' library (-lcrypto). */ -#undef HAVE_LIBCRYPTO - -/* Define to 1 if you have the `curl' library (-lcurl). */ -#undef HAVE_LIBCURL - -/* Define to 1 if you have the `ewf' library (-lewf). */ -#undef HAVE_LIBEWF - -/* Define to 1 if you have the header file. */ -#undef HAVE_LIBEWF_H - -/* Define to 1 if you have the `expat' library (-lexpat). */ -#undef HAVE_LIBEXPAT - -/* Define to 1 if you have the `md' library (-lmd). */ -#undef HAVE_LIBMD - -/* Define to 1 if you have the `ncurses' library (-lncurses). */ -#undef HAVE_LIBNCURSES - -/* Define to 1 if you have the `readline' library (-lreadline). */ -#undef HAVE_LIBREADLINE - -/* Define to 1 if you have the `ssl' library (-lssl). */ -#undef HAVE_LIBSSL - -/* Define to 1 if you have the `z' library (-lz). */ -#undef HAVE_LIBZ - -/* Define to 1 if you have the header file. */ -#undef HAVE_LINUX_FS_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_MALLOC_H - -/* Define to 1 if you have the `MD5' function. */ -#undef HAVE_MD5 - -/* Define to 1 if you have the header file. */ -#undef HAVE_MEMORY_H - -/* Define to 1 if you have the `memset' function. */ -#undef HAVE_MEMSET - -/* Define to 1 if you have the `mkdir' function. */ -#undef HAVE_MKDIR - -/* Define to 1 if you have the header file. */ -#undef HAVE_NCURSES_TERM_H - -/* Define to 1 if you have the header file, and it defines `DIR'. */ -#undef HAVE_NDIR_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_NETINET_IN_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_OPENSSL_AES_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_OPENSSL_BIO_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_OPENSSL_EVP_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_OPENSSL_FIPS_SHA_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_OPENSSL_HMAC_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_OPENSSL_MD5_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_OPENSSL_PEM_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_OPENSSL_RAND_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_OPENSSL_RSA_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_OPENSSL_SHA_H - -/* Define to 1 if you have the `PEM_read_bio_RSA_PUBKEY' function. */ -#undef HAVE_PEM_READ_BIO_RSA_PUBKEY - -/* Define to 1 if you have the `popen' function. */ -#undef HAVE_POPEN - -/* Define to 1 if you have the `printw' function. */ -#undef HAVE_PRINTW - -/* Define to 1 if you have the `putenv' function. */ -#undef HAVE_PUTENV - -/* Define to 1 if you have the `RAND_pseudo_bytes' function. */ -#undef HAVE_RAND_PSEUDO_BYTES - -/* Define to 1 if you have the header file. */ -#undef HAVE_READLINE_READLINE_H - -/* Define to 1 if you have the `regcomp' function. */ -#undef HAVE_REGCOMP - -/* Define to 1 if you have the `setupterm' function. */ -#undef HAVE_SETUPTERM - -/* Define to 1 if you have the header file. */ -#undef HAVE_SHA256_H - -/* Define to 1 if you have the `SHA256_Init' function. */ -#undef HAVE_SHA256_INIT - -/* Define to 1 if you have the header file. */ -#undef HAVE_SIGNAL_H - -/* Do we have sockaddr.sin_len? */ -#undef HAVE_SOCKADDR_SIN_LEN - -/* Define to 1 if you have the `srandom' function. */ -#undef HAVE_SRANDOM - -/* Define to 1 if you have the `srandomdev' function. */ -#undef HAVE_SRANDOMDEV - -/* Define to 1 if stdbool.h conforms to C99. */ -#undef HAVE_STDBOOL_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_STDINT_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_STDIO_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_STDLIB_H - -/* Define to 1 if you have the `strcasecmp' function. */ -#undef HAVE_STRCASECMP - -/* Define to 1 if you have the `strchr' function. */ -#undef HAVE_STRCHR - -/* Define to 1 if you have the `strdup' function. */ -#undef HAVE_STRDUP - -/* Define to 1 if you have the `strerror' function. */ -#undef HAVE_STRERROR - -/* Define to 1 if you have the header file. */ -#undef HAVE_STRINGS_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_STRING_H - -/* Define to 1 if you have the `strlcat' function. */ -#undef HAVE_STRLCAT - -/* Define to 1 if you have the `strlcpy' function. */ -#undef HAVE_STRLCPY - -/* Define to 1 if you have the `strrchr' function. */ -#undef HAVE_STRRCHR - -/* Define to 1 if `st_blocks' is member of `struct stat'. */ -#undef HAVE_STRUCT_STAT_ST_BLOCKS - -/* Define to 1 if `st_rdev' is member of `struct stat'. */ -#undef HAVE_STRUCT_STAT_ST_RDEV - -/* Define to 1 if your `struct stat' has `st_blocks'. Deprecated, use - `HAVE_STRUCT_STAT_ST_BLOCKS' instead. */ -#undef HAVE_ST_BLOCKS - -/* Define to 1 if your `struct stat' has `st_rdev'. Deprecated, use - `HAVE_STRUCT_STAT_ST_RDEV' instead. */ -#undef HAVE_ST_RDEV - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYSEXITS_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYS_CDEFS_H - -/* Define to 1 if you have the header file, and it defines `DIR'. - */ -#undef HAVE_SYS_DIR_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYS_DISK_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYS_IOCTL_H - -/* Define to 1 if you have the header file, and it defines `DIR'. - */ -#undef HAVE_SYS_NDIR_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYS_PARAM_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYS_SOCKET_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYS_STAT_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYS_TIME_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYS_TYPES_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYS_VFS_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_TERMCAP_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_TERM_H - -/* Define to 1 if you have the `tgetnum' function. */ -#undef HAVE_TGETNUM - -/* Define to 1 if you have the `tgetstr' function. */ -#undef HAVE_TGETSTR - -/* Define to 1 if you have the `tgoto' function. */ -#undef HAVE_TGOTO - -/* Define to 1 if you have the header file. */ -#undef HAVE_TIME_H - -/* Define to 1 if you have the `tputs' function. */ -#undef HAVE_TPUTS - -/* Define to 1 if you have the header file. */ -#undef HAVE_UNISTD_H - -/* Define to 1 if you have the `valloc' function. */ -#undef HAVE_VALLOC - -/* Define to 1 if you have the `warn' function. */ -#undef HAVE_WARN - -/* Define to 1 if you have the `warnx' function. */ -#undef HAVE_WARNX - -/* Define to 1 if you have the header file. */ -#undef HAVE_ZLIB_H - -/* Define to 1 if the system has the type `_Bool'. */ -#undef HAVE__BOOL - -/* Define to 1 if your C compiler doesn't accept -c and -o together. */ -#undef NO_MINUS_C_MINUS_O - -/* Name of package */ -#undef PACKAGE - -/* Define to the address where bug reports for this package should be sent. */ -#undef PACKAGE_BUGREPORT - -/* Define to the full name of this package. */ -#undef PACKAGE_NAME - -/* Define to the full name and version of this package. */ -#undef PACKAGE_STRING - -/* Define to the one symbol short name of this package. */ -#undef PACKAGE_TARNAME - -/* Define to the version of this package. */ -#undef PACKAGE_VERSION - -/* If using the C implementation of alloca, define if you know the - direction of stack growth for your system; otherwise it will be - automatically deduced at runtime. - STACK_DIRECTION > 0 => grows toward higher addresses - STACK_DIRECTION < 0 => grows toward lower addresses - STACK_DIRECTION = 0 => direction of growth unknown */ -#undef STACK_DIRECTION - -/* Define to 1 if you have the ANSI C header files. */ -#undef STDC_HEADERS - -/* Define to 1 if your declares `struct tm'. */ -#undef TM_IN_SYS_TIME - -/* Use FUSE to mount AFF images */ -#undef USE_FUSE - -/* Use libewf to read EnCase files */ -#undef USE_LIBEWF - -/* Enable support for Amazon S3 */ -#undef USE_S3 - -/* Version number of package */ -#undef VERSION - -/* Number of bits in a file offset, on hosts where this is settable. */ -#undef _FILE_OFFSET_BITS - -/* Define to 1 to make fseeko visible on some hosts (e.g. glibc 2.2). */ -#undef _LARGEFILE_SOURCE - -/* Define for large files, on AIX-style hosts. */ -#undef _LARGE_FILES - -/* Define to empty if `const' does not conform to ANSI C. */ -#undef const - -/* Define to `__inline__' or `__inline' if that's what the C compiler - calls it, or to nothing if 'inline' is not supported under any name. */ -#ifndef __cplusplus -#undef inline -#endif - -/* Define to `int' if does not define. */ -#undef mode_t - -/* Define to `long int' if does not define. */ -#undef off_t - -/* Define to `unsigned int' if does not define. */ -#undef size_t From 7f0c2a7af089ccc3805c3cb197a3c5906845ed70 Mon Sep 17 00:00:00 2001 From: John Lindgren Date: Mon, 25 Feb 2013 16:16:46 -0500 Subject: [PATCH 04/14] Remove trailing whitespace in configure.ac. --- configure.ac | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/configure.ac b/configure.ac index 86d7692..16dbf50 100644 --- a/configure.ac +++ b/configure.ac @@ -8,7 +8,7 @@ AC_INIT([AFFLIB],[3.7.1],[bugs@afflib.org]) AM_INIT_AUTOMAKE AM_MAINTAINER_MODE -AC_CONFIG_FILES([Makefile lib/Makefile tools/Makefile lzma443/Makefile +AC_CONFIG_FILES([Makefile lib/Makefile tools/Makefile lzma443/Makefile tests/Makefile doc/Makefile pyaff/Makefile man/Makefile lib/version.h]) AC_CONFIG_FILES([afflib.spec]) AC_CONFIG_HEADERS([affconfig.h]) @@ -43,13 +43,13 @@ if test x"${cross_compiling}" = "xno" ; then AC_MSG_NOTICE([ *** ADDING ${spfx}/lib to LDFLAGS *** ]) fi done - AC_MSG_NOTICE([ CPPFLAGS = ${CPPFLAGS} ]) - AC_MSG_NOTICE([ LDFLAGS = ${LDFLAGS} ]) + AC_MSG_NOTICE([ CPPFLAGS = ${CPPFLAGS} ]) + AC_MSG_NOTICE([ LDFLAGS = ${LDFLAGS} ]) else AC_MSG_NOTICE([Cross Compiling --- will not update CPPFALGS or LDFLAGS with /usr/local, /opt/local or /sw]) LIBS="$LIBS -lws2_32 -lgdi32" fi - + if test -r /bin/uname.exe ; then if test `uname -o` == "Msys" ; then AC_MSG_NOTICE([Compiling with Msys. Setting flags appropriately.]) @@ -75,7 +75,7 @@ AC_C_INLINE AC_TYPE_MODE_T AC_TYPE_OFF_T AC_TYPE_SIZE_T -AC_STRUCT_TM +AC_STRUCT_TM AC_STRUCT_ST_BLOCKS AC_STRUCT_ST_RDEV @@ -166,7 +166,7 @@ AC_SUBST(S3_BIN) AM_CONDITIONAL([MAYBE_S3],[test "x${enable_s3}" = "xyes"]) # used by automake ################################################################ -### QEMU +### QEMU ################################################################ AC_ARG_ENABLE([qemu]) @@ -191,9 +191,9 @@ AM_CONDITIONAL([MAYBE_QEMU],[test "x${enable_qemu}" = "xyes"]) # used by automa ################################################################ -### PyAFF +### PyAFF ################################################################ -# python checks +# python checks # (requires autoconf 1.5+ and the macros in acinclude.m4) AC_ARG_ENABLE([python], AC_HELP_STRING([--enable-python=no], [Build python bindings (pyaff)]), @@ -267,7 +267,7 @@ if test x"${AFF_NOOPT}" != "x" ; then fi if test "${with_noopt}" = "yes" ; then - CFLAGS=`echo "$CFLAGS" | sed s/-O[[0-9]]//` # note the double quoting! + CFLAGS=`echo "$CFLAGS" | sed s/-O[[0-9]]//` # note the double quoting! CXXFLAGS=`echo "$CXXFLAGS" | sed s/-O[[0-9]]//` fi From e8bf86a0ec644999d6e44350b789fe5b2742b9e8 Mon Sep 17 00:00:00 2001 From: John Lindgren Date: Mon, 25 Feb 2013 16:23:10 -0500 Subject: [PATCH 05/14] Define USE_LZMA in affconfig.h rather than hard-coding it in afflib.h. --- configure.ac | 5 +++++ lib/afflib.h | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 16dbf50..a838f85 100644 --- a/configure.ac +++ b/configure.ac @@ -135,6 +135,11 @@ AC_CHECK_HEADER([expat.h]) AC_CHECK_LIB([expat],[XML_ParserCreate],,[have_expat="no ";AC_MSG_WARN([expat not found; S3 and Digital Signatures not enabled])]) +################################################################ +## LZMA (always enabled) +AC_DEFINE(USE_LZMA, 1, [Enable LZMA compression]) + + ################################################################ ## Amazon S3 ## S3 requires curl and expat; otherwise we don't need them diff --git a/lib/afflib.h b/lib/afflib.h index 5a97f22..d1ff54a 100644 --- a/lib/afflib.h +++ b/lib/afflib.h @@ -123,7 +123,6 @@ typedef __int64 int64_t; /* If our types still aren't defined, give some kind of error */ -#define USE_LZMA struct affcallback_info; struct aff_pagebuf { int64_t pagenum; // -1 means no page loaded From ec7fa742093e71adf5c50a178e8c4af35b7e3ba1 Mon Sep 17 00:00:00 2001 From: John Lindgren Date: Mon, 25 Feb 2013 16:36:17 -0500 Subject: [PATCH 06/14] Fix build warnings/errors. --- lib/afflib.cpp | 9 +++------ lib/afflib_stream.cpp | 3 --- lib/crypto.cpp | 2 +- lib/vnode_aff.cpp | 2 +- lib/vnode_raw.cpp | 2 +- lib/vnode_split_raw.cpp | 2 +- 6 files changed, 7 insertions(+), 13 deletions(-) diff --git a/lib/afflib.cpp b/lib/afflib.cpp index c607a8b..ea4c22d 100644 --- a/lib/afflib.cpp +++ b/lib/afflib.cpp @@ -229,7 +229,7 @@ AFFILE *af_open_with(const char *url,int flags,int mode, struct af_vnode *v) AFFILE *af = (AFFILE *)calloc(sizeof(AFFILE),1); af_crypto_allocate(af); #ifdef HAVE_PTHREAD - pthread_rwlock_init(&af->rwlock); + pthread_rwlock_init(&af->rwlock, 0); AF_WRLOCK(af); #endif af->v = v; @@ -517,10 +517,7 @@ int af_eof(AFFILE *af) if(af_vstat(af,&vni)) return -1; // this is bad; we need vstat... if(vni.use_eof) return vni.at_eof; // if implementation wants to use it, use it - if(af->pos<0){ // pos shouldn't be <0 - errno = EINVAL; - return -1; // this is bad - } + int ret = (int64_t)af->pos >= (int64_t)vni.imagesize; AF_UNLOCK(af); return ret; @@ -889,7 +886,7 @@ int af_update_seg(AFFILE *af, const char *segname, #ifdef HAVE_OPENSSL_BIO_H /* Requires no locking */ -int af_update_seg_frombio(AFFILE *af,const char *segname,uint32_t arg,BIO *bio) +int af_update_seg_frombio(AFFILE *af,const char *segname,uint32_t /*arg*/,BIO *bio) { /* Get the buffer to write out */ u_char *buf=0; diff --git a/lib/afflib_stream.cpp b/lib/afflib_stream.cpp index 7f5143e..8f14fb9 100644 --- a/lib/afflib_stream.cpp +++ b/lib/afflib_stream.cpp @@ -84,7 +84,6 @@ extern "C" ASIZE af_read(AFFILE *af,unsigned char *buf,ASIZE count) /* performance improvement: use af->image_size if it is set */ uint64_t offset = af->pos; /* where to start */ - if(af->image_size<0) {total=-1;goto done;} // error if(af->image_size==0) {goto done;} // no data in file if(af->pos > af->image_size) {goto done;} // seeked beyond end of file if(af->pos+count > af->image_size) count = af->image_size - af->pos; // only this much left in file @@ -138,7 +137,6 @@ extern "C" ASIZE af_read(AFFILE *af,unsigned char *buf,ASIZE count) if(bytes_to_read > page_left) bytes_to_read = page_left; if(bytes_to_read > af->image_size - offset) bytes_to_read = (u_int)(af->image_size - offset); - assert(bytes_to_read >= 0); // if(bytes_to_read==0) break; // that's all we could get /* Copy out the bytes for the user */ @@ -253,7 +251,6 @@ int af_write(AFFILE *af,unsigned char *buf,size_t count) if(bytes_to_write > seg_left) bytes_to_write = seg_left; - assert(bytes_to_write >= 0); // if(bytes_to_write==0) break; // that's all we could get /* Copy out the bytes for the user */ diff --git a/lib/crypto.cpp b/lib/crypto.cpp index ee2ad49..446edf5 100644 --- a/lib/crypto.cpp +++ b/lib/crypto.cpp @@ -633,7 +633,7 @@ int af_hash_verify_seg2(AFFILE *af,const char *segname,u_char *sigbuf_,size_t si } /* Verify a segment against a particular signature and public key */ -int af_sig_verify_seg2(AFFILE *af,const char *segname,EVP_PKEY *pubkey,u_char *sigbuf,size_t sigbuf_len,int sigmode) +int af_sig_verify_seg2(AFFILE *af,const char *segname,EVP_PKEY */*pubkey*/,u_char *sigbuf,size_t sigbuf_len,int sigmode) { const EVP_MD *sha256 = EVP_get_digestbyname("SHA256"); if(!sha256){ diff --git a/lib/vnode_aff.cpp b/lib/vnode_aff.cpp index 499ca4c..23db5b8 100644 --- a/lib/vnode_aff.cpp +++ b/lib/vnode_aff.cpp @@ -72,7 +72,7 @@ static int aff_write_ignore(AFFILE *af,size_t bytes) char next[AF_MAX_NAME_LEN]; size_t segsize2=0; int count=0; - while(af_probe_next_seg(af,next,sizeof(next),0,0,&segsize2,1)==0 && next[0]==0 && segsize2>=0){ + while(af_probe_next_seg(af,next,sizeof(next),0,0,&segsize2,1)==0 && next[0]==0){ count++; if(count>10) break; // something is wrong; just get out. //printf("*** next %d segment at %qd len=%d will be deleted\n",count,ftello(af->aseg),segsize2); diff --git a/lib/vnode_raw.cpp b/lib/vnode_raw.cpp index 7c7badb..8bf36ed 100644 --- a/lib/vnode_raw.cpp +++ b/lib/vnode_raw.cpp @@ -202,7 +202,7 @@ static int raw_get_seg(AFFILE *af,const char *name, int raw_update_seg(AFFILE *af, const char *name, - uint32_t arg,const u_char *value,uint32_t vallen) + uint32_t /*arg*/,const u_char *value,uint32_t vallen) { struct raw_private *rp = RAW_PRIVATE(af); diff --git a/lib/vnode_split_raw.cpp b/lib/vnode_split_raw.cpp index df9c208..46c37ae 100644 --- a/lib/vnode_split_raw.cpp +++ b/lib/vnode_split_raw.cpp @@ -603,7 +603,7 @@ static int split_raw_rewind_seg(AFFILE *af) } static int split_raw_update_seg(AFFILE *af, const char *name, - uint32_t arg,const u_char *value,uint32_t vallen) + uint32_t /*arg*/,const u_char *value,uint32_t vallen) { int64_t page_num = af_segname_page_number(name); From acd7bd3d58f79165490c8595b7f4415c9a598878 Mon Sep 17 00:00:00 2001 From: John Lindgren Date: Mon, 25 Feb 2013 17:23:42 -0500 Subject: [PATCH 07/14] Fix opening and creating raw image files. Raw files were always opened read-write due to a "&&" instead of a "&". Respect file flags such as O_SYNC or O_CLOEXEC by using fdopen(). raw_identify_file() now returns true if and only if the extension is ".raw". --- lib/vnode_raw.cpp | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/lib/vnode_raw.cpp b/lib/vnode_raw.cpp index 8bf36ed..4c69987 100644 --- a/lib/vnode_raw.cpp +++ b/lib/vnode_raw.cpp @@ -20,10 +20,9 @@ struct raw_private { #define RAW_PRIVATE(af) ((struct raw_private *)(af->vnodeprivate)) /* Return 1 if a file is a raw file... */ -static int raw_identify_file(const char *filename,int exists) +static int raw_identify_file(const char *filename,int /*exists*/) { - if(exists && access(filename,R_OK)!=0) return 0; // needs to exist and it doesn't - return access(filename,R_OK)==0; // if we can read it, it's raw... + return af_ext_is(filename, "raw"); } @@ -52,16 +51,21 @@ static int64_t raw_filesize(AFFILE *af) static int raw_open(AFFILE *af) { /* Raw is the passthrough system. - * Right now, it is read only... */ - const char *mode = "rb"; - if(af->openflags && (O_RDWR | O_WRONLY)) mode = "r+b"; + int fd = open(af->fname, af->openflags | O_BINARY, af->openmode); + if(fd < 0) + return -1; + + FILE *file = fdopen(fd, (af->openflags & (O_RDWR | O_WRONLY)) ? "r+b" : "rb"); + if(!file) + { + close(fd); + return -1; + } af->vnodeprivate = (void *)calloc(1,sizeof(struct raw_private)); struct raw_private *rp = RAW_PRIVATE(af); - - if(af->fname) rp->raw=fopen(af->fname,mode); - if(rp->raw==0) return -1; // raw open failed + rp->raw = file; af->image_size = raw_filesize(af); af->image_pagesize = RAW_PAGESIZE; af->cur_page = 0; From d996986565d39917a60969fa03839a99d604abaf Mon Sep 17 00:00:00 2001 From: John Lindgren Date: Thu, 21 Mar 2013 16:40:04 -0400 Subject: [PATCH 08/14] Make error handling and reporting more robust. Some functions were returning no error or the wrong type of error. --- lib/afflib.cpp | 8 ++-- lib/afflib_pages.cpp | 14 +++++-- lib/afflib_stream.cpp | 55 ++++++++++++------------- lib/vnode_afd.cpp | 17 ++++---- lib/vnode_aff.cpp | 93 ++++++++++++++++++++++--------------------- lib/vnode_raw.cpp | 16 ++++++-- 6 files changed, 110 insertions(+), 93 deletions(-) diff --git a/lib/afflib.cpp b/lib/afflib.cpp index ea4c22d..e89ed9f 100644 --- a/lib/afflib.cpp +++ b/lib/afflib.cpp @@ -839,10 +839,13 @@ int af_update_segf(AFFILE *af, const char *segname, } #endif int r = (*af->v->update_seg)(af,segname,arg,data,datalen); // actually update the segment - if(r==0) af->bytes_written += datalen; + if(r < 0) + { AF_UNLOCK(af); return r; } + + af->bytes_written += datalen; #ifdef HAVE_AES_ENCRYPT /* if we encrypted, make sure the unencrypted segment is deleted */ - if(r==0 && oldname) (*af->v->del_seg)(af,oldname); + if(oldname) (*af->v->del_seg)(af,oldname); if(newdata){ free(newdata); // free any allocated data newdata = 0; @@ -865,7 +868,6 @@ int af_update_segf(AFFILE *af, const char *segname, #ifdef USE_AFFSIGS const u_char *signdata = data; // remember the original data location if(AF_SEALING(af) - && (r==0) && af->crypto->sign_privkey && ((flag & AF_SIGFLAG_NOSIG)==0) && !ends_with(segname,AF_SIG256_SUFFIX)){ diff --git a/lib/afflib_pages.cpp b/lib/afflib_pages.cpp index f539ea6..70145f2 100644 --- a/lib/afflib_pages.cpp +++ b/lib/afflib_pages.cpp @@ -138,7 +138,8 @@ int af_get_page_raw(AFFILE *af,int64_t pagenum,uint32_t *arg, memset(segname,0,sizeof(segname)); sprintf(segname,AF_PAGE,pagenum); int r = af_get_seg(af,segname,arg,data,bytes); - if(r!=0){ + if(r < 0 && errno == ENOENT) + { /* Couldn't read with AF_PAGE; try AF_SEG_D. * This is legacy for the old AFF files. Perhaps we should delete it. */ @@ -176,12 +177,15 @@ int af_get_page(AFFILE *af,int64_t pagenum,unsigned char *data,size_t *bytes) * If we have been provided with a buffer, * fill buffer with the 'bad segment' flag and return. */ - if (data && (af->openmode&AF_BADBLOCK_FILL)) { + if(data && (af->openmode & AF_BADBLOCK_FILL) && errno == ENOENT) + { for(size_t i = 0;i <= af->image_pagesize - af->image_sectorsize; i+= af->image_sectorsize){ memcpy(data+i,af->badflag,af->image_sectorsize); af->bytes_memcpy += af->image_sectorsize; } + + r = 0; } return r; // segment doesn't exist } @@ -561,7 +565,11 @@ void af_cache_writethrough(AFFILE *af,int64_t pagenum,const unsigned char *buf,i struct aff_pagebuf *af_cache_alloc(AFFILE *af,int64_t pagenum) { if(af_trace) fprintf(af_trace,"af_cache_alloc(%p,%"I64d")\n",af,pagenum); - af_cache_flush(af); // make sure nothing in cache is dirty + + /* Make sure nothing in the cache is dirty */ + if(af_cache_flush(af) < 0) + return 0; + /* See if this page is already in the cache */ for(int i=0;inum_pbufs;i++){ struct aff_pagebuf *p = &af->pbcache[i]; diff --git a/lib/afflib_stream.cpp b/lib/afflib_stream.cpp index 8f14fb9..c86dd49 100644 --- a/lib/afflib_stream.cpp +++ b/lib/afflib_stream.cpp @@ -56,16 +56,26 @@ const unsigned char *af_badflag(AFFILE *af) *** Stream-level interface ****************************************************************/ - -/* Throw out the current segment */ -int af_purge(AFFILE *af) +static int af_get_pagebuf(AFFILE *af, int64_t pagenum) { - AF_WRLOCK(af); - if (af_trace) fprintf(af_trace,"af_purge(%p)\n",af); - int ret = af_cache_flush(af); // flush the cache - af->pb = 0; // no longer have a current page - AF_UNLOCK(af); - return ret; + if(!af->pb || af->pb->pagenum != pagenum) + { + af->pb = af_cache_alloc(af, pagenum); + if(!af->pb) + return -1; + } + + if(!af->pb->pagebuf_valid) + { + size_t pagebytes = af->image_pagesize; + if(af_get_page(af, pagenum, af->pb->pagebuf, &pagebytes) < 0) + return -1; + + af->pb->pagebuf_valid = 1; + af->pb->pagebuf_bytes = pagebytes; + } + + return 0; } extern "C" ASIZE af_read(AFFILE *af,unsigned char *buf,ASIZE count) @@ -97,30 +107,17 @@ extern "C" ASIZE af_read(AFFILE *af,unsigned char *buf,ASIZE count) } while(count>0){ - /* If the correct segment is not loaded, purge the segment */ int64_t new_page = offset / af->image_pagesize; - if(af->pb==0 || new_page != af->pb->pagenum){ - af_cache_flush(af); - af->pb = 0; + if(af_get_pagebuf(af, new_page) < 0) + { + /* if nothing was read yet, return 0 for EOF or -1 for read error */ + /* ENOENT (page not found) means EOF, other errno means read error */ + if(!total && errno != ENOENT) + total = -1; + break; } - /* If no segment is loaded in cache, load the current segment */ - if(af->pb==0){ - int64_t pagenum = offset / af->image_pagesize; - af->pb = af_cache_alloc(af,pagenum); - if(af->pb->pagebuf_valid==0){ - /* page buffer isn't valid; need to get it */ - af->pb->pagebuf_bytes = af->image_pagesize; // we can hold this much - if(af_get_page(af,af->pb->pagenum,af->pb->pagebuf, &af->pb->pagebuf_bytes)){ - /* Page doesn't exist; fill with NULs */ - memset(af->pb->pagebuf,0,af->pb->pagebuf_bytes); - /* TK: Should fill with BADBLOCK here if desired */ - /* previously had BREAK here */ - } - af->pb->pagebuf_valid = 1; // contents of the page buffer are valid - } - } // Compute how many bytes can be copied... // where we were reading from u_int page_offset = (u_int)(offset - af->pb->pagenum * af->image_pagesize); diff --git a/lib/vnode_afd.cpp b/lib/vnode_afd.cpp index 011386d..590bd92 100644 --- a/lib/vnode_afd.cpp +++ b/lib/vnode_afd.cpp @@ -191,7 +191,6 @@ static AFFILE *afd_file_with_seg(AFFILE *af,const char *name) return ap->afs[i]; } } - errno = ENOTDIR; // get ready for error return return 0; } @@ -452,10 +451,10 @@ static int afd_get_seg(AFFILE *af,const char *name,uint32_t *arg,unsigned char * size_t *datalen) { AFFILE *af2 = afd_file_with_seg(af,name); - if(af2){ - return af_get_seg(af2,name,arg,data,datalen); // use this one - } - return -1; // not found + if(!af2) + { errno = ENOENT; return -1; } + + return af_get_seg(af2,name,arg,data,datalen); // use this one } @@ -532,10 +531,10 @@ static int afd_update_seg(AFFILE *af, const char *name, int afd_del_seg(AFFILE *af,const char *segname) { AFFILE *af2 = afd_file_with_seg(af,segname); - if(af2){ - return af_del_seg(af2,segname); - } - return -1; // not found + if(!af2) + { errno = ENOENT; return -1; } + + return af_del_seg(af2,segname); } diff --git a/lib/vnode_aff.cpp b/lib/vnode_aff.cpp index 23db5b8..7122fff 100644 --- a/lib/vnode_aff.cpp +++ b/lib/vnode_aff.cpp @@ -218,7 +218,11 @@ static int aff_get_seg(AFFILE *af,const char *name, * we will rewind to the beginning and go to the end. */ struct aff_toc_mem *adm = aff_toc(af,name); - if(!adm) return -1; + if(!adm) + { errno = ENOENT; return -1; } + + if(!arg && !data && !datalen) + return 0; // caller only wants to know whether the segment exists fseeko(af->aseg,adm->offset,SEEK_SET); int ret = aff_get_next_seg(af,next,sizeof(next),arg,data,datalen); @@ -572,78 +576,77 @@ static int aff_identify_file(const char *filename,int exists) return 0; } +static int err_close(int fd) +{ + int tmp = errno; + close(fd); + errno = tmp; + return -1; +} +static int err_close(AFFILE *af) +{ + int tmp = errno; + fclose(af->aseg); + af->aseg = 0; + errno = tmp; + return -1; +} static int aff_open(AFFILE *af) { - if(af_is_filestream(af->fname)==0) return -1; // not a file stream + // must be a file stream with read access + if(!af_is_filestream(af->fname) || (af->openflags & O_ACCMODE) == O_WRONLY) + { errno = EINVAL; return -1; } + + bool canWrite = (af->openflags & O_ACCMODE) == O_RDWR; /* Open the raw file */ int fd = open(af->fname,af->openflags | O_BINARY,af->openmode); - if(fd<0){ // couldn't open + if(fd < 0) return -1; - } /* Lock the file if writing */ #ifdef HAVE_FLOCK - if(af->openflags & O_RDWR){ - int lockmode = LOCK_SH; // default - if((af->openflags & O_ACCMODE)==O_RDWR) lockmode = LOCK_EX; // there can be only one - if(flock(fd,lockmode)){ - warn("Cannot exclusively lock %s:",af->fname); - } - } + if(flock(fd, canWrite ? LOCK_EX : LOCK_SH) < 0) + return err_close(fd); #endif /* Set defaults */ + af->compression_type = AF_COMPRESSION_ALG_ZLIB; + af->compression_level = Z_DEFAULT_COMPRESSION; - af->compression_type = AF_COMPRESSION_ALG_ZLIB; - af->compression_level = Z_DEFAULT_COMPRESSION; - - /* Open the FILE for the AFFILE */ - char strflag[8]; - strcpy(strflag,"rb"); // we have to be able to read - if(af->openflags & O_RDWR) strcpy(strflag,"w+b"); - - af->aseg = fdopen(fd,strflag); - if(!af->aseg){ - (*af->error_reporter)("fdopen(%d,%s)",fd,strflag); - return -1; - } + /* Open the FILE for the AFFILE */ + af->aseg = fdopen(fd, canWrite ? "w+b" : "rb"); + if(!af->aseg) + return err_close(fd); /* Get file size */ struct stat sb; - if(fstat(fd,&sb)){ - (*af->error_reporter)("aff_open: fstat(%s): ",af->fname); // this should not happen - return -1; - } + if(fstat(fd, &sb) < 0) + return err_close(af); /* If file is empty, then put out an AFF header, badflag, and AFF version */ - if(sb.st_size==0){ + if(canWrite && sb.st_size == 0) return aff_create(af); - } /* We are opening an existing file. Verify once more than it is an AFF file * and skip past the header... */ - - char buf[8]; - if(fread(buf,sizeof(buf),1,af->aseg)!=1){ - /* Hm. End of file. That shouldn't happen here. */ - (*af->error_reporter)("aff_open: couldn't read AFF header on existing file?"); - return -1; // should not happen - } - - if(strcmp(buf,AF_HEADER)!=0){ - buf[7] = 0; - (*af->error_reporter)("aff_open: %s is not an AFF file (header=%s)\n", - af->fname,buf); - return -1; + char buf[8]; errno = 0; + size_t itemsRead = fread(buf, sizeof(buf), 1, af->aseg); + if(itemsRead != 1 || strcmp(buf, AF_HEADER)) + { + if(!errno) + errno = EIO; + return err_close(af); } /* File has been validated */ - if(aff_toc_build(af)) return -1; // build the TOC - return 0; // everything must be okay. + if(aff_toc_build(af) < 0) + return err_close(af); + + return 0; } diff --git a/lib/vnode_raw.cpp b/lib/vnode_raw.cpp index 4c69987..83c91b5 100644 --- a/lib/vnode_raw.cpp +++ b/lib/vnode_raw.cpp @@ -289,15 +289,23 @@ static int raw_get_next_seg(AFFILE *af,char *segname,size_t segname_len,uint32_t static int raw_read(AFFILE *af, unsigned char *buf, uint64_t pos,size_t count) { struct raw_private *rp = RAW_PRIVATE(af); - fseeko(rp->raw,pos,SEEK_SET); - return fread(buf,1,count,rp->raw); + if(fseeko(rp->raw, pos, SEEK_SET) < 0) + return -1; + + errno = 0; + count = fread(buf, 1, count, rp->raw); + return (!count && errno) ? -1 : count; } static int raw_write(AFFILE *af, unsigned char *buf, uint64_t pos,size_t count) { struct raw_private *rp = RAW_PRIVATE(af); - if(fseeko(rp->raw,pos,SEEK_SET)<0) return -1; - return fwrite(buf,1,count,rp->raw); + if(fseeko(rp->raw, pos, SEEK_SET) < 0) + return -1; + + errno = 0; + count = fwrite(buf, 1, count, rp->raw); + return (!count && errno) ? -1 : count; } From a0b42ef79e3981102a58b42622d32e4c2c589a23 Mon Sep 17 00:00:00 2001 From: John Lindgren Date: Wed, 17 Apr 2013 08:24:08 -0400 Subject: [PATCH 09/14] Fix file descriptor leak. --- lib/vnode_afd.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/vnode_afd.cpp b/lib/vnode_afd.cpp index 590bd92..e5afbac 100644 --- a/lib/vnode_afd.cpp +++ b/lib/vnode_afd.cpp @@ -389,6 +389,7 @@ static int afd_open(AFFILE *af) strlcat(path,"/",sizeof(path)); strlcat(path,dp->d_name,sizeof(path)); if(afd_add_file(af,path)){ + closedir(dirp); return -1; } } From 4d396e5e26b4b457ef7003f287f6ab5afe2cfb1e Mon Sep 17 00:00:00 2001 From: John Lindgren Date: Wed, 17 Apr 2013 08:24:35 -0400 Subject: [PATCH 10/14] af_write()'s whole-page optimization now works for multiple pages. --- lib/afflib_stream.cpp | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/lib/afflib_stream.cpp b/lib/afflib_stream.cpp index c86dd49..9db1604 100644 --- a/lib/afflib_stream.cpp +++ b/lib/afflib_stream.cpp @@ -206,21 +206,30 @@ int af_write(AFFILE *af,unsigned char *buf,size_t count) * and if an entire page is being written, * just write it out and update the pointers, then return. */ - if(af->pb==0 && af->image_pagesize==(unsigned)count && write_page_offset == 0){ - // copy into cache if we have this page anywhere in our cache - af_cache_writethrough(af,write_page,buf,count); - int ret = af_update_page(af,write_page,buf,count); - if(ret==0){ // no error - af->pos += count; - if(af->pos > af->image_size) af->image_size = af->pos; - AF_UNLOCK(af); - return count; + if(!af->pb && !write_page_offset && !(count % af->image_pagesize)) + { + for(size_t written = 0; written < count; written += af->image_pagesize) + { + // copy into cache if we have this page anywhere in our cache + af_cache_writethrough(af, write_page, buf + written, af->image_pagesize); + + if(af_update_page(af, write_page, buf + written, af->image_pagesize) < 0) + { + AF_UNLOCK(af); + return -1; + } + + af->pos += af->image_pagesize; + if(af->pos > af->image_size) + af->image_size = af->pos; + + write_page++; } + AF_UNLOCK(af); - return -1; // error + return count; } - /* Can't use high-speed optimization; write through the cache */ int total = 0; while(count>0){ From 62ed1cb8c29eba12b37190ebbb84ab3cff4f7816 Mon Sep 17 00:00:00 2001 From: John Lindgren Date: Wed, 1 May 2013 16:02:17 -0400 Subject: [PATCH 11/14] Set errno to EIO if af_get_next_seg() fails in aff_toc_build(). --- lib/aff_toc.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/aff_toc.cpp b/lib/aff_toc.cpp index 76006e6..9a55d9c 100644 --- a/lib/aff_toc.cpp +++ b/lib/aff_toc.cpp @@ -95,6 +95,7 @@ int aff_toc_build(AFFILE *af) // build the dir if we couldn't find it uint64_t pos = ftello(af->aseg); size_t datalen=0; + errno = 0; int r = af_get_next_seg(af,segname,segname_len,0,0,&datalen); switch(r){ case AF_ERROR_NO_ERROR: @@ -105,7 +106,8 @@ int aff_toc_build(AFFILE *af) // build the dir if we couldn't find it case AF_ERROR_EOF: return 0; // end of file; no errors default: /* unknown error */ - fseeko(af->aseg,pos,SEEK_SET); // go back + if(!errno) + errno = EIO; return r; // send up the error code } } From 6a52df7a26f8732b85ff1178749f6b0ee807833b Mon Sep 17 00:00:00 2001 From: John Lindgren Date: Wed, 1 May 2013 16:15:32 -0400 Subject: [PATCH 12/14] Rewrite aff_update_seg() so it doesn't parse through the whole AFF file. --- lib/aff_toc.cpp | 51 ++++++++++++++++++++ lib/afflib_i.h | 3 ++ lib/vnode_aff.cpp | 116 ++++++++++++++-------------------------------- 3 files changed, 89 insertions(+), 81 deletions(-) diff --git a/lib/aff_toc.cpp b/lib/aff_toc.cpp index 9a55d9c..1f699d5 100644 --- a/lib/aff_toc.cpp +++ b/lib/aff_toc.cpp @@ -47,6 +47,10 @@ static int toc_sort(const void *a_,const void *b_) static int aff_toc_append(AFFILE *af,const char *segname,uint64_t offset,uint64_t datalen) { + /* don't add AF_IGNORE segments to TOC */ + if(!segname[0]) + return 0; + af->toc = (aff_toc_mem *)realloc(af->toc,sizeof(*af->toc)*(af->toc_count+1)); if(af->toc==0){ (*af->error_reporter)("realloc() failed in aff_toc_append. toc_count=%d\n",af->toc_count); @@ -64,6 +68,10 @@ static int aff_toc_append(AFFILE *af,const char *segname,uint64_t offset,uint64_ */ void aff_toc_update(AFFILE *af,const char *segname,uint64_t offset,uint64_t datalen) { + /* don't add AF_IGNORE segments to TOC */ + if(!segname[0]) + return; + for(int i=0;itoc_count;i++){ if(af->toc[i].name==0 || strcmp(af->toc[i].name,segname)==0){ if(af->toc[i].name==0){ // if name was empty, copy it over @@ -139,3 +147,46 @@ int aff_toc_del(AFFILE *af,const char *segname) return -1; } +/* Finds the next segment starting at or beyond a given byte offset. */ +/* Returns the TOC entry if found; null if not found. */ +struct aff_toc_mem *aff_toc_next_seg(AFFILE *af, uint64_t offset) +{ + struct aff_toc_mem *next_seg = 0; + struct aff_toc_mem *end = af->toc + af->toc_count; + + for(struct aff_toc_mem *seg = af->toc; seg != end; seg++) + { + if(!seg->name) + continue; + + if(seg->offset >= offset && (!next_seg || seg->offset < next_seg->offset)) + next_seg = seg; + } + + return next_seg; +} + +/* Finds the smallest unused block of at least min_size bytes. */ +/* Returns 0 if found, -1 if not found. */ +int aff_toc_find_hole(AFFILE *af, uint64_t min_size, uint64_t *offset, uint64_t *size) +{ + int ret = -1; + uint64_t search_offset = 0; + struct aff_toc_mem *seg; + + while((seg = aff_toc_next_seg(af, search_offset))) + { + uint64_t test_size = seg->offset - search_offset; + + if(test_size >= min_size && (ret < 0 || test_size < *size)) + { + *offset = search_offset; + *size = test_size; + ret = 0; + } + + search_offset = seg->offset + seg->segment_len; + } + + return ret; +} diff --git a/lib/afflib_i.h b/lib/afflib_i.h index de58aa2..6fa02b6 100644 --- a/lib/afflib_i.h +++ b/lib/afflib_i.h @@ -687,6 +687,9 @@ struct aff_toc_mem *aff_toc(AFFILE *af,const char *segname); int aff_toc_del(AFFILE *af,const char *segname); void aff_toc_update(AFFILE *af,const char *segname,uint64_t offset,uint64_t datalen); +struct aff_toc_mem *aff_toc_next_seg(AFFILE *af, uint64_t offset); +int aff_toc_find_hole(AFFILE *af, uint64_t min_size, uint64_t *offset, uint64_t *size); + /* lzma_glue.cpp: * For the LZMA compression engine */ diff --git a/lib/vnode_aff.cpp b/lib/vnode_aff.cpp index 7122fff..5e38d30 100644 --- a/lib/vnode_aff.cpp +++ b/lib/vnode_aff.cpp @@ -360,102 +360,56 @@ int af_truncate_blank(AFFILE *af) static int aff_update_seg(AFFILE *af, const char *name, uint32_t arg,const u_char *value,uint32_t vallen) { - char next_segment_name[AF_MAX_NAME_LEN]; - size_t next_segsize = 0; - size_t next_datasize = 0; - - /* if we are updating with a different size, - * remember the location and size of the AF_IGNORE segment that - * has the smallest size that is >= strlen(name)+vallen - */ size_t size_needed = vallen+aff_segment_overhead(name); - size_t size_closest = 0; - uint64_t loc_closest = 0; struct aff_toc_mem *adm = aff_toc(af,name); if(af_trace) fprintf(af_trace,"aff_update_seg(name=%s,arg=%"PRIu32",vallen=%u)\n",name,arg,vallen); + if(adm) + { + /* segment already exists */ + if(fseeko(af->aseg, adm->offset, SEEK_SET) < 0) + return -1; - if(adm){ - /* Segment is in the TOC; seek to it */ - fseeko(af->aseg,adm->offset,SEEK_SET); - } - else { - /* Otherwise, go to the beginning of the file and try to find a suitable hole - * TK: This could be made significantly faster by just scanning the TOC for a hole. - */ - af_rewind_seg(af); // start at the beginning - } - - while(af_probe_next_seg(af,next_segment_name,sizeof(next_segment_name),0,&next_datasize,&next_segsize,1)==0){ - /* Remember this information */ - uint64_t next_segment_loc = ftello(af->aseg); -#ifdef DEBUG2 - fprintf(stderr," next_segment_name=%s next_datasize=%d next_segsize=%d next_segment_loc=%qd\n", - next_segment_name, next_datasize, next_segsize,next_segment_loc); -#endif - if(strcmp(next_segment_name,name)==0){ // found the segment - if(next_datasize == vallen){ // Does it exactly fit? - int r = aff_write_seg(af,name,arg,value,vallen); // Yes, just write in place! - return r; - } - - //printf("** Segment '%s' doesn't fit at %qd; invalidating.\n",name,ftello(af->aseg)); - aff_write_ignore(af,next_datasize+strlen(name)); + /* if its size matches, just overwrite it */ + if(adm->segment_len == size_needed) + return aff_write_seg(af, name, arg, value, vallen); - /* If we are in random mode, jump back to the beginning of the file. - * This does a good job filling in the holes. - */ - if(af->random_access){ - af_rewind_seg(af); - continue; - } + /* otherwise, invalidate it */ + if(aff_write_ignore(af, adm->segment_len - aff_segment_overhead(0)) < 0) + return -1; - /* Otherwise just go to the end. Experience has shown that sequential access - * tends not to generate holes. - */ - fseeko(af->aseg,(uint64_t)0,SEEK_END); // go to the end of the file - break; // and exit this loop + aff_toc_del(af, name); + } - } + /* search through TOC for a hole */ + /* need space for a new AF_IGNORE segment also */ + uint64_t hole_offset, hole_size; + if(aff_toc_find_hole(af, size_needed + aff_segment_overhead(0), &hole_offset, &hole_size) == 0) + { + /* found a large enough hole */ + if(fseeko(af->aseg, hole_offset, SEEK_SET) < 0) + return -1; - if((next_segment_name[0]==0) && (next_datasize>=size_needed)){ - //printf(" >> %d byte blank\n",next_datasize); - } + /* write segment */ + if(aff_write_seg(af, name, arg, value, vallen) < 0) + return -1; - /* If this is an AF_IGNORE, see if it is a close match */ - if((next_segment_name[0]==AF_IGNORE[0]) && - (next_datasize>=size_needed) && - ((next_datasize=1024 && size_needed>=1024))){ - size_closest = next_datasize; - loc_closest = next_segment_loc; - } - fseeko(af->aseg,next_segsize,SEEK_CUR); // skip this segment + /* fill in any remaining space with AF_IGNORE */ + return aff_write_ignore(af, hole_size - size_needed - aff_segment_overhead(0)); } - /* Ready to write */ - if(size_closest>0){ - /* Yes. Put it here and put a new AF_IGNORE in the space left-over - * TODO: If the following space is also an AF_IGNORE, then combine the two. - */ - //printf("*** Squeezing it in at %qd. name=%s. vallen=%d size_closest=%d\n",loc_closest,name,vallen,size_closest); + /* no holes; seek to end of file and truncate any trailing AF_IGNORE */ + if(fseeko(af->aseg, 0, SEEK_END) < 0) + return -1; - fseeko(af->aseg,loc_closest,SEEK_SET); // move to the location - aff_write_seg(af,name,arg,value,vallen); // write the new segment + while(af_truncate_blank(af) == 0) {} - size_t newsize = size_closest - vallen - aff_segment_overhead(0) - strlen(name); - aff_write_ignore(af,newsize); // write the smaller ignore - return 0; - } - /* If we reach here we are positioned at the end of the file. */ - /* If the last segment is an ignore, truncate the file before writing */ - while(af_truncate_blank(af)==0){ - /* Keep truncating until there is nothing left */ - } - //printf("*** appending '%s' bytes=%d to the end\n",name,vallen); - fseeko(af->aseg,0L,SEEK_END); // move back to the end of the file - return aff_write_seg(af,name,arg,value,vallen); // just write at the end + /* write segment at end of file */ + if(fseeko(af->aseg, 0, SEEK_END) < 0) + return -1; + + return aff_write_seg(af, name, arg, value, vallen); } From 726984832dc4d261294d30818704a9dd5f0aa780 Mon Sep 17 00:00:00 2001 From: John Lindgren Date: Wed, 1 May 2013 16:16:34 -0400 Subject: [PATCH 13/14] Optimize is_buffer_zero(). --- lib/afflib_pages.cpp | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/lib/afflib_pages.cpp b/lib/afflib_pages.cpp index 70145f2..e2c470f 100644 --- a/lib/afflib_pages.cpp +++ b/lib/afflib_pages.cpp @@ -314,9 +314,35 @@ int af_get_page(AFFILE *af,int64_t pagenum,unsigned char *data,size_t *bytes) static bool is_buffer_zero(unsigned char *buf,int buflen) { - for(int i=0;i= (int)sizeof(long)) + { + // align to word boundary + buflen -= (intptr_t)buf % sizeof(long); + + while((intptr_t)buf % sizeof(long)) + { + if(*buf++) + return false; + } + + // read in words + long *ptr = (long*)buf; + buf += buflen - buflen % sizeof(long); + buflen %= sizeof(long); + + while(ptr < (long*)buf) + { + if(*ptr++) + return false; + } } + + while(buflen--) + { + if(*buf++) + return false; + } + return true; } From 168831be71165368752d2a8819cf9c2f54d5f062 Mon Sep 17 00:00:00 2001 From: John Lindgren Date: Wed, 1 May 2013 16:17:25 -0400 Subject: [PATCH 14/14] Recover compression settings from existing AFF files as best we can. --- lib/aff_toc.cpp | 37 ++++++++++++++++++++++++++++++++++++- lib/vnode_aff.cpp | 4 ---- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/lib/aff_toc.cpp b/lib/aff_toc.cpp index 1f699d5..67b9408 100644 --- a/lib/aff_toc.cpp +++ b/lib/aff_toc.cpp @@ -85,6 +85,35 @@ void aff_toc_update(AFFILE *af,const char *segname,uint64_t offset,uint64_t data aff_toc_append(af,segname,offset,datalen); /* Need to append it to the directory */ } +/* exact compression settings are not saved in the AFF file, so make a best guess */ +static void guess_compression(AFFILE *af, const char *segname, uint32_t flags) +{ + /* ignore non-data segments */ + if(af_segname_page_number(segname) < 0) + return; + + if(flags & AF_PAGE_COMPRESSED) + { + switch(flags & AF_PAGE_COMP_ALG_MASK) + { + case AF_PAGE_COMP_ALG_ZLIB: + af->compression_type = AF_COMPRESSION_ALG_ZLIB; + af->compression_level = (flags & AF_PAGE_COMP_MAX) ? AF_COMPRESSION_MAX : AF_COMPRESSION_DEFAULT; + break; + + case AF_PAGE_COMP_ALG_LZMA: + af->compression_type = AF_COMPRESSION_ALG_LZMA; + af->compression_level = (flags & AF_PAGE_COMP_MAX) ? AF_COMPRESSION_MAX : AF_COMPRESSION_DEFAULT; + break; + } + } + else + { + af->compression_type = AF_COMPRESSION_ALG_NONE; + af->compression_level = 0; + } +} + /* * aff_toc_build: * Build the directory by reading the existing file. @@ -95,18 +124,24 @@ int aff_toc_build(AFFILE *af) // build the dir if we couldn't find it aff_toc_free(af); // clear the old one af_rewind_seg(af); // start at the beginning + /* default compression settings */ + af->compression_type = AF_COMPRESSION_ALG_ZLIB; + af->compression_level = AF_COMPRESSION_DEFAULT; + // note: was malloc(0), but that causes problems under Borland af->toc = (aff_toc_mem *)malloc(sizeof(aff_toc_mem)); while(1){ char segname[AF_MAX_NAME_LEN]; size_t segname_len=sizeof(segname); uint64_t pos = ftello(af->aseg); + uint32_t flags = 0; size_t datalen=0; errno = 0; - int r = af_get_next_seg(af,segname,segname_len,0,0,&datalen); + int r = af_get_next_seg(af, segname, segname_len, &flags, 0, &datalen); switch(r){ case AF_ERROR_NO_ERROR: + guess_compression(af, segname, flags); if(aff_toc_append(af,segname,pos,datalen)){ return -1; // malloc error? } diff --git a/lib/vnode_aff.cpp b/lib/vnode_aff.cpp index 5e38d30..1fac55e 100644 --- a/lib/vnode_aff.cpp +++ b/lib/vnode_aff.cpp @@ -566,10 +566,6 @@ static int aff_open(AFFILE *af) return err_close(fd); #endif - /* Set defaults */ - af->compression_type = AF_COMPRESSION_ALG_ZLIB; - af->compression_level = Z_DEFAULT_COMPRESSION; - /* Open the FILE for the AFFILE */ af->aseg = fdopen(fd, canWrite ? "w+b" : "rb"); if(!af->aseg)