From d02d5d771f44f5fd87cd9f2a24788b61d238f8fa Mon Sep 17 00:00:00 2001 From: Kamila Szewczyk Date: Thu, 24 Nov 2022 14:18:51 +0100 Subject: [PATCH] bzip3 bs fix --- ar-mrzip/ar-mrzip.cpp | 260 +++++++++++++++++++++------------------- include/mrzip_private.h | 17 ++- rs-mrzip/rs-mrzip.c | 26 ++-- src/main.c | 16 +-- src/mrzip.c | 26 ++-- src/stream.c | 117 +++++------------- src/util.c | 18 +-- 7 files changed, 226 insertions(+), 254 deletions(-) diff --git a/ar-mrzip/ar-mrzip.cpp b/ar-mrzip/ar-mrzip.cpp index 5debb70..5ec17d6 100644 --- a/ar-mrzip/ar-mrzip.cpp +++ b/ar-mrzip/ar-mrzip.cpp @@ -2,15 +2,15 @@ #include #include +#include +#include #include +#include #include #include #include -#include -#include -#include -#include #include +#include #include #include #include @@ -20,8 +20,9 @@ #include #endif -#include "../common/blake2b.h" #include + +#include "../common/blake2b.h" namespace fs = std::filesystem; using namespace std::literals::chrono_literals; @@ -29,21 +30,21 @@ using namespace std::literals::chrono_literals; #include "tlsh.h" class latch { - private: - unsigned delay; - std::atomic count; - - public: - latch(unsigned delay) : delay(delay), count(0) {} - - bool operator()() { - count++; - if(count == delay) { - count = 0; - return true; - } - return false; + private: + unsigned delay; + std::atomic count; + + public: + latch(unsigned delay) : delay(delay), count(0) {} + + bool operator()() { + count++; + if (count == delay) { + count = 0; + return true; } + return false; + } }; class blake2b_cksum { @@ -62,16 +63,15 @@ class blake2b_cksum { // to speed up the TSP solution. For now we just use an approximation :). class tlsh_digest { - public: - uint8_t digest[TLSH_STRING_BUFFER_LEN]; - - int compare_to(const tlsh_digest & other) const { - // Return the amount of bytes that are the same. - int score = 0; - for(int i = 0; i < TLSH_STRING_BUFFER_LEN; i++) - score += digest[i] == other.digest[i]; - return score; - } + public: + uint8_t digest[TLSH_STRING_BUFFER_LEN]; + + int compare_to(const tlsh_digest & other) const { + // Return the amount of bytes that are the same. + int score = 0; + for (int i = 0; i < TLSH_STRING_BUFFER_LEN; i++) score += digest[i] == other.digest[i]; + return score; + } }; class file { @@ -101,7 +101,9 @@ void write_u64(uint64_t value) { uint64_t read_u64() { uint8_t bytes[8]; fread(bytes, 1, 8, stdin); - return ((uint64_t)bytes[0] << 56) | ((uint64_t)bytes[1] << 48) | ((uint64_t)bytes[2] << 40) | ((uint64_t)bytes[3] << 32) | ((uint64_t)bytes[4] << 24) | ((uint64_t)bytes[5] << 16) | ((uint64_t)bytes[6] << 8) | bytes[7]; + return ((uint64_t)bytes[0] << 56) | ((uint64_t)bytes[1] << 48) | ((uint64_t)bytes[2] << 40) | + ((uint64_t)bytes[3] << 32) | ((uint64_t)bytes[4] << 24) | ((uint64_t)bytes[5] << 16) | + ((uint64_t)bytes[6] << 8) | bytes[7]; } void write_u32(uint32_t value) { @@ -131,7 +133,7 @@ void compute_checksums(file & f, const fs::path & e) { } char buffer[4096]; ssize_t read_size; - if(f.size > 500) { + if (f.size > 500) { while ((read_size = read(fd, buffer, sizeof(buffer))) > 0) { tlsh.update((const unsigned char *)buffer, read_size); blake2b_update(&state, buffer, read_size); @@ -147,7 +149,7 @@ void compute_checksums(file & f, const fs::path & e) { exit(1); } close(fd); - if(f.size > 500) { + if (f.size > 500) { tlsh.final((const unsigned char *)buffer, read_size, 0); tlsh.getHash((char *)f.digest.digest, TLSH_STRING_BUFFER_LEN, 0); } else { @@ -157,7 +159,8 @@ void compute_checksums(file & f, const fs::path & e) { } int64_t current_time_secs() { - return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) + .count(); } // Create an archive from directory `dir' and output it to the standard output. @@ -196,7 +199,7 @@ void create(const char * dir) { std::cerr << std::endl << "* Computing checksums..." << std::endl; int processors = std::thread::hardware_concurrency(); - if(processors == 0) processors = 4; + if (processors == 0) processors = 4; // Compute checksums in parallel displaying status every 100MB. { @@ -205,11 +208,11 @@ void create(const char * dir) { std::vector threads; - for(int i = 0; i < processors; i++) { + for (int i = 0; i < processors; i++) { threads.emplace_back([&]() { - while(true) { + while (true) { size_t index = checksums_done++; - if(index >= files.size()) break; + if (index >= files.size()) break; compute_checksums(files[index], base_dir / files[index].name); checksum_total_bytes += files[index].size; checksum_running_bytes += files[index].size; @@ -220,8 +223,8 @@ void create(const char * dir) { std::atomic_bool stop = false; std::thread display = std::thread([&]() { - while(!stop) { - if(checksum_running_bytes > 100000000) { + while (!stop) { + if (checksum_running_bytes > 100000000) { std::cerr << "\33[2K\r" << checksum_total_bytes / 1000000 << "MB done ..." << std::flush; checksum_running_bytes = 0; } @@ -229,7 +232,7 @@ void create(const char * dir) { } }); - for(auto & t : threads) t.join(); + for (auto & t : threads) t.join(); stop = true; display.join(); @@ -250,13 +253,18 @@ void create(const char * dir) { std::atomic_size_t files_processed = 0; auto now = current_time_secs(); uint64_t next = 0, next_score = 0, c = 0, first_node = 0, last_node = files.size(); - while(c + 1 < last_node) { + while (c + 1 < last_node) { auto elapsed = current_time_secs() - now; - std::cerr << "\33[2K\rOrdering files " << files_processed++ << "/" << files.size() << ", " << (c / (elapsed + 1)) << " files/s..." << std::flush; + std::cerr << "\33[2K\rOrdering files " << files_processed++ << "/" << files.size() << ", " + << (c / (elapsed + 1)) << " files/s..." << std::flush; - for(uint64_t i = c + 1; i < last_node; i++) { + for (uint64_t i = c + 1; i < last_node; i++) { int score = files[c].digest.compare_to(files[i].digest); - if(next_score < score) { next_score = score; next = i; if(score > 130) break; } + if (next_score < score) { + next_score = score; + next = i; + if (score > 130) break; + } } // Swap. @@ -286,11 +294,13 @@ void create(const char * dir) { files_size += f.size; - if(output_latch()) std::cerr << "\33[2K\r" << dedup_size / 1024 << "KB / " << files_size / 1024 << "KB deduped" << std::flush; + if (output_latch()) + std::cerr << "\33[2K\r" << dedup_size / 1024 << "KB / " << files_size / 1024 << "KB deduped" + << std::flush; } } - std::cerr << std::endl << "* Writing metadata (" << (metadata_size/1024) << " KB)..." << std::endl; + std::cerr << std::endl << "* Writing metadata (" << (metadata_size / 1024) << " KB)..." << std::endl; // Write the metadata. for (auto & f : files) { @@ -314,7 +324,8 @@ void create(const char * dir) { } assert(f.archive_offset == current_offset); if (!fs::exists(base_dir / f.name)) { - std::cerr << "File " << f.name << " does not exist anymore, the header has been written already. Fatal error." << std::endl; + std::cerr << "File " << f.name + << " does not exist anymore, the header has been written already. Fatal error." << std::endl; exit(1); } if (fs::last_write_time(base_dir / f.name).time_since_epoch().count() != f.modification_date) { @@ -339,8 +350,9 @@ void create(const char * dir) { exit(1); } close(fd); - if(output_latch()) std::cerr << "\33[2K\r" << current_offset / 1024 << "KB / " << (files_size - dedup_size) / 1024 << "KB written" - << std::flush; + if (output_latch()) + std::cerr << "\33[2K\r" << current_offset / 1024 << "KB / " << (files_size - dedup_size) / 1024 + << "KB written" << std::flush; } std::cerr << std::endl << "* Done." << std::endl; @@ -373,11 +385,11 @@ void extract() { fread(name, 1, name_length, stdin); name[name_length] = 0; f.name = name; - if(fs::path(f.name).is_absolute()) { + if (fs::path(f.name).is_absolute()) { std::cerr << "Absolute path in archive: " << f.name << std::endl; exit(1); } - if(fs::path(f.name).lexically_normal() != fs::path(f.name)) { + if (fs::path(f.name).lexically_normal() != fs::path(f.name)) { std::cerr << "Path not normalized: " << f.name << std::endl; exit(1); } @@ -386,35 +398,39 @@ void extract() { } // Sort by the archive offset - std::sort(files.begin(), files.end(), [](const file & a, const file & b) { return a.archive_offset < b.archive_offset; }); + std::sort(files.begin(), files.end(), + [](const file & a, const file & b) { return a.archive_offset < b.archive_offset; }); // Extract. uint64_t current_offset = 0; - for(size_t i = 0; i < files.size(); i++) { - if(i + 1 < files.size() && files[i].archive_offset == files[i + 1].archive_offset) { + for (size_t i = 0; i < files.size(); i++) { + if (i + 1 < files.size() && files[i].archive_offset == files[i + 1].archive_offset) { // Two or more files point to the same place. size_t duplicates = 0, orig_i = i; do { - i++; duplicates++; - } while(i < files.size() && files[i].archive_offset == files[i - 1].archive_offset); + i++; + duplicates++; + } while (i < files.size() && files[i].archive_offset == files[i - 1].archive_offset); i--; // Create files, update their modification dates. std::vector fds; - for(size_t j = 0; j < duplicates; j++) { + for (size_t j = 0; j < duplicates; j++) { fs::create_directories(fs::path(files[orig_i + j].name).parent_path()); - if(fs::exists(files[orig_i + j].name)) + if (fs::exists(files[orig_i + j].name)) std::cerr << "File " << files[orig_i + j].name << " already exists, overwriting." << std::endl; int dest_fd = open(files[orig_i + j].name.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); - if(dest_fd == -1) { + if (dest_fd == -1) { std::cerr << "open(" << files[orig_i + j].name << ") failed: " << strerror(errno) << std::endl; exit(1); } fds.push_back(dest_fd); - fs::last_write_time(files[orig_i + j].name, fs::file_time_type(fs::file_time_type::duration(files[orig_i + j].modification_date))); + fs::last_write_time( + files[orig_i + j].name, + fs::file_time_type(fs::file_time_type::duration(files[orig_i + j].modification_date))); } - + // Copy the data. char buffer[4096]; ssize_t read_size; @@ -422,8 +438,8 @@ void extract() { blake2b_state state; blake2b_init(&state, 64); while ((read_size = fread(buffer, 1, std::min(sizeof(buffer), bytes_left), stdin)) > 0) { - for(int fd : fds) { - if(write(fd, buffer, read_size) != read_size) { + for (int fd : fds) { + if (write(fd, buffer, read_size) != read_size) { std::cerr << "write failed: " << strerror(errno) << std::endl; exit(1); } @@ -436,26 +452,27 @@ void extract() { std::cerr << "fread failed: " << strerror(errno) << std::endl; exit(1); } - for(int fd : fds) close(fd); + for (int fd : fds) close(fd); char current_digest[64]; blake2b_final(&state, current_digest, 64); - if(memcmp(current_digest, files[orig_i].checksum.digest, 64) != 0) { + if (memcmp(current_digest, files[orig_i].checksum.digest, 64) != 0) { std::cerr << "Checksum mismatch for " << files[orig_i].name << std::endl; exit(1); } } else { // Create the file, update its modification date. fs::create_directories(fs::path(files[i].name).parent_path()); - if(fs::exists(files[i].name)) + if (fs::exists(files[i].name)) std::cerr << "File " << files[i].name << " already exists, overwriting." << std::endl; int dest_fd = open(files[i].name.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); - if(dest_fd == -1) { + if (dest_fd == -1) { std::cerr << "open(" << files[i].name << ") failed: " << strerror(errno) << std::endl; exit(1); } - fs::last_write_time(files[i].name, fs::file_time_type(fs::file_time_type::duration(files[i].modification_date))); + fs::last_write_time(files[i].name, + fs::file_time_type(fs::file_time_type::duration(files[i].modification_date))); // Copy the data. char buffer[4096]; @@ -464,7 +481,7 @@ void extract() { blake2b_state state; blake2b_init(&state, 64); while ((read_size = fread(buffer, 1, std::min(sizeof(buffer), bytes_left), stdin)) > 0) { - if(write(dest_fd, buffer, read_size) != read_size) { + if (write(dest_fd, buffer, read_size) != read_size) { std::cerr << "write failed: " << strerror(errno) << std::endl; exit(1); } @@ -481,7 +498,7 @@ void extract() { char current_digest[64]; blake2b_final(&state, current_digest, 64); - if(memcmp(current_digest, files[i].checksum.digest, 64) != 0) { + if (memcmp(current_digest, files[i].checksum.digest, 64) != 0) { std::cerr << "Checksum mismatch for " << files[i].name << std::endl; exit(1); } @@ -489,68 +506,69 @@ void extract() { } } -#include "../include/config.h" #include -static struct option long_options[] = { - {"help", no_argument, 0, 'h'}, - {"version", no_argument, 0, 'V'}, - {"extract", no_argument, 0, 'x'}, - {"create", required_argument, 0, 'c'}, - {"regex", required_argument, 0, 'r'}, - {"verbose", no_argument, 0, 'v'}, - {"force", no_argument, 0, 'f'}, - {"skip", no_argument, 0, 's'}, - {"dest", required_argument, 0, 'd'}, - {0, 0, 0, 0} -}; +#include "../include/config.h" + +static struct option long_options[] = { { "help", no_argument, 0, 'h' }, { "version", no_argument, 0, 'V' }, + { "extract", no_argument, 0, 'x' }, { "create", required_argument, 0, 'c' }, + { "regex", required_argument, 0, 'r' }, { "verbose", no_argument, 0, 'v' }, + { "force", no_argument, 0, 'f' }, { "skip", no_argument, 0, 's' }, + { "dest", required_argument, 0, 'd' }, { 0, 0, 0, 0 } }; static const char * short_options = "hVxcr:fsd:"; static void usage(void) { - std::cerr << (PACKAGE - " version " PACKAGE_VERSION - "\n" - "Copyright (C) Kamila Szewczyk 2022\n" - "Usage: ar-mrzip [options] -d < [archive] OR ar-mrzip [options] [source] > [archive]\n" - "General options:\n" - "--------------------\n" - " -x, --extract extract from the archive\n" - " -c, --create create an archive from files in directory\n" - " -r, --regex perform the operations only on files that match a regex\n" - " -v, --verbose enable verbose output for progress monitoring\n" - " -h, --help display this message\n" - " -V, --version display version information\n" - " -f, --force force overwriting of existing files\n" - " -s, --skip skip existing files\n" - " -d, --dir set the destination directory for extraction or source directory for archiving\n" - "\n" - "The archive data for extraction is read from standard input. The created archive data is written to standard output.\n"); + std::cerr + << (PACKAGE + " version " PACKAGE_VERSION + "\n" + "Copyright (C) Kamila Szewczyk 2022\n" + "Usage: ar-mrzip [options] -d < [archive] OR ar-mrzip [options] [source] > [archive]\n" + "General options:\n" + "--------------------\n" + " -x, --extract extract from the archive\n" + " -c, --create create an archive from files in directory\n" + " -r, --regex perform the operations only on files that match a regex\n" + " -v, --verbose enable verbose output for progress monitoring\n" + " -h, --help display this message\n" + " -V, --version display version information\n" + " -f, --force force overwriting of existing files\n" + " -s, --skip skip existing files\n" + " -d, --dir set the destination directory for extraction or source directory for archiving\n" + "\n" + "The archive data for extraction is read from standard input. The created archive data is written to " + "standard output.\n"); } static void version(void) { std::cerr << (PACKAGE " version " PACKAGE_VERSION - "\n" - "Copyright (C) Kamila Szewczyk 2022\n" - "This is free software. You may redistribute copies of it under the terms of\n" - "the GNU General Public License .\n" - "There is NO WARRANTY, to the extent permitted by law.\n"); + "\n" + "Copyright (C) Kamila Szewczyk 2022\n" + "This is free software. You may redistribute copies of it under the terms of\n" + "the GNU General Public License .\n" + "There is NO WARRANTY, to the extent permitted by law.\n"); } enum { OP_EXTRACT, OP_CREATE }; enum { FILE_BEHAVIOUR_FORCE, FILE_BEHAVIOUR_SKIP, FILE_BEHAVIOUR_ASK }; int main(int argc, char * argv[]) { - #if defined(__MSVCRT__) - setmode(STDIN_FILENO, O_BINARY); - setmode(STDOUT_FILENO, O_BINARY); - #endif +#if defined(__MSVCRT__) + setmode(STDIN_FILENO, O_BINARY); + setmode(STDOUT_FILENO, O_BINARY); +#endif // Parse arguments using getopt_long. - int c; std::string destdir = "."; int operation = OP_EXTRACT; std::string regex = ""; int verbose = 0; int file_behaviour = FILE_BEHAVIOUR_ASK; - - while((c = getopt_long(argc, argv, short_options, long_options, NULL)) != -1) { - switch(c) { + int c; + std::string destdir = "."; + int operation = OP_EXTRACT; + std::string regex = ""; + int verbose = 0; + int file_behaviour = FILE_BEHAVIOUR_ASK; + + while ((c = getopt_long(argc, argv, short_options, long_options, NULL)) != -1) { + switch (c) { case 'h': usage(); return 0; @@ -584,24 +602,24 @@ int main(int argc, char * argv[]) { abort(); } } - - if(operation == OP_EXTRACT) { - if(optind != argc) { + + if (operation == OP_EXTRACT) { + if (optind != argc) { std::cerr << "Too many arguments." << std::endl; return 1; } extract(); - } else if(operation == OP_CREATE) { - if(optind == argc) { + } else if (operation == OP_CREATE) { + if (optind == argc) { std::cerr << "No source directory specified." << std::endl; return 1; } - if(optind + 1 != argc) { + if (optind + 1 != argc) { std::cerr << "Too many arguments." << std::endl; return 1; } create(argv[optind]); } - + return 0; } diff --git a/include/mrzip_private.h b/include/mrzip_private.h index 11832c8..cc8a82d 100644 --- a/include/mrzip_private.h +++ b/include/mrzip_private.h @@ -235,14 +235,12 @@ static inline unsigned char lzma2_prop_from_dic(u32 dicSize) { } /* re-purposed for bzip3. This value will be the actual block size from 32MB to 512MB - 1 */ -#define BZIP3_BLOCK_SIZE_FROM_PROP(p) (p == 8 ? 0x1FFFFFFF : (((u32)2 | ((p) & 1)) << ((p) / 2 + 24))) -static inline unsigned char bzip3_prop_from_block_size(u32 bs) -{ - unsigned i; - for (i = 0; i <= 8; i++) - if (bs <= BZIP3_BLOCK_SIZE_FROM_PROP(i)) - break; - return (unsigned char)i; +#define BZIP3_BLOCK_SIZE_FROM_PROP(p) (p == 8 ? 0x1FFFFFFF : (((u32)2 | ((p)&1)) << ((p) / 2 + 24))) +static inline unsigned char bzip3_prop_from_block_size(u32 bs) { + unsigned i; + for (i = 0; i <= 8; i++) + if (bs <= BZIP3_BLOCK_SIZE_FROM_PROP(i)) break; + return (unsigned char)i; } #define FLAG_SHOW_PROGRESS (1 << 0) @@ -451,7 +449,8 @@ struct rzip_control { unsigned delta; // delta flag offset (default 1) unsigned zpaq_level; // zpaq level unsigned zpaq_bs; // zpaq default block size - unsigned bzip3_bs; // bzip3 block size + unsigned bzip3_bs; // bzip3 block size code (0-8) + u32 bzip3_block_size; // actual block size decoded i64 window; unsigned long flags; i64 ramsize; diff --git a/rs-mrzip/rs-mrzip.c b/rs-mrzip/rs-mrzip.c index 6a25a7a..5a72160 100644 --- a/rs-mrzip/rs-mrzip.c +++ b/rs-mrzip/rs-mrzip.c @@ -158,7 +158,8 @@ static void encode(void) { } static void version(void) { - fprintf(stderr, "rs-mrzip (" PACKAGE " version " PACKAGE_VERSION ").\n" + fprintf(stderr, "rs-mrzip (" PACKAGE " version " PACKAGE_VERSION + ").\n" "Copyright (C) Kamila Szewczyk 2022\n" "Copyright (C) Phil Karn 1999\n" "This is free software. You may redistribute copies of it under the terms of\n" @@ -168,20 +169,21 @@ static void version(void) { static void usage(void) { version(); - fprintf(stderr, "usage: rs-mrzip [-e/-d/-h/-v] < input > output\n" - "options:\n" - " -e: encode (default)\n" - " -d: decode\n" - " -h: print this help\n" - " -v: print version\n"); + fprintf(stderr, + "usage: rs-mrzip [-e/-d/-h/-v] < input > output\n" + "options:\n" + " -e: encode (default)\n" + " -d: decode\n" + " -h: print this help\n" + " -v: print version\n"); } int main(int argc, char * argv[]) { - #if defined(__MSVCRT__) - setmode(STDIN_FILENO, O_BINARY); - setmode(STDOUT_FILENO, O_BINARY); - #endif - +#if defined(__MSVCRT__) + setmode(STDIN_FILENO, O_BINARY); + setmode(STDOUT_FILENO, O_BINARY); +#endif + if (argc == 1) encode(), exit(0); else if (argc != 2) diff --git a/src/main.c b/src/main.c index 4e8e723..87e42bf 100644 --- a/src/main.c +++ b/src/main.c @@ -110,7 +110,7 @@ static void usage(void) { " --fast alias for -L1\n" " --best alias for -L9\n" " --zpaqbs Set ZPAQ Block Size overriding defaults. 1-11, 2^zpaqbs * 1MB\n" - " --bzip3bs Set bzip3 Block Size. 1-8, 2^bzip3bs * 1MB.\n" + " --bzip3bs Set bzip3 Block Size. 0-8, 32 to 511MiB.\n" " Additional Compression Options:\n" " -C, --comment [comment] Add a comment up to 64 chars\n" " -e, --encrypt [=password] password protected sha512/aes128 encryption on compression\n" @@ -227,7 +227,8 @@ static void show_summary(void) { if (ZPAQ_COMPRESS) print_verbose("ZPAQ Compression Level: %'d, ZPAQ initial Block Size: %'d\n", control->zpaq_level, control->zpaq_bs); - if (BZIP3_COMPRESS) print_verbose("BZIP3 Compression Block Size: %'d\n", control->bzip3_bs); + if (BZIP3_COMPRESS) + print_verbose("BZIP3 Compression Block Size: %'" PRIu32 "\n", control->bzip3_block_size); print_verbose("%s Hashing Used\n", control->hash_label); if (ENCRYPT) print_verbose("%s Encryption Used\n", control->enc_label); if (control->window) @@ -306,10 +307,10 @@ static void set_stdout(struct rzip_control * control) { static const char * loptions = "BcC:dDe::E:fhH::iKlL:nN:o:O:p:PqR:sS:tT::Um:vVw:zZ?"; int main(int argc, char * argv[]) { - #if defined(__MSVCRT__) - setmode(STDIN_FILENO, O_BINARY); - setmode(STDOUT_FILENO, O_BINARY); - #endif +#if defined(__MSVCRT__) + setmode(STDIN_FILENO, O_BINARY); + setmode(STDOUT_FILENO, O_BINARY); +#endif bool lrzncat = false; bool options_file = false, @@ -559,8 +560,9 @@ int main(int argc, char * argv[]) { else { ds = strtol(optarg, &endptr, 10); if (*endptr) fatal("Extra characters after block size: \'%s\'\n", endptr); - if (ds < 0 || ds > 8) fatal("BZIP3 Block Size must be between 1 and 8\n"); + if (ds < 0 || ds > 8) fatal("BZIP3 Block Size must be between 0 and 8\n"); control->bzip3_bs = ds; + control->bzip3_block_size = BZIP3_BLOCK_SIZE_FROM_PROP(ds); } break; } // switch diff --git a/src/mrzip.c b/src/mrzip.c index 542c441..a484e36 100644 --- a/src/mrzip.c +++ b/src/mrzip.c @@ -161,7 +161,7 @@ bool write_magic(rzip_control * control) { } else if (BZIP3_COMPRESS) { /* Save block size. ZPAQ compression level is from 3 to 5, so this is sound. bzip3 blocksize is from 1 to 8 (or 0 to 7). */ - magic[17] = 0b11111000 | (control->bzip3_bs - 1); + magic[17] = 0b11110000 + bzip3_prop_from_block_size(control->bzip3_block_size); } /* save compression levels @@ -263,14 +263,17 @@ static void get_magic_v8(rzip_control * control, unsigned char * magic) { get_expected_size(control, magic); get_encryption(control, &magic[15], &magic[6]); - if ((magic[17] & 0b11111000) == 0b11111000) { - // bzip3 block size stuff. - control->bzip3_bs = (magic[17] & 0b00000111) + 1; - } else if (magic[17] & 0b10000000) // zpaq block and compression level stored + if ((magic[17] & 0b10000000)) // bzip3 or zpaq block sizes/levels stored { - control->zpaq_bs = magic[17] & 0b00001111; // low order bits are block size - magic[17] &= 0b01110000; // strip high bit - control->zpaq_level = magic[17] >> 4; // divide by 16 + if ((magic[17] & 0b11110000) == 0b11110000) { // bzip3 block size + control->bzip3_bs = magic[17] & 0b00001111; // bzip3 block size code 0 to 8 + control->bzip3_block_size = BZIP3_BLOCK_SIZE_FROM_PROP(control->bzip3_bs); // Real Block Size + } else // zpaq block and compression level stored + { + control->zpaq_bs = magic[17] & 0b00001111; // low order bits are block size + magic[17] &= 0b01110000; // strip high bit + control->zpaq_level = magic[17] >> 4; // divide by 16 + } } get_hash_from_magic(control, &magic[14]); @@ -420,8 +423,8 @@ bool write_fdout(rzip_control * control, void * buf, i64 len) { nmemb = len; ret = write(control->fd_out, offset_buf, (size_t)nmemb); /* error if ret == -1 only. Otherwise, buffer not wholly written */ - if (unlikely(ret == -1)) /* error, not underflow */ - fatal("Failed to write %'"PRId64" bytes to fd_out in write_fdout\n", nmemb); + if (unlikely(ret == -1)) /* error, not underflow */ + fatal("Failed to write %'" PRId64 " bytes to fd_out in write_fdout\n", nmemb); len -= ret; offset_buf += ret; } @@ -975,7 +978,8 @@ bool get_fileinfo(rzip_control * control) { else // early 0.8 or <0.8 file without zpaq coding in magic header print_output("\n"); } else if (save_ctype == BZIP3_COMPRESS) { - print_output("rzip + bzip3 -- Block Size: %d", (1 << control->bzip3_bs) * ONE_MB); + print_output("rzip + bzip3 -- Block Size: %d - %'" PRIu32 "\n", control->bzip3_bs, + control->bzip3_block_size); } else print_output("Dunno wtf\n"); diff --git a/src/stream.c b/src/stream.c index 6b1b33f..ee2d597 100644 --- a/src/stream.c +++ b/src/stream.c @@ -115,77 +115,12 @@ bool join_pthread(rzip_control * control, pthread_t th, void ** thread_return) { */ static int lz4_compresses(rzip_control * control, uchar * s_buf, i64 s_len); -/* BZIP3 COMPRESSION WRAPPER */ -static pthread_mutex_t bz3_statemutex = PTHREAD_MUTEX_INITIALIZER; -static struct bz3_state ** states = NULL; -static int * statequeue = NULL; - -static void setup_states(rzip_control * control) { - int i; - states = malloc(sizeof(struct bz3_state *) * (control->threads + 1)); - statequeue = malloc(sizeof(int) * (control->threads + 1)); - memset(statequeue, 0, sizeof(int) * (control->threads + 1)); - if (!states) fatal("Failed to allocate memory for bzip3 states\n"); - for (i = 0; i < (control->threads + 1); i++) { - states[i] = bz3_new((1 << control->bzip3_bs) * ONE_MB); - if (!states[i]) fatal("Failed to allocate %dMB bzip3 state #%d.\n", (1 << control->bzip3_bs), i); - } -} - -static struct bz3_state * lock_state(rzip_control * control) { - lock_mutex(control, &bz3_statemutex); - int i; - for (i = 0; i < (control->threads + 1); i++) { - if (!statequeue[i]) { - statequeue[i] = 1; - unlock_mutex(control, &bz3_statemutex); - return states[i]; - } - } - - fatal("internal error: out of thread states."); - return NULL; -} - -static void unlock_state(rzip_control * control, struct bz3_state * state) { - lock_mutex(control, &bz3_statemutex); - int i; - for (i = 0; i < (control->threads + 1); i++) { - if (states[i] == state) { - statequeue[i] = 0; - unlock_mutex(control, &bz3_statemutex); - return; - } - } - - fatal("internal error: state not in list."); -} - -static void bzip3_compress(rzip_control * control, uchar * c_buf, i64 * c_len, i64 s_len, int ct) { - if (!states) setup_states(control); - struct bz3_state * state = lock_state(control); - *c_len = bz3_encode_block(state, c_buf, s_len); - if (bz3_last_error(state) != BZ3_OK) { - print_err("Failed to compress with bz3: %s\n", bz3_strerror(state)); - return; - } - unlock_state(control, state); -} -static void bzip3_decompress(rzip_control * control, uchar * s_buf, i64 * s_len, i64 c_len, int ct) { - if (!states) setup_states(control); - struct bz3_state * state = lock_state(control); - *s_len = bz3_decode_block(state, s_buf, c_len, *s_len); - if (bz3_last_error(state) != BZ3_OK) { - print_err("Failed to decompress with bz3: %s\n", bz3_strerror(state)); - return; - } - unlock_state(control, state); -} - static int bzip3_compress_buf(rzip_control * control, struct compress_thread * cthread, int current_thread) { i64 c_len, c_size; uchar * c_buf; + struct bz3_state * state; + if (LZ4_TEST) { if (!lz4_compresses(control, cthread->s_buf, cthread->s_len)) return 0; } @@ -199,9 +134,13 @@ static int bzip3_compress_buf(rzip_control * control, struct compress_thread * c memcpy(c_buf, cthread->s_buf, cthread->s_len); c_len = 0; - print_verbose("Starting bzip3 (bs=%d) backend...\n", control->bzip3_bs); + print_verbose("Starting bzip3: bs=%d - %'" PRIu32 " bytes backend...\n", control->bzip3_bs, + control->bzip3_block_size); - bzip3_compress(control, c_buf, &c_len, cthread->s_len, current_thread); + state = bz3_new(control->bzip3_block_size); // allocate bzip3 state + if (!state) fatal("Failed to allocate %'" PRIu32 " bytes bzip3 state.\n", control->bzip3_block_size); + + c_len = bz3_encode_block(state, c_buf, cthread->s_len); if (unlikely(c_len >= cthread->c_len)) { print_maxverbose("Incompressible block\n"); @@ -214,6 +153,7 @@ static int bzip3_compress_buf(rzip_control * control, struct compress_thread * c dealloc(cthread->s_buf); cthread->s_buf = c_buf; cthread->c_type = CTYPE_BZIP3; + bz3_free(state); // free bzip3 state return 0; } @@ -414,6 +354,8 @@ static int bzip3_decompress_buf(rzip_control * control, struct uncomp_thread * u uchar * c_buf; int ret = 0; + struct bz3_state * state; + c_buf = ucthread->s_buf; ucthread->s_buf = malloc(round_up_page(control, dlen)); if (unlikely(!ucthread->s_buf)) { @@ -423,7 +365,10 @@ static int bzip3_decompress_buf(rzip_control * control, struct uncomp_thread * u } memcpy(ucthread->s_buf, c_buf, ucthread->c_len); - bzip3_decompress(control, ucthread->s_buf, &dlen, ucthread->c_len, current_thread); + state = bz3_new(control->bzip3_block_size); + + dlen = bz3_decode_block(state, ucthread->s_buf, ucthread->c_len, ucthread->u_len); + if (bz3_last_error(state) != BZ3_OK) fatal("Failed to decompress with bz3 %s\n", bz3_strerror(state)); if (unlikely(dlen != ucthread->u_len)) { print_err("Inconsistent length after decompression. Got %'" PRId64 " bytes, expected %'" PRId64 "\n", dlen, @@ -436,6 +381,7 @@ static int bzip3_decompress_buf(rzip_control * control, struct uncomp_thread * u dealloc(ucthread->s_buf); ucthread->s_buf = c_buf; } + bz3_free(state); return ret; } @@ -890,11 +836,6 @@ bool close_streamout_threads(rzip_control * control) { } dealloc(cthreads); dealloc(control->pthreads); - if (states != NULL) { - for (i = 0; i < control->threads; i++) bz3_free(states[i]); - free(states); - free(statequeue); - } return true; } @@ -962,31 +903,35 @@ void * open_stream_out(rzip_control * control, int f, unsigned int n, i64 chunk_ control->zpaq_bs = save_bs; // restore block size goto retry_zpaq; } - if (control->zpaq_bs != save_bs) print_verbose("ZPAQ Block Size reduced to %'d\n", control->zpaq_bs); + if (control->zpaq_bs != save_bs) print_verbose("ZPAQ Block Size reduced to %d\n", control->zpaq_bs); } else if (BZIP3_COMPRESS) { /* compute max possible block size. NB: This code sucks but I don't want to refactor it. */ int save_bs = control->bzip3_bs; - int BZIP3BSMIN = 3; + u32 BZIP3BSMIN = 1 << 25; retry_bzip3: do { for (control->threads = save_threads; control->threads >= thread_limit; control->threads--) { - if (limit >= control->overhead * control->threads) { + if (limit >= control->overhead * control->threads / testbufs) { overhead_set = true; break; } } // thread loop if (overhead_set == true) break; - else - control->bzip3_bs--; // decrement block size - setup_overhead(control); // recompute overhead - } while (control->bzip3_bs > BZIP3BSMIN); // block size loop - if (!overhead_set && thread_limit > 1) { // try again and lower thread_limit + else { + control->bzip3_bs--; // decrement block size + control->bzip3_block_size = BZIP3_BLOCK_SIZE_FROM_PROP(control->bzip3_bs); + } + setup_overhead(control); // recompute overhead + } while (control->bzip3_block_size > BZIP3BSMIN); // block size loop + if (!overhead_set && thread_limit > 1) { // try again and lower thread_limit thread_limit--; control->bzip3_bs = save_bs; // restore block size goto retry_bzip3; } - if (control->bzip3_bs != save_bs) print_verbose("BZIP3 Block Size reduced to %'d\n", control->bzip3_bs); + if (control->bzip3_bs != save_bs) + print_verbose("BZIP3 Block Size reduced to %d - %'" PRIu32 "\n", control->bzip3_bs, + control->bzip3_block_size); } if (control->threads != save_threads) print_verbose("Threads reduced to %'d\n", control->threads); @@ -1030,8 +975,8 @@ void * open_stream_out(rzip_control * control, int f, unsigned int n, i64 chunk_ if (ZPAQ_COMPRESS && (limit / control->threads > 0x100000 << control->zpaq_bs)) // ZPAQ buffer always larger than STREAM_BUFSIZE stream_bufsize = round_up_page(control, (0x100000 << control->zpaq_bs) - 0x1000); - else if (BZIP3_COMPRESS && (limit / control->threads > 0x100000 << control->bzip3_bs)) - stream_bufsize = round_up_page(control, (0x100000 << control->bzip3_bs) - 0x1000); + else if (BZIP3_COMPRESS && (limit / control->threads > control->bzip3_block_size)) + stream_bufsize = round_up_page(control, control->bzip3_block_size - 0x1000); else if (PPM_COMPRESS && (limit / control->threads > 0x100000 << control->bzip3_bs)) stream_bufsize = round_up_page(control, (0x100000 << control->bzip3_bs) - 0x1000); else if (LZMA_COMPRESS && limit / control->threads > STREAM_BUFSIZE) diff --git a/src/util.c b/src/util.c index fcbd920..7ad8d8f 100644 --- a/src/util.c +++ b/src/util.c @@ -120,32 +120,34 @@ void setup_overhead(rzip_control * control) { } control->overhead = (i64)(1 << control->zpaq_bs) * ONE_MB * 8; // times 8 or 16. Out for now } else if (BZIP3_COMPRESS) { - if (control->bzip3_bs == 0) { + if (control->bzip3_block_size == 0) { switch (control->compression_level) { case 1: case 2: case 3: case 4: case 5: - control->bzip3_bs = 5; + control->bzip3_bs = 0; break; // 32MB case 6: - control->bzip3_bs = 6; + control->bzip3_bs = 2; break; // 64MB case 7: - control->bzip3_bs = 7; + control->bzip3_bs = 4; break; // 128MB case 8: + control->bzip3_bs = 6; + break; // 256MB case 9: control->bzip3_bs = 8; - break; // 256MB + break; // 512MB-1 default: - control->bzip3_bs = 5; + control->bzip3_bs = 0; break; // should never reach here } } - - control->overhead = (i64)(1 << control->bzip3_bs) * ONE_MB * 6; + control->bzip3_block_size = BZIP3_BLOCK_SIZE_FROM_PROP(control->bzip3_bs); + control->overhead = (i64)control->bzip3_block_size * 6; } else if (PPM_COMPRESS) { control->overhead = (i64)(1 << control->compression_level) * ONE_MB; }