diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 81044ae3..30f2755c 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -64,6 +64,127 @@ jobs:
- name: Run tests
run: cargo test --all-features --workspace
+ cross-compile-aarch64:
+ runs-on: ubuntu-latest
+ name: Cross-compile for aarch64
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Install Rust
+ uses: dtolnay/rust-toolchain@nightly
+ with:
+ targets: aarch64-unknown-linux-gnu
+
+ - name: Setup Rust cache
+ uses: Swatinem/rust-cache@v2
+ with:
+ shared-key: "cross-compile-aarch64"
+ cache-on-failure: true
+
+ - name: Install cross
+ run: cargo install cross --locked
+
+ - name: Build library for aarch64
+ run: cross build --lib --target aarch64-unknown-linux-gnu
+
+ - name: Build tests for aarch64
+ run: cross build --tests --target aarch64-unknown-linux-gnu
+
+ - name: Verify aarch64 build
+ run: |
+ echo "✓ aarch64-unknown-linux-gnu build succeeded"
+ echo " - Architecture separation validated"
+ echo " - x86_64 backends: hasher_input, md5x2_scalar, md5x2_sse2,"
+ echo " md5x2_bmi1, md5x2_avx512"
+ echo " - aarch64 backends: md5x2_scalar, md5x2_neon, crc_armcrc"
+ echo " - Shared: md5x2 trait"
+
+ build-windows-binaries:
+ runs-on: windows-latest
+ name: Build Windows binaries
+ if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Install Rust
+ uses: dtolnay/rust-toolchain@nightly
+ with:
+ targets: x86_64-pc-windows-msvc
+
+ - name: Setup Rust cache
+ uses: Swatinem/rust-cache@v2
+ with:
+ shared-key: "build-windows-binaries"
+ cache-on-failure: true
+
+ - name: Build release binaries
+ run: cargo build --release --bins --target x86_64-pc-windows-msvc
+
+ - name: Package Windows binaries
+ shell: pwsh
+ run: |
+ New-Item -ItemType Directory -Force -Path dist | Out-Null
+ Compress-Archive `
+ -Path `
+ target/x86_64-pc-windows-msvc/release/par2.exe, `
+ target/x86_64-pc-windows-msvc/release/par2create.exe, `
+ target/x86_64-pc-windows-msvc/release/par2repair.exe, `
+ target/x86_64-pc-windows-msvc/release/par2verify.exe `
+ -DestinationPath dist/par2rs-windows-x86_64.zip
+
+ - name: Upload Windows binaries
+ uses: actions/upload-artifact@v4
+ with:
+ name: par2rs-windows-x86_64
+ path: |
+ dist/par2rs-windows-x86_64.zip
+ target/x86_64-pc-windows-msvc/release/par2.exe
+ target/x86_64-pc-windows-msvc/release/par2create.exe
+ target/x86_64-pc-windows-msvc/release/par2repair.exe
+ target/x86_64-pc-windows-msvc/release/par2verify.exe
+
+ build-linux-x86_64-binaries:
+ runs-on: ubuntu-latest
+ name: Build Linux x86_64 binaries
+ if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Install Rust
+ uses: dtolnay/rust-toolchain@nightly
+ with:
+ targets: x86_64-unknown-linux-gnu
+
+ - name: Setup Rust cache
+ uses: Swatinem/rust-cache@v2
+ with:
+ shared-key: "build-linux-x86_64-binaries"
+ cache-on-failure: true
+
+ - name: Build release binaries
+ run: cargo build --release --bins --target x86_64-unknown-linux-gnu
+
+ - name: Package Linux x86_64 binaries
+ run: |
+ mkdir -p dist
+ tar -C target/x86_64-unknown-linux-gnu/release -czf \
+ dist/par2rs-linux-x86_64-nightly.tar.gz \
+ par2 par2create par2repair par2verify
+
+ - name: Upload Linux x86_64 binaries
+ uses: actions/upload-artifact@v4
+ with:
+ name: par2rs-linux-x86_64-nightly
+ path: |
+ dist/par2rs-linux-x86_64-nightly.tar.gz
+ target/x86_64-unknown-linux-gnu/release/par2
+ target/x86_64-unknown-linux-gnu/release/par2create
+ target/x86_64-unknown-linux-gnu/release/par2repair
+ target/x86_64-unknown-linux-gnu/release/par2verify
+
coverage:
runs-on: ubuntu-latest
diff --git a/Cargo.lock b/Cargo.lock
index ebedbc23..ead4b0f8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -166,6 +166,16 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
+[[package]]
+name = "cc"
+version = "1.2.61"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d"
+dependencies = [
+ "find-msvc-tools",
+ "shlex",
+]
+
[[package]]
name = "cfg-if"
version = "1.0.4"
@@ -238,6 +248,16 @@ version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0dabb6555f92fb9ee4140454eb5dcd14c7960e1225c6d1a6cc361f032947713e"
+[[package]]
+name = "crc-fast"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5"
+dependencies = [
+ "digest 0.10.7",
+ "spin",
+]
+
[[package]]
name = "crc32fast"
version = "1.5.0"
@@ -311,6 +331,16 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
+[[package]]
+name = "crypto-common"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
+dependencies = [
+ "generic-array",
+ "typenum",
+]
+
[[package]]
name = "crypto-common"
version = "0.2.0-rc.5"
@@ -340,6 +370,15 @@ dependencies = [
"syn",
]
+[[package]]
+name = "digest"
+version = "0.10.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
+dependencies = [
+ "crypto-common 0.1.7",
+]
+
[[package]]
name = "digest"
version = "0.11.0-rc.4"
@@ -348,7 +387,7 @@ checksum = "ea390c940e465846d64775e55e3115d5dc934acb953de6f6e6360bc232fe2bf7"
dependencies = [
"block-buffer",
"const-oid",
- "crypto-common",
+ "crypto-common 0.2.0-rc.5",
]
[[package]]
@@ -396,12 +435,28 @@ version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
+[[package]]
+name = "find-msvc-tools"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
+
[[package]]
name = "fnv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+[[package]]
+name = "generic-array"
+version = "0.14.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
+dependencies = [
+ "typenum",
+ "version_check",
+]
+
[[package]]
name = "getrandom"
version = "0.3.4"
@@ -555,7 +610,7 @@ version = "0.11.0-rc.2"
source = "git+https://github.com/mjc/hashes.git?branch=mjc-asm#6a68d94060f1675411fd6ecebfd0fff36d1fef8e"
dependencies = [
"cfg-if",
- "digest",
+ "digest 0.11.0-rc.4",
]
[[package]]
@@ -604,12 +659,15 @@ dependencies = [
"anyhow",
"binrw",
"bytemuck",
+ "cc",
"clap",
+ "crc-fast",
"crc32fast",
"criterion",
"env_logger",
"hex",
"iai-callgrind",
+ "libc",
"log",
"md-5",
"proptest",
@@ -619,6 +677,7 @@ dependencies = [
"smallvec",
"tempfile",
"thiserror",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -926,12 +985,24 @@ dependencies = [
"serde_core",
]
+[[package]]
+name = "shlex"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+
[[package]]
name = "smallvec"
version = "1.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
+[[package]]
+name = "spin"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591"
+
[[package]]
name = "strsim"
version = "0.11.1"
@@ -1016,6 +1087,12 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
+[[package]]
+name = "version_check"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
+
[[package]]
name = "wait-timeout"
version = "0.2.1"
diff --git a/Cargo.toml b/Cargo.toml
index ff3e17b4..9924795c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -2,11 +2,13 @@
name = "par2rs"
version = "0.1.0"
edition = "2021"
-license = "GPL-3.0-only"
+license = "GPL-2.0-or-later"
[features]
# Feature flag to skip tests that are unsuitable in Nix builds
nix-build = []
+# Feature for ParPar C++ FFI comparison (benchmarks only, adds build time)
+parpar-compare = []
[dependencies]
binrw = "*"
@@ -24,6 +26,10 @@ rustc-hash = "2.1"
thiserror = "2.0.17"
anyhow = "1.0"
smallvec = { version = "1.15.1", features = ["const_generics"] }
+libc = "0.2"
+
+[target.'cfg(windows)'.dependencies]
+windows-sys = { version = "0.61", features = ["Win32_System_Memory", "Win32_System_SystemInformation"] }
# Binaries
[[bin]]
@@ -42,6 +48,10 @@ path = "src/bin/par2repair.rs"
name = "par2create"
path = "src/bin/par2create.rs"
+[build-dependencies]
+# C++ compiler for conditional ParPar FFI compilation
+cc = "1.0"
+
[dev-dependencies]
# Temporary directory management for tests
tempfile = "3"
@@ -53,6 +63,8 @@ iai-callgrind = "0.16"
proptest = "1.5"
# Random number generation for tests
rand = "0.9"
+# CRC32 alternative under evaluation vs crc32fast (bench-only for now)
+crc-fast = "1.4"
[[bench]]
name = "repair_benchmark"
@@ -78,6 +90,42 @@ harness = false
name = "iai_simd"
harness = false
+[[bench]]
+name = "fused_hashing"
+harness = false
+
+[[bench]]
+name = "crc_compare"
+harness = false
+
+[[bench]]
+name = "md5x2_crc_fused"
+harness = false
+
+[[bench]]
+name = "parpar_hasher_input"
+harness = false
+
+[[bench]]
+name = "iai_hasher_input"
+harness = false
+
+[[bench]]
+name = "compare_with_parpar"
+harness = false
+
+[[bench]]
+name = "iai_parpar_comparison"
+harness = false
+
+[[bench]]
+name = "par2verify_compare"
+harness = false
+
+[[bench]]
+name = "iai_par2verify_compare"
+harness = false
+
[profile.release]
debug = true
strip = false
diff --git a/LICENSE b/LICENSE
index f288702d..9efa6fbc 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,622 +1,281 @@
GNU GENERAL PUBLIC LICENSE
- Version 3, 29 June 2007
+ Version 2, June 1991
- Copyright (C) 2007 Free Software Foundation, Inc.
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
- The GNU General Public License is a free, copyleft license for
-software and other kinds of works.
-
- The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works. By contrast,
-the GNU General Public License is intended to guarantee your freedom to
-share and change all versions of a program--to make sure it remains free
-software for all its users. We, the Free Software Foundation, use the
-GNU General Public License for most of our software; it applies also to
-any other work released this way by its authors. You can apply it to
+ The licenses for most software are designed to take away your
+freedom to share and change it. By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users. This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it. (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.) You can apply it to
your programs, too.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
- To protect your rights, we need to prevent others from denying you
-these rights or asking you to surrender the rights. Therefore, you have
-certain responsibilities if you distribute copies of the software, or if
-you modify it: responsibilities to respect the freedom of others.
+ To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must pass on to the recipients the same
-freedoms that you received. You must make sure that they, too, receive
-or can get the source code. And you must show them these terms so they
-know their rights.
-
- Developers that use the GNU GPL protect your rights with two steps:
-(1) assert copyright on the software, and (2) offer you this License
-giving you legal permission to copy, distribute and/or modify it.
-
- For the developers' and authors' protection, the GPL clearly explains
-that there is no warranty for this free software. For both users' and
-authors' sake, the GPL requires that modified versions be marked as
-changed, so that their problems will not be attributed erroneously to
-authors of previous versions.
-
- Some devices are designed to deny users access to install or run
-modified versions of the software inside them, although the manufacturer
-can do so. This is fundamentally incompatible with the aim of
-protecting users' freedom to change the software. The systematic
-pattern of such abuse occurs in the area of products for individuals to
-use, which is precisely where it is most unacceptable. Therefore, we
-have designed this version of the GPL to prohibit the practice for those
-products. If such problems arise substantially in other domains, we
-stand ready to extend this provision to those domains in future versions
-of the GPL, as needed to protect the freedom of users.
-
- Finally, every program is threatened constantly by software patents.
-States should not allow patents to restrict development and use of
-software on general-purpose computers, but in those that do, we wish to
-avoid the special danger that patents applied to a free program could
-make it effectively proprietary. To prevent this, the GPL assures that
-patents cannot be used to render the program non-free.
+gratis or for a fee, you must give the recipients all the rights that
+you have. You must make sure that they, too, receive or can get the
+source code. And you must show them these terms so they know their
+rights.
+
+ We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+ Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software. If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+ Finally, any free program is threatened constantly by software
+patents. We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary. To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
The precise terms and conditions for copying, distribution and
modification follow.
- TERMS AND CONDITIONS
-
- 0. Definitions.
-
- "This License" refers to version 3 of the GNU General Public License.
-
- "Copyright" also means copyright-like laws that apply to other kinds of
-works, such as semiconductor masks.
-
- "The Program" refers to any copyrightable work licensed under this
-License. Each licensee is addressed as "you". "Licensees" and
-"recipients" may be individuals or organizations.
-
- To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of an
-exact copy. The resulting work is called a "modified version" of the
-earlier work or a work "based on" the earlier work.
-
- A "covered work" means either the unmodified Program or a work based
-on the Program.
-
- To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy. Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
-
- To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies. Mere interaction with a user through
-a computer network, with no transfer of a copy, is not conveying.
-
- An interactive user interface displays "Appropriate Legal Notices"
-to the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License. If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
-
- 1. Source Code.
-
- The "source code" for a work means the preferred form of the work
-for making modifications to it. "Object code" means any non-source
-form of a work.
-
- A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
-
- The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form. A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
-
- The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities. However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work. For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
- The Corresponding Source need not include anything that users
-can regenerate automatically from other parts of the Corresponding
-Source.
-
- The Corresponding Source for a work in source code form is that
-same work.
-
- 2. Basic Permissions.
-
- All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met. This License explicitly affirms your unlimited
-permission to run the unmodified Program. The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work. This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
- You may make, run and propagate covered works that you do not
-convey, without conditions so long as your license otherwise remains
-in force. You may convey covered works to others for the sole purpose
-of having them make modifications exclusively for you, or provide you
-with facilities for running those works, provided that you comply with
-the terms of this License in conveying all material for which you do
-not control copyright. Those thus making or running the covered works
-for you must do so exclusively on your behalf, under your direction
-and control, on terms that prohibit them from making any copies of
-your copyrighted material outside their relationship with you.
-
- Conveying under any other circumstances is permitted solely under
-the conditions stated below. Sublicensing is not allowed; section 10
-makes it unnecessary.
-
- 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
- No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
- When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such circumvention
-is effected by exercising rights under this License with respect to
-the covered work, and you disclaim any intention to limit operation or
-modification of the work as a means of enforcing, against the work's
-users, your or third parties' legal rights to forbid circumvention of
-technological measures.
-
- 4. Conveying Verbatim Copies.
-
- You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
- You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
- 5. Conveying Modified Source Versions.
-
- You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
- a) The work must carry prominent notices stating that you modified
- it, and giving a relevant date.
-
- b) The work must carry prominent notices stating that it is
- released under this License and any conditions added under section
- 7. This requirement modifies the requirement in section 4 to
- "keep intact all notices".
-
- c) You must license the entire work, as a whole, under this
- License to anyone who comes into possession of a copy. This
- License will therefore apply, along with any applicable section 7
- additional terms, to the whole of the work, and all its parts,
- regardless of how they are packaged. This License gives no
- permission to license the work in any other way, but it does not
- invalidate such permission if you have separately received it.
-
- d) If the work has interactive user interfaces, each must display
- Appropriate Legal Notices; however, if the Program has interactive
- interfaces that do not display Appropriate Legal Notices, your
- work need not make them do so.
-
- A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit. Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
- 6. Conveying Non-Source Forms.
-
- You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
- a) Convey the object code in, or embodied in, a physical product
- (including a physical distribution medium), accompanied by the
- Corresponding Source fixed on a durable physical medium
- customarily used for software interchange.
-
- b) Convey the object code in, or embodied in, a physical product
- (including a physical distribution medium), accompanied by a
- written offer, valid for at least three years and valid for as
- long as you offer spare parts or customer support for that product
- model, to give anyone who possesses the object code either (1) a
- copy of the Corresponding Source for all the software in the
- product that is covered by this License, on a durable physical
- medium customarily used for software interchange, for a price no
- more than your reasonable cost of physically performing this
- conveying of source, or (2) access to copy the
- Corresponding Source from a network server at no charge.
-
- c) Convey individual copies of the object code with a copy of the
- written offer to provide the Corresponding Source. This
- alternative is allowed only occasionally and noncommercially, and
- only if you received the object code with such an offer, in accord
- with subsection 6b.
-
- d) Convey the object code by offering access from a designated
- place (gratis or for a charge), and offer equivalent access to the
- Corresponding Source in the same way through the same place at no
- further charge. You need not require recipients to copy the
- Corresponding Source along with the object code. If the place to
- copy the object code is a network server, the Corresponding Source
- may be on a different server (operated by you or a third party)
- that supports equivalent copying facilities, provided you maintain
- clear directions next to the object code saying where to find the
- Corresponding Source. Regardless of what server hosts the
- Corresponding Source, you remain obligated to ensure that it is
- available for as long as needed to satisfy these requirements.
-
- e) Convey the object code using peer-to-peer transmission, provided
- you inform other peers where the object code and Corresponding
- Source of the work are being offered to the general public at no
- charge under subsection 6d.
-
- A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
- A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling. In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage. For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product. A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
- "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source. The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
- If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information. But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
- The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed. Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
- Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
- 7. Additional Terms.
-
- "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law. If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
- When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it. (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.) You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
- Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
- a) Disclaiming warranty or limiting liability differently from the
- terms of sections 15 and 16 of this License; or
-
- b) Requiring preservation of specified reasonable legal notices or
- author attributions in that material or in the Appropriate Legal
- Notices displayed by works containing it; or
-
- c) Prohibiting misrepresentation of the origin of that material, or
- requiring that modified versions of such material be marked in
- reasonable ways as different from the original version; or
-
- d) Limiting the use for publicity purposes of names of licensors or
- authors of the material; or
-
- e) Declining to grant rights under trademark law for use of some
- trade names, trademarks, or service marks; or
-
- f) Requiring indemnification of licensors and authors of that
- material by anyone who conveys the material (or modified versions of
- it) with contractual assumptions of liability to the recipient, for
- any liability that these contractual assumptions directly impose on
- those licensors and authors.
-
- All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10. If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term. If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
- If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
- Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
- 8. Termination.
-
- You may not propagate or modify a covered work except as expressly
-provided under this License. Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
- However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
- Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
- Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License. If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
- 9. Acceptance Not Required for Having Copies.
-
- You are not required to accept this License in order to receive or
-run a copy of the Program. Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance. However,
-nothing other than this License grants you permission to propagate or
-modify any covered work. These actions infringe copyright if you do
-not accept this License. Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
- 10. Automatic Licensing of Downstream Recipients.
-
- Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License. You are not responsible
-for enforcing compliance by third parties with this License.
-
- An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations. If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
- You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License. For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
- 11. Patents.
-
- A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based. The
-work thus licensed is called the contributor's "contributor version".
-
- A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version. For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
+ GNU GENERAL PUBLIC LICENSE
+ TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+ 0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License. The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language. (Hereinafter, translation is included without limitation in
+the term "modification".) Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope. The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+ 1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+ 2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+ a) You must cause the modified files to carry prominent notices
+ stating that you changed the files and the date of any change.
+
+ b) You must cause any work that you distribute or publish, that in
+ whole or in part contains or is derived from the Program or any
+ part thereof, to be licensed as a whole at no charge to all third
+ parties under the terms of this License.
+
+ c) If the modified program normally reads commands interactively
+ when run, you must cause it, when started running for such
+ interactive use in the most ordinary way, to print or display an
+ announcement including an appropriate copyright notice and a
+ notice that there is no warranty (or else, saying that you provide
+ a warranty) and that users may redistribute the program under
+ these conditions, and telling the user how to view a copy of this
+ License. (Exception: if the Program itself is interactive but
+ does not normally print such an announcement, your work based on
+ the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole. If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works. But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+ 3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+ a) Accompany it with the complete corresponding machine-readable
+ source code, which must be distributed under the terms of Sections
+ 1 and 2 above on a medium customarily used for software interchange; or,
+
+ b) Accompany it with a written offer, valid for at least three
+ years, to give any third party, for a charge no more than your
+ cost of physically performing source distribution, a complete
+ machine-readable copy of the corresponding source code, to be
+ distributed under the terms of Sections 1 and 2 above on a medium
+ customarily used for software interchange; or,
+
+ c) Accompany it with the information you received as to the offer
+ to distribute corresponding source code. (This alternative is
+ allowed only for noncommercial distribution and only if you
+ received the program in object code or executable form with such
+ an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it. For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable. However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+ 4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License. Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+ 5. You are not required to accept this License, since you have not
+signed it. However, nothing else grants you permission to modify or
+distribute the Program or its derivative works. These actions are
+prohibited by law if you do not accept this License. Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+ 6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions. You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
this License.
- Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
- In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement). To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
- If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients. "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
- If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
- A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License. You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
- Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
- 12. No Surrender of Others' Freedom.
-
- If conditions are imposed on you (whether by court order, agreement or
+ 7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License. If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all. For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
- 13. Use with the GNU Affero General Public License.
-
- Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU Affero General Public License into a single
-combined work, and to convey the resulting work. The terms of this
-License will continue to apply to the part which is the covered work,
-but the special requirements of the GNU Affero General Public License,
-section 13, concerning interaction through a network will apply to the
-combination as such.
-
- 14. Revised Versions of this License.
-
- The Free Software Foundation may publish revised and/or new versions of
-the GNU General Public License from time to time. Such new versions will
+excuse you from the conditions of this License. If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all. For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices. Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+ 8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded. In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+ 9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time. Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
- Each version is given a distinguishing version number. If the
-Program specifies that a certain numbered version of the GNU General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation. If the Program does not specify a version number of the
-GNU General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
- If the Program specifies that a proxy can decide which future
-versions of the GNU General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
- Later license versions may give you additional or different
-permissions. However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
- 15. Disclaimer of Warranty.
-
- THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
- 16. Limitation of Liability.
-
- IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
- 17. Interpretation of Sections 15 and 16.
-
- If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
+Each version is given a distinguishing version number. If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation. If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+ 10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission. For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this. Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+ NO WARRANTY
+
+ 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+ 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
END OF TERMS AND CONDITIONS
@@ -628,15 +287,15 @@ free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
+convey the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
Copyright (C)
- This program is free software: you can redistribute it and/or modify
+ This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
+ the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
@@ -644,31 +303,36 @@ the "copyright" line and a pointer to where the full notice is found.
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, see .
Also add information on how to contact you by electronic and paper mail.
- If the program does terminal interaction, make it output a short
-notice like this when it starts in an interactive mode:
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
- Copyright (C)
- This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+ Gnomovision version 69, Copyright (C) year name of author
+ Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
This is free software, and you are welcome to redistribute it
under certain conditions; type `show c' for details.
The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License. Of course, your program's commands
-might be different; for a GUI interface, you would use an "about box".
-
- You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU GPL, see
-.
-
- The GNU General Public License does not permit incorporating your program
-into proprietary programs. If your program is a subroutine library, you
-may consider it more useful to permit linking proprietary applications with
-the library. If this is what you want to do, use the GNU Lesser General
-Public License instead of this License. But first, please read
-.
+parts of the General Public License. Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary. Here is a sample; alter the names:
+
+ Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+ `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+ , 1 April 1989
+ Moe Ghoul, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs. If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library. If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.
diff --git a/Makefile b/Makefile
index 68dc087f..aa462816 100644
--- a/Makefile
+++ b/Makefile
@@ -8,7 +8,7 @@ help:
@echo ""
@echo "Available targets:"
@echo " test - Run all tests"
- @echo " benchmark-create-perf - Compare par2rs create vs par2cmdline-turbo with perf counters"
+ @echo " benchmark-create-perf - Compare par2rs create vs par2cmdline-turbo with profiling support"
@echo " coverage - Generate HTML coverage report"
@echo " coverage-html - Generate HTML coverage report"
@echo " coverage-llvm - Generate text, HTML, LCOV, Cobertura, and Codecov JSON reports"
diff --git a/README.md b/README.md
index ad87aced..b9ab428b 100644
--- a/README.md
+++ b/README.md
@@ -193,28 +193,32 @@ cargo build --release --bins
### Create Performance Benchmarking
-For create-side comparisons on loaded machines, use the perf-counter benchmark
-rather than wall-clock timing:
+For create-side comparisons, use the benchmark harness:
```bash
nix develop --command make benchmark-create-perf
```
-The benchmark compares `par2rs` create against `par2cmdline-turbo` (`par2` by
-default), runs 30 measured iterations per case, reports mean/stddev/CV for
-Linux `perf` counters, and writes a `par2rs` create flamegraph. Results are
-saved under `target/perf-results/create/`.
+The benchmark compares `par2rs` create variants against `par2cmdline-turbo`
+(`par2` by default), runs measured iterations per case, verifies outputs across
+tools, reports wall time, records Linux `perf` counters when available, and
+writes a selected `par2rs` create flamegraph. On macOS, the flamegraph is
+generated with `xctrace`, `inferno-collapse-xctrace`, and `inferno-flamegraph`.
+Results are saved under
+`target/perf-results/create/`.
Useful overrides:
```bash
-nix develop --command env ITERATIONS=40 THREADS=16 \
- CASES='single_256m:1:256:1048576,multi_1g:64:16:1048576' \
+nix develop --command env ITERATIONS=10 THREADS=16 PROFILE_CASE=single_5g \
+ CASES='single_256m:1:256:1048576,multi_1g:64:16:1048576,single_5g:1:5120:1048576' \
scripts/benchmark_create_perf.sh
```
-The primary comparison is `par2rs/turbo` instructions. Wall-clock seconds are
-included only as context because they are noisy on a busy system.
+The pass/fail signal is mean wall time: `par2rs-xor-jit` must match or
+beat `turbo-auto` for every configured case. Linux `perf` counters are recorded
+to explain the wall-time result when the benchmark is run on Linux. Set
+`INCLUDE_PSHUFB=1` to add the `par2rs-pshufb` diagnostic baseline.
### Testing
@@ -418,9 +422,9 @@ This keeps the common case efficient while matching `par2cmdline` behavior for t
## License
-This project is licensed under the GNU General Public License v3.0 only - see the [LICENSE](LICENSE) file for details.
+This project is licensed under the GNU General Public License, version 2 or (at your option) any later version - see the [LICENSE](LICENSE) file for details.
-The project was relicensed to GPL-3.0-only so future PAR2 compatibility and performance work can incorporate GPL-compatible implementation details from established PAR2 tools when appropriate, with attribution and source availability preserved.
+The project uses GPL-2.0-or-later so future PAR2 compatibility and performance work can incorporate GPL-compatible implementation details from established PAR2 tools (notably the par2cmdline / par2cmdline-turbo / ParPar lineage, which is GPL-2.0-or-later) when appropriate, with attribution and source availability preserved.
## Acknowledgments
diff --git a/benches/common/par2verify_fixture.rs b/benches/common/par2verify_fixture.rs
new file mode 100644
index 00000000..d60a95ae
--- /dev/null
+++ b/benches/common/par2verify_fixture.rs
@@ -0,0 +1,286 @@
+#![allow(dead_code)]
+
+use par2rs::create::{CreateContextBuilder, SilentCreateReporter};
+use std::env;
+use std::ffi::OsString;
+use std::fs::{self, File};
+use std::io::{self, BufWriter, Write};
+use std::path::{Path, PathBuf};
+use std::sync::OnceLock;
+
+const DEFAULT_TURBO_ROOT: &str = "/home/mjc/projects/par2cmdline-turbo";
+const DATA_CHUNK_BYTES: usize = 1024 * 1024;
+
+#[derive(Debug, Clone)]
+pub struct VerifyFixture {
+ pub case_name: &'static str,
+ pub dir: PathBuf,
+ pub par2_file: PathBuf,
+ pub protected_bytes: u64,
+}
+
+#[derive(Debug, Clone)]
+pub struct Invocation {
+ pub program: PathBuf,
+ pub args: Vec,
+ pub current_dir: PathBuf,
+}
+
+#[derive(Debug, Clone)]
+struct TurboCommand {
+ program: PathBuf,
+ use_verify_subcommand: bool,
+}
+
+static CRITERION_FIXTURE: OnceLock = OnceLock::new();
+static IAI_FIXTURE: OnceLock = OnceLock::new();
+static PAR2RS_VERIFY: OnceLock = OnceLock::new();
+static TURBO_VERIFY: OnceLock = OnceLock::new();
+
+pub fn criterion_fixture() -> &'static VerifyFixture {
+ CRITERION_FIXTURE
+ .get_or_init(|| create_fixture("criterion-intact-64m", 64 * 1024 * 1024, 1024 * 1024))
+}
+
+pub fn iai_fixture() -> &'static VerifyFixture {
+ IAI_FIXTURE.get_or_init(|| create_fixture("iai-intact-8m", 8 * 1024 * 1024, 256 * 1024))
+}
+
+pub fn iai_fixture_arg() -> String {
+ iai_fixture().par2_file.to_string_lossy().into_owned()
+}
+
+pub fn par2rs_verify_invocation(fixture: &VerifyFixture) -> Invocation {
+ verify_invocation(resolve_par2rs_verify().clone(), false, &fixture.par2_file)
+}
+
+pub fn turbo_verify_invocation(fixture: &VerifyFixture) -> Invocation {
+ let turbo = resolve_turbo_verify().clone();
+ verify_invocation(
+ turbo.program,
+ turbo.use_verify_subcommand,
+ &fixture.par2_file,
+ )
+}
+
+pub fn par2rs_verify_invocation_for_path(par2_file: impl AsRef) -> Invocation {
+ verify_invocation(resolve_par2rs_verify().clone(), false, par2_file.as_ref())
+}
+
+pub fn turbo_verify_invocation_for_path(par2_file: impl AsRef) -> Invocation {
+ let turbo = resolve_turbo_verify().clone();
+ verify_invocation(
+ turbo.program,
+ turbo.use_verify_subcommand,
+ par2_file.as_ref(),
+ )
+}
+
+fn verify_invocation(
+ program: PathBuf,
+ use_verify_subcommand: bool,
+ par2_file: &Path,
+) -> Invocation {
+ let mut args = Vec::with_capacity(3);
+ if use_verify_subcommand {
+ args.push(OsString::from("verify"));
+ }
+ args.push(OsString::from("-q"));
+ args.push(par2_file.as_os_str().to_os_string());
+
+ let current_dir = par2_file
+ .parent()
+ .map(Path::to_path_buf)
+ .unwrap_or_else(|| manifest_dir());
+
+ Invocation {
+ program,
+ args,
+ current_dir,
+ }
+}
+
+fn create_fixture(
+ case_name: &'static str,
+ protected_bytes: usize,
+ block_size: u64,
+) -> VerifyFixture {
+ let dir = target_dir()
+ .join("bench-fixtures")
+ .join("par2verify")
+ .join(case_name);
+ let source_file = dir.join("payload.bin");
+ let par2_file = dir.join("payload.par2");
+
+ if !source_file.is_file() || !par2_file.is_file() {
+ if dir.exists() {
+ fs::remove_dir_all(&dir).expect("failed to clear stale benchmark fixture directory");
+ }
+ fs::create_dir_all(&dir).expect("failed to create benchmark fixture directory");
+ write_payload_file(&source_file, protected_bytes)
+ .expect("failed to create benchmark payload");
+
+ let reporter = Box::new(SilentCreateReporter);
+ let mut context = CreateContextBuilder::new()
+ .output_name(par2_file.to_string_lossy().into_owned())
+ .source_files(vec![source_file.clone()])
+ .block_size(block_size)
+ .redundancy_percentage(10)
+ .overwrite_existing(true)
+ .reporter(reporter)
+ .build()
+ .expect("failed to build benchmark create context");
+
+ context
+ .create()
+ .expect("failed to create benchmark PAR2 fixture");
+ }
+
+ VerifyFixture {
+ case_name,
+ dir,
+ par2_file,
+ protected_bytes: protected_bytes as u64,
+ }
+}
+
+fn write_payload_file(path: &Path, total_bytes: usize) -> io::Result<()> {
+ let file = File::create(path)?;
+ let mut writer = BufWriter::new(file);
+ let mut state = 0x1234_5678u32;
+ let mut chunk = vec![0u8; DATA_CHUNK_BYTES];
+ let mut remaining = total_bytes;
+
+ while remaining > 0 {
+ let take = remaining.min(chunk.len());
+ for byte in &mut chunk[..take] {
+ state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
+ *byte = (state >> 16) as u8;
+ }
+ writer.write_all(&chunk[..take])?;
+ remaining -= take;
+ }
+
+ writer.flush()
+}
+
+fn resolve_par2rs_verify() -> &'static PathBuf {
+ PAR2RS_VERIFY.get_or_init(|| {
+ if let Some(path) = option_env!("CARGO_BIN_EXE_par2verify") {
+ let path = PathBuf::from(path);
+ if path.is_file() {
+ return path;
+ }
+ }
+
+ for candidate in [
+ target_dir().join("release").join("par2verify"),
+ target_dir().join("debug").join("par2verify"),
+ ] {
+ if candidate.is_file() {
+ return candidate;
+ }
+ }
+
+ panic!(
+ "could not resolve par2rs par2verify binary; run cargo build --release --bin par2verify or cargo bench from a Cargo-built environment"
+ );
+ })
+}
+
+fn resolve_turbo_verify() -> &'static TurboCommand {
+ TURBO_VERIFY.get_or_init(|| {
+ let turbo_root = env::var_os("TURBO_ROOT")
+ .map(PathBuf::from)
+ .unwrap_or_else(|| PathBuf::from(DEFAULT_TURBO_ROOT));
+
+ if let Some(path) = env::var_os("TURBO_PAR2VERIFY")
+ .map(PathBuf::from)
+ .filter(|path| path.is_file())
+ {
+ return TurboCommand {
+ program: path,
+ use_verify_subcommand: false,
+ };
+ }
+
+ if let Some(path) = find_in_path("par2verify") {
+ return TurboCommand {
+ program: path,
+ use_verify_subcommand: false,
+ };
+ }
+
+ if let Some(path) = env::var_os("TURBO_PAR2")
+ .map(PathBuf::from)
+ .filter(|path| path.is_file())
+ {
+ let wrapper = path
+ .parent()
+ .map(|dir| dir.join("par2verify"))
+ .filter(|wrapper| wrapper.is_file());
+ if let Some(wrapper) = wrapper {
+ return TurboCommand {
+ program: wrapper,
+ use_verify_subcommand: false,
+ };
+ }
+ return TurboCommand {
+ program: path,
+ use_verify_subcommand: true,
+ };
+ }
+
+ let root_wrapper = turbo_root.join("par2verify");
+ if root_wrapper.is_file() {
+ return TurboCommand {
+ program: root_wrapper,
+ use_verify_subcommand: false,
+ };
+ }
+
+ let root_par2 = turbo_root.join("par2");
+ if root_par2.is_file() {
+ return TurboCommand {
+ program: root_par2,
+ use_verify_subcommand: true,
+ };
+ }
+
+ if let Some(path) = find_in_path("par2") {
+ return TurboCommand {
+ program: path,
+ use_verify_subcommand: true,
+ };
+ }
+
+ panic!(
+ "could not resolve par2cmdline-turbo verify command; run inside nix develop or set TURBO_PAR2VERIFY/TURBO_PAR2"
+ );
+ })
+}
+
+fn find_in_path(binary: &str) -> Option {
+ let paths = env::var_os("PATH")?;
+ env::split_paths(&paths)
+ .map(|dir| dir.join(binary))
+ .find(|candidate| candidate.is_file())
+}
+
+fn manifest_dir() -> PathBuf {
+ PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+}
+
+fn target_dir() -> PathBuf {
+ match env::var_os("CARGO_TARGET_DIR") {
+ Some(path) => {
+ let path = PathBuf::from(path);
+ if path.is_absolute() {
+ path
+ } else {
+ manifest_dir().join(path)
+ }
+ }
+ None => manifest_dir().join("target"),
+ }
+}
diff --git a/benches/compare_with_parpar.rs b/benches/compare_with_parpar.rs
new file mode 100644
index 00000000..c118b1ac
--- /dev/null
+++ b/benches/compare_with_parpar.rs
@@ -0,0 +1,182 @@
+//! Wall-clock comparison between par2rs and embedded ParPar hasher sources.
+
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
+use std::hint::black_box;
+
+fn make_data(len: usize) -> Vec {
+ (0..len)
+ .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7))
+ .collect()
+}
+
+#[cfg(target_arch = "x86_64")]
+fn par2rs_md5_scalar(data: &[u8]) -> [u8; 16] {
+ use par2rs::parpar_hasher::hasher_input::HasherInput;
+ use par2rs::parpar_hasher::md5x2_scalar::Scalar;
+
+ let mut hasher = HasherInput::::new();
+ hasher.update(data);
+ hasher.end()
+}
+
+#[cfg(target_arch = "x86_64")]
+fn par2rs_md5_sse2(data: &[u8]) -> [u8; 16] {
+ use par2rs::parpar_hasher::hasher_input::HasherInput;
+ use par2rs::parpar_hasher::md5x2_sse2::Sse2;
+
+ let mut hasher = HasherInput::::new();
+ hasher.update(data);
+ hasher.end()
+}
+
+#[cfg(target_arch = "x86_64")]
+fn par2rs_md5_bmi1(data: &[u8]) -> [u8; 16] {
+ use par2rs::parpar_hasher::hasher_input::HasherInput;
+ use par2rs::parpar_hasher::md5x2_bmi1::Bmi1;
+
+ let mut hasher = HasherInput::::new();
+ hasher.update(data);
+ hasher.end()
+}
+
+#[cfg(target_arch = "x86_64")]
+fn par2rs_md5_avx512(data: &[u8]) -> Option<[u8; 16]> {
+ if !is_x86_feature_detected!("avx512f")
+ || !is_x86_feature_detected!("avx512vl")
+ || !is_x86_feature_detected!("avx512bw")
+ {
+ return None;
+ }
+
+ use par2rs::parpar_hasher::hasher_input::HasherInput;
+ use par2rs::parpar_hasher::md5x2_avx512::Avx512;
+
+ let mut hasher = HasherInput::::new();
+ hasher.update(data);
+ Some(hasher.end())
+}
+
+fn par2rs_crc32_fast(data: &[u8]) -> u32 {
+ let mut hasher = crc32fast::Hasher::new();
+ hasher.update(data);
+ hasher.finalize()
+}
+
+#[cfg(target_arch = "x86_64")]
+fn avx512_runtime_available() -> bool {
+ is_x86_feature_detected!("avx512f")
+ && is_x86_feature_detected!("avx512vl")
+ && is_x86_feature_detected!("avx512bw")
+}
+
+#[cfg(all(
+ feature = "parpar-compare",
+ target_arch = "x86_64",
+ parpar_compare_embedded
+))]
+fn parpar_hasher_input(data: &[u8], method: par2rs::ffi::HasherInputMethod) -> [u8; 16] {
+ use par2rs::ffi::hasher_input::ParParHasherInput;
+
+ if !method.is_available() {
+ panic!("ParPar method {:?} is unavailable", method);
+ }
+ let mut hasher = ParParHasherInput::new(method).expect("ParPar hasher unavailable");
+ hasher.update(data);
+ *hasher.finalize().as_bytes()
+}
+
+#[cfg(all(
+ feature = "parpar-compare",
+ target_arch = "x86_64",
+ parpar_compare_embedded
+))]
+fn parpar_crc32(data: &[u8]) -> u32 {
+ par2rs::ffi::crc32::crc32_compute(data)
+}
+
+#[cfg(target_arch = "x86_64")]
+fn bench_md5_variants(c: &mut Criterion) {
+ let sizes = [(16 * 1024, "16k"), (4 * 1024 * 1024, "4m")];
+
+ for (size, label) in &sizes {
+ let data = black_box(make_data(*size));
+ let mut group = c.benchmark_group(format!("md5_variants_{label}"));
+ group.throughput(Throughput::Bytes(*size as u64));
+
+ let rust_variants: &[(&str, fn(&[u8]) -> [u8; 16])] = &[
+ ("par2rs_scalar", par2rs_md5_scalar),
+ ("par2rs_sse2", par2rs_md5_sse2),
+ ("par2rs_bmi1", par2rs_md5_bmi1),
+ ];
+ for (name, func) in rust_variants {
+ group.bench_with_input(BenchmarkId::new(*name, label), &data, |b, data| {
+ b.iter(|| func(black_box(data)))
+ });
+ }
+ if avx512_runtime_available() && par2rs_md5_avx512(&data).is_some() {
+ group.bench_with_input(
+ BenchmarkId::new("par2rs_avx512", label),
+ &data,
+ |b, data| b.iter(|| par2rs_md5_avx512(black_box(data)).unwrap()),
+ );
+ }
+
+ #[cfg(all(
+ feature = "parpar-compare",
+ target_arch = "x86_64",
+ parpar_compare_embedded
+ ))]
+ {
+ let parpar_variants: &[(&str, par2rs::ffi::HasherInputMethod)] = &[
+ ("parpar_scalar", par2rs::ffi::HasherInputMethod::Scalar),
+ ("parpar_simd", par2rs::ffi::HasherInputMethod::Simd),
+ ("parpar_crc", par2rs::ffi::HasherInputMethod::Crc),
+ ("parpar_simd_crc", par2rs::ffi::HasherInputMethod::SimdCrc),
+ ("parpar_bmi1", par2rs::ffi::HasherInputMethod::Bmi1),
+ ("parpar_avx512", par2rs::ffi::HasherInputMethod::Avx512),
+ ];
+
+ for (name, method) in parpar_variants {
+ if method.is_available()
+ && (method != &par2rs::ffi::HasherInputMethod::Avx512
+ || avx512_runtime_available())
+ {
+ group.bench_with_input(BenchmarkId::new(*name, label), &data, |b, data| {
+ b.iter(|| parpar_hasher_input(black_box(data), *method))
+ });
+ }
+ }
+ }
+
+ group.finish();
+ }
+}
+
+fn bench_crc32_variants(c: &mut Criterion) {
+ let data = black_box(make_data(4 * 1024 * 1024));
+ let mut group = c.benchmark_group("crc32_4m");
+ group.throughput(Throughput::Bytes(data.len() as u64));
+
+ group.bench_function("par2rs_crc32fast", |b| {
+ b.iter(|| par2rs_crc32_fast(black_box(&data)))
+ });
+
+ #[cfg(all(
+ feature = "parpar-compare",
+ target_arch = "x86_64",
+ parpar_compare_embedded
+ ))]
+ group.bench_function("parpar_crc32", |b| {
+ b.iter(|| parpar_crc32(black_box(&data)))
+ });
+
+ group.finish();
+}
+
+#[cfg(target_arch = "x86_64")]
+criterion_group!(benches, bench_md5_variants, bench_crc32_variants);
+
+#[cfg(not(target_arch = "x86_64"))]
+criterion_group!(benches, bench_crc32_variants);
+
+criterion_main!(benches);
diff --git a/benches/crc_compare.rs b/benches/crc_compare.rs
new file mode 100644
index 00000000..c3538bcd
--- /dev/null
+++ b/benches/crc_compare.rs
@@ -0,0 +1,96 @@
+//! Bench: `crc32fast` vs `crc-fast` over realistic par2 block sizes.
+//!
+//! Decision driver for whether to swap the CRC backend. We care about:
+//! - 64 B: the fused HasherInput inner-loop granularity.
+//! - 16 KiB: the Tier-1 sub-slice size used by `update_md5_crc32_fused`.
+//! - 4 MiB & 64 MiB: full-buffer one-shots representative of large par2 blocks.
+//!
+//! Both crates produce IsoHdlc/zlib CRC-32. Each bench also asserts equality
+//! on the first iteration so we don't ship a result that doesn't match.
+
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
+use std::hint::black_box;
+
+fn make_data(len: usize) -> Vec {
+ // Deterministic non-zero pattern so optimizers can't constant-fold.
+ (0..len)
+ .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7))
+ .collect()
+}
+
+fn crc32fast_one_shot(data: &[u8]) -> u32 {
+ let mut h = crc32fast::Hasher::new();
+ h.update(data);
+ h.finalize()
+}
+
+fn crc_fast_one_shot(data: &[u8]) -> u32 {
+ use crc_fast::{CrcAlgorithm, Digest};
+ let mut d = Digest::new(CrcAlgorithm::Crc32IsoHdlc);
+ d.update(data);
+ d.finalize() as u32
+}
+
+fn bench_oneshot(c: &mut Criterion) {
+ let sizes = [64usize, 16 * 1024, 4 * 1024 * 1024, 64 * 1024 * 1024];
+ let mut group = c.benchmark_group("crc_oneshot");
+
+ for &len in &sizes {
+ let data = make_data(len);
+
+ // Sanity: both backends must agree.
+ assert_eq!(
+ crc32fast_one_shot(&data),
+ crc_fast_one_shot(&data),
+ "crc32fast and crc-fast disagree at len={len}"
+ );
+
+ group.throughput(Throughput::Bytes(len as u64));
+
+ group.bench_with_input(BenchmarkId::new("crc32fast", len), &data, |b, d| {
+ b.iter(|| black_box(crc32fast_one_shot(black_box(d))));
+ });
+ group.bench_with_input(BenchmarkId::new("crc-fast", len), &data, |b, d| {
+ b.iter(|| black_box(crc_fast_one_shot(black_box(d))));
+ });
+ }
+
+ group.finish();
+}
+
+/// Streaming bench: feed 64 B at a time. This is the actual fused-HasherInput
+/// access pattern — what matters for the T2.c port decision.
+fn bench_streaming_64b(c: &mut Criterion) {
+ let total_sizes = [16 * 1024usize, 4 * 1024 * 1024];
+ let mut group = c.benchmark_group("crc_stream_64B");
+
+ for &total in &total_sizes {
+ let data = make_data(total);
+ group.throughput(Throughput::Bytes(total as u64));
+
+ group.bench_with_input(BenchmarkId::new("crc32fast", total), &data, |b, d| {
+ b.iter(|| {
+ let mut h = crc32fast::Hasher::new();
+ for chunk in d.chunks_exact(64) {
+ h.update(black_box(chunk));
+ }
+ black_box(h.finalize())
+ });
+ });
+ group.bench_with_input(BenchmarkId::new("crc-fast", total), &data, |b, d| {
+ b.iter(|| {
+ use crc_fast::{CrcAlgorithm, Digest};
+ let mut h = Digest::new(CrcAlgorithm::Crc32IsoHdlc);
+ for chunk in d.chunks_exact(64) {
+ h.update(black_box(chunk));
+ }
+ black_box(h.finalize() as u32)
+ });
+ });
+ }
+
+ group.finish();
+}
+
+criterion_group!(benches, bench_oneshot, bench_streaming_64b);
+criterion_main!(benches);
diff --git a/benches/create_benchmark.rs b/benches/create_benchmark.rs
index 0b4aaa9b..26a97919 100644
--- a/benches/create_benchmark.rs
+++ b/benches/create_benchmark.rs
@@ -71,7 +71,9 @@ fn bench_par2_creation_redundancy(c: &mut Criterion) {
for redundancy in redundancy_levels {
group.bench_with_input(
- BenchmarkId::new("redundancy", format!("{}%", redundancy)),
+ // Avoid `%` in the display label; Criterion forwards benchmark IDs into
+ // plot generation, and gnuplot treats `%` as a format character.
+ BenchmarkId::new("redundancy", format!("{}pct", redundancy)),
&redundancy,
|b, &redundancy| {
b.iter_batched(
diff --git a/benches/fused_hashing.rs b/benches/fused_hashing.rs
new file mode 100644
index 00000000..38f7d693
--- /dev/null
+++ b/benches/fused_hashing.rs
@@ -0,0 +1,123 @@
+//! Microbenchmark for the Tier 1 fused-hashing helpers in `src/checksum.rs`.
+//!
+//! Compares four strategies over the same input buffer:
+//!
+//! * `naive_2pass` — `Md5::update(whole)` then `Crc32::update(whole)`.
+//! This is what `par2cmdline-turbo` would call the "double scan" baseline:
+//! for any buffer larger than L1d, the second pass eats cold cache lines.
+//!
+//! * `fused_2hash` — `compute_md5_crc32_simultaneous`, which walks the
+//! buffer in 16 KiB sub-slices feeding both hashers back-to-back so the
+//! CRC reads what MD5 just brought into L1.
+//!
+//! * `naive_3pass` — file-MD5 + block-MD5 + CRC32 each as full-buffer
+//! passes. Mirrors the pre-Tier-1 create() loop where the chunk was
+//! scanned three times.
+//!
+//! * `fused_3hash` — `update_file_md5_block_md5_crc32_fused`, the
+//! three-hasher variant that backs `encode_and_hash_files` post-Tier-1.
+//!
+//! The interesting buffer sizes are the ones that span L1 / L2 / L3 / DRAM:
+//! 16 KiB sits exactly in L1, 256 KiB falls out of L1, 4 MiB falls out of L2,
+//! 64 MiB falls out of L3 on most desktops. The fused helpers should shine
+//! once the buffer is bigger than L1.
+//!
+//! For the cache-miss number itself, run under `perf stat`:
+//!
+//! ```text
+//! perf stat -e cache-misses,cache-references,L1-dcache-load-misses,\
+//! LLC-load-misses,instructions,cycles,task-clock \
+//! cargo bench --bench fused_hashing -- --profile-time 5 fused_2hash/4MiB
+//! ```
+
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
+use std::hint::black_box;
+
+use crc32fast::Hasher as Crc32Hasher;
+use md5::{Digest, Md5};
+
+use par2rs::checksum::{compute_md5_crc32_simultaneous, update_file_md5_block_md5_crc32_fused};
+
+/// Buffer sizes, chosen to span L1 / L2 / L3 / DRAM on a typical desktop.
+const SIZES: &[(&str, usize)] = &[
+ ("16KiB", 16 * 1024), // fits in L1d (~32-48 KiB)
+ ("256KiB", 256 * 1024), // fits in L2 (~256 KiB-1 MiB)
+ ("4MiB", 4 * 1024 * 1024), // fits in L3
+ ("64MiB", 64 * 1024 * 1024), // exceeds typical L3
+];
+
+fn make_buffer(size: usize) -> Vec {
+ // Repeating low-entropy pattern is fine: we're measuring memory traffic +
+ // hash compress throughput, not anything data-dependent.
+ (0..size).map(|i| (i & 0xff) as u8).collect()
+}
+
+fn bench_two_hashes(c: &mut Criterion) {
+ let mut group = c.benchmark_group("two_hashes_md5_crc32");
+
+ for &(label, size) in SIZES {
+ let data = make_buffer(size);
+ group.throughput(Throughput::Bytes(size as u64));
+
+ // Pre-Tier-1 baseline: two full-buffer scans.
+ group.bench_with_input(BenchmarkId::new("naive_2pass", label), &data, |b, data| {
+ b.iter(|| {
+ let mut md5 = Md5::new();
+ let mut crc = Crc32Hasher::new();
+ md5.update(black_box(data));
+ crc.update(black_box(data));
+ black_box((md5.finalize(), crc.finalize()))
+ });
+ });
+
+ // Tier 1: cache-resident sub-slice walker.
+ group.bench_with_input(BenchmarkId::new("fused_2hash", label), &data, |b, data| {
+ b.iter(|| black_box(compute_md5_crc32_simultaneous(black_box(data))));
+ });
+ }
+
+ group.finish();
+}
+
+fn bench_three_hashes(c: &mut Criterion) {
+ let mut group = c.benchmark_group("three_hashes_file_block_crc");
+
+ for &(label, size) in SIZES {
+ let data = make_buffer(size);
+ group.throughput(Throughput::Bytes(size as u64));
+
+ // Pre-Tier-1: three full-buffer scans (the original create() loop).
+ group.bench_with_input(BenchmarkId::new("naive_3pass", label), &data, |b, data| {
+ b.iter(|| {
+ let mut file_md5 = Md5::new();
+ let mut block_md5 = Md5::new();
+ let mut crc = Crc32Hasher::new();
+ file_md5.update(black_box(data));
+ block_md5.update(black_box(data));
+ crc.update(black_box(data));
+ black_box((file_md5.finalize(), block_md5.finalize(), crc.finalize()))
+ });
+ });
+
+ // Tier 1: cache-resident three-hasher walker.
+ group.bench_with_input(BenchmarkId::new("fused_3hash", label), &data, |b, data| {
+ b.iter(|| {
+ let mut file_md5 = Md5::new();
+ let mut block_md5 = Md5::new();
+ let mut crc = Crc32Hasher::new();
+ update_file_md5_block_md5_crc32_fused(
+ &mut file_md5,
+ &mut block_md5,
+ &mut crc,
+ black_box(data),
+ );
+ black_box((file_md5.finalize(), block_md5.finalize(), crc.finalize()))
+ });
+ });
+ }
+
+ group.finish();
+}
+
+criterion_group!(benches, bench_two_hashes, bench_three_hashes);
+criterion_main!(benches);
diff --git a/benches/iai_hasher_input.rs b/benches/iai_hasher_input.rs
new file mode 100644
index 00000000..36946e37
--- /dev/null
+++ b/benches/iai_hasher_input.rs
@@ -0,0 +1,148 @@
+//! iai-callgrind instruction-count benchmark for the ParPar HasherInput
+//! port (T2.c). Complement to `parpar_hasher_input.rs` (criterion
+//! wall-clock): this one measures deterministic instruction / branch /
+//! cache counts so regressions show up without timing noise.
+//!
+//! Compares the same three implementations:
+//! 1. naive 3-pass (md-5 + crc32fast)
+//! 2. tier-1 helper (`update_file_md5_block_md5_crc32_fused`)
+//! 3. HasherInput (the new ParPar fused 64 B driver)
+//!
+//! at two single-block sizes (16 KiB sub-slice, 4 MiB block).
+
+use iai_callgrind::{library_benchmark, library_benchmark_group, main};
+use md5::Digest as _;
+use md5::Md5;
+use par2rs::checksum::update_file_md5_block_md5_crc32_fused;
+#[cfg(target_arch = "x86_64")]
+use par2rs::parpar_hasher::hasher_input::HasherInput;
+#[cfg(target_arch = "x86_64")]
+use par2rs::parpar_hasher::md5x2::Md5x2;
+#[cfg(target_arch = "x86_64")]
+use par2rs::parpar_hasher::md5x2_scalar::Scalar;
+#[cfg(target_arch = "x86_64")]
+use par2rs::parpar_hasher::md5x2_sse2::Sse2;
+use std::hint::black_box;
+
+type Out = ([u8; 16], [u8; 16], u32);
+
+fn make_data(len: usize) -> Vec {
+ (0..len)
+ .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7))
+ .collect()
+}
+
+// Naive 3-pass: file-MD5 + block-MD5 + block-CRC, three independent walks.
+fn naive_seq(data: &[u8]) -> Out {
+ let mut f = Md5::new();
+ f.update(data);
+ let mut b = Md5::new();
+ b.update(data);
+ let mut c = crc32fast::Hasher::new();
+ c.update(data);
+ (f.finalize().into(), b.finalize().into(), c.finalize())
+}
+
+// Tier-1 helper: one walk, three single-lane hashers.
+fn tier1_helper(data: &[u8]) -> Out {
+ let mut f = Md5::new();
+ let mut b = Md5::new();
+ let mut c = crc32fast::Hasher::new();
+ update_file_md5_block_md5_crc32_fused(&mut f, &mut b, &mut c, data);
+ (f.finalize().into(), b.finalize().into(), c.finalize())
+}
+
+// HasherInput: MD5x2 + CLMul CRC32 fused at 64 B granularity.
+// Single-block scenario: one update + one get_block(0) + one end().
+#[cfg(target_arch = "x86_64")]
+fn hasher_input(data: &[u8]) -> Out {
+ let mut h: HasherInput = HasherInput::new();
+ h.update(data);
+ let bh = h.get_block(0);
+ let file = h.end();
+ (file, bh.md5, bh.crc32)
+}
+
+// ----------------------- 16 KiB single block -----------------------
+
+#[library_benchmark]
+#[bench::sixteen_kib(make_data(16 * 1024))]
+fn bench_naive_seq_16k(data: Vec) -> Out {
+ black_box(naive_seq(black_box(&data)))
+}
+
+#[library_benchmark]
+#[bench::sixteen_kib(make_data(16 * 1024))]
+fn bench_tier1_helper_16k(data: Vec) -> Out {
+ black_box(tier1_helper(black_box(&data)))
+}
+
+#[cfg(target_arch = "x86_64")]
+#[library_benchmark]
+#[bench::sixteen_kib(make_data(16 * 1024))]
+fn bench_hasher_input_scalar_16k(data: Vec) -> Out {
+ black_box(hasher_input::(black_box(&data)))
+}
+
+#[cfg(target_arch = "x86_64")]
+#[library_benchmark]
+#[bench::sixteen_kib(make_data(16 * 1024))]
+fn bench_hasher_input_sse2_16k(data: Vec) -> Out {
+ black_box(hasher_input::(black_box(&data)))
+}
+
+// ----------------------- 4 MiB single block ------------------------
+
+#[library_benchmark]
+#[bench::four_mib(make_data(4 * 1024 * 1024))]
+fn bench_naive_seq_4m(data: Vec) -> Out {
+ black_box(naive_seq(black_box(&data)))
+}
+
+#[library_benchmark]
+#[bench::four_mib(make_data(4 * 1024 * 1024))]
+fn bench_tier1_helper_4m(data: Vec) -> Out {
+ black_box(tier1_helper(black_box(&data)))
+}
+
+#[cfg(target_arch = "x86_64")]
+#[library_benchmark]
+#[bench::four_mib(make_data(4 * 1024 * 1024))]
+fn bench_hasher_input_scalar_4m(data: Vec) -> Out {
+ black_box(hasher_input::(black_box(&data)))
+}
+
+#[cfg(target_arch = "x86_64")]
+#[library_benchmark]
+#[bench::four_mib(make_data(4 * 1024 * 1024))]
+fn bench_hasher_input_sse2_4m(data: Vec) -> Out {
+ black_box(hasher_input::(black_box(&data)))
+}
+
+// ----------------------- groups + main -----------------------------
+
+#[cfg(target_arch = "x86_64")]
+library_benchmark_group!(
+ name = hasher_input_group;
+ benchmarks =
+ bench_naive_seq_16k,
+ bench_tier1_helper_16k,
+ bench_hasher_input_scalar_16k,
+ bench_hasher_input_sse2_16k,
+ bench_naive_seq_4m,
+ bench_tier1_helper_4m,
+ bench_hasher_input_scalar_4m,
+ bench_hasher_input_sse2_4m
+);
+
+#[cfg(not(target_arch = "x86_64"))]
+library_benchmark_group!(
+ name = hasher_input_group;
+ benchmarks =
+ bench_naive_seq_16k,
+ bench_tier1_helper_16k,
+ bench_naive_seq_4m,
+ bench_tier1_helper_4m
+);
+
+main!(library_benchmark_groups = hasher_input_group);
diff --git a/benches/iai_par2verify_compare.rs b/benches/iai_par2verify_compare.rs
new file mode 100644
index 00000000..3789e3fe
--- /dev/null
+++ b/benches/iai_par2verify_compare.rs
@@ -0,0 +1,36 @@
+use iai_callgrind::{binary_benchmark, binary_benchmark_group, main, Command};
+
+#[path = "common/par2verify_fixture.rs"]
+mod par2verify_fixture;
+
+fn into_iai_command(invocation: par2verify_fixture::Invocation) -> Command {
+ let mut command = Command::new(invocation.program);
+ command.current_dir(invocation.current_dir);
+ for arg in invocation.args {
+ command.arg(arg);
+ }
+ command.build()
+}
+
+#[binary_benchmark]
+#[bench::intact_8m(par2verify_fixture::iai_fixture_arg())]
+fn bench_par2rs_verify(par2_file: String) -> Command {
+ into_iai_command(par2verify_fixture::par2rs_verify_invocation_for_path(
+ par2_file,
+ ))
+}
+
+#[binary_benchmark]
+#[bench::intact_8m(par2verify_fixture::iai_fixture_arg())]
+fn bench_turbo_verify(par2_file: String) -> Command {
+ into_iai_command(par2verify_fixture::turbo_verify_invocation_for_path(
+ par2_file,
+ ))
+}
+
+binary_benchmark_group!(
+ name = par2verify_group;
+ benchmarks = bench_par2rs_verify, bench_turbo_verify
+);
+
+main!(binary_benchmark_groups = par2verify_group);
diff --git a/benches/iai_parpar_comparison.rs b/benches/iai_parpar_comparison.rs
new file mode 100644
index 00000000..02aa3f03
--- /dev/null
+++ b/benches/iai_parpar_comparison.rs
@@ -0,0 +1,212 @@
+//! Instruction-count comparison for par2rs vs embedded ParPar hasher sources.
+
+use iai_callgrind::{library_benchmark, library_benchmark_group, main};
+use std::hint::black_box;
+
+fn make_data(len: usize) -> Vec {
+ (0..len)
+ .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7))
+ .collect()
+}
+
+#[cfg(target_arch = "x86_64")]
+fn par2rs_md5_scalar(data: &[u8]) -> [u8; 16] {
+ use par2rs::parpar_hasher::hasher_input::HasherInput;
+ use par2rs::parpar_hasher::md5x2_scalar::Scalar;
+
+ let mut hasher = HasherInput::::new();
+ hasher.update(data);
+ hasher.end()
+}
+
+#[cfg(target_arch = "x86_64")]
+fn par2rs_md5_sse2(data: &[u8]) -> [u8; 16] {
+ use par2rs::parpar_hasher::hasher_input::HasherInput;
+ use par2rs::parpar_hasher::md5x2_sse2::Sse2;
+
+ let mut hasher = HasherInput::::new();
+ hasher.update(data);
+ hasher.end()
+}
+
+#[cfg(target_arch = "x86_64")]
+fn par2rs_md5_bmi1(data: &[u8]) -> [u8; 16] {
+ use par2rs::parpar_hasher::hasher_input::HasherInput;
+ use par2rs::parpar_hasher::md5x2_bmi1::Bmi1;
+
+ let mut hasher = HasherInput::::new();
+ hasher.update(data);
+ hasher.end()
+}
+
+fn par2rs_crc32(data: &[u8]) -> u32 {
+ let mut hasher = crc32fast::Hasher::new();
+ hasher.update(data);
+ hasher.finalize()
+}
+
+#[cfg(all(
+ feature = "parpar-compare",
+ target_arch = "x86_64",
+ parpar_compare_embedded
+))]
+fn parpar_hasher_input(data: &[u8], method: par2rs::ffi::HasherInputMethod) -> [u8; 16] {
+ use par2rs::ffi::hasher_input::ParParHasherInput;
+
+ let mut hasher = ParParHasherInput::new(method).expect("ParPar hasher unavailable");
+ hasher.update(data);
+ *hasher.finalize().as_bytes()
+}
+
+#[cfg(all(
+ feature = "parpar-compare",
+ target_arch = "x86_64",
+ parpar_compare_embedded
+))]
+fn parpar_crc32(data: &[u8]) -> u32 {
+ par2rs::ffi::crc32::crc32_compute(data)
+}
+
+#[cfg(target_arch = "x86_64")]
+#[library_benchmark]
+#[bench::par2rs_scalar_16k(make_data(16 * 1024))]
+fn bench_par2rs_scalar_16k(data: Vec) -> [u8; 16] {
+ black_box(par2rs_md5_scalar(black_box(&data)))
+}
+
+#[cfg(target_arch = "x86_64")]
+#[library_benchmark]
+#[bench::par2rs_sse2_16k(make_data(16 * 1024))]
+fn bench_par2rs_sse2_16k(data: Vec) -> [u8; 16] {
+ black_box(par2rs_md5_sse2(black_box(&data)))
+}
+
+#[cfg(target_arch = "x86_64")]
+#[library_benchmark]
+#[bench::par2rs_bmi1_16k(make_data(16 * 1024))]
+fn bench_par2rs_bmi1_16k(data: Vec) -> [u8; 16] {
+ black_box(par2rs_md5_bmi1(black_box(&data)))
+}
+
+#[cfg(all(
+ feature = "parpar-compare",
+ target_arch = "x86_64",
+ parpar_compare_embedded
+))]
+#[library_benchmark]
+#[bench::parpar_scalar_16k(make_data(16 * 1024))]
+fn bench_parpar_scalar_16k(data: Vec) -> [u8; 16] {
+ black_box(parpar_hasher_input(
+ black_box(&data),
+ par2rs::ffi::HasherInputMethod::Scalar,
+ ))
+}
+
+#[cfg(all(
+ feature = "parpar-compare",
+ target_arch = "x86_64",
+ parpar_compare_embedded
+))]
+#[library_benchmark]
+#[bench::parpar_sse_16k(make_data(16 * 1024))]
+fn bench_parpar_sse_16k(data: Vec) -> [u8; 16] {
+ black_box(parpar_hasher_input(
+ black_box(&data),
+ par2rs::ffi::HasherInputMethod::Simd,
+ ))
+}
+
+#[cfg(all(
+ feature = "parpar-compare",
+ target_arch = "x86_64",
+ parpar_compare_embedded
+))]
+#[library_benchmark]
+#[bench::parpar_bmi1_16k(make_data(16 * 1024))]
+fn bench_parpar_bmi1_16k(data: Vec) -> [u8; 16] {
+ black_box(parpar_hasher_input(
+ black_box(&data),
+ par2rs::ffi::HasherInputMethod::Bmi1,
+ ))
+}
+
+#[library_benchmark]
+#[bench::crc32_1m(make_data(1024 * 1024))]
+fn bench_crc32_1m(data: Vec) -> u32 {
+ black_box(par2rs_crc32(black_box(&data)))
+}
+
+#[cfg(all(
+ feature = "parpar-compare",
+ target_arch = "x86_64",
+ parpar_compare_embedded
+))]
+#[library_benchmark]
+#[bench::parpar_crc32_1m(make_data(1024 * 1024))]
+fn bench_parpar_crc32_1m(data: Vec) -> u32 {
+ black_box(parpar_crc32(black_box(&data)))
+}
+
+#[cfg(target_arch = "x86_64")]
+library_benchmark_group!(
+ name = md5_group;
+ benchmarks =
+ bench_par2rs_scalar_16k,
+ bench_par2rs_sse2_16k,
+ bench_par2rs_bmi1_16k
+);
+
+#[cfg(all(
+ feature = "parpar-compare",
+ target_arch = "x86_64",
+ parpar_compare_embedded
+))]
+library_benchmark_group!(
+ name = md5_parpar_group;
+ benchmarks =
+ bench_parpar_scalar_16k,
+ bench_parpar_sse_16k,
+ bench_parpar_bmi1_16k
+);
+
+library_benchmark_group!(
+ name = crc32_group;
+ benchmarks =
+ bench_crc32_1m
+);
+
+#[cfg(all(
+ feature = "parpar-compare",
+ target_arch = "x86_64",
+ parpar_compare_embedded
+))]
+library_benchmark_group!(
+ name = crc32_parpar_group;
+ benchmarks =
+ bench_parpar_crc32_1m
+);
+
+#[cfg(all(
+ feature = "parpar-compare",
+ target_arch = "x86_64",
+ parpar_compare_embedded
+))]
+main!(
+ library_benchmark_groups = md5_group,
+ md5_parpar_group,
+ crc32_group,
+ crc32_parpar_group
+);
+
+#[cfg(any(
+ all(not(feature = "parpar-compare"), target_arch = "x86_64"),
+ all(
+ feature = "parpar-compare",
+ target_arch = "x86_64",
+ not(parpar_compare_embedded)
+ )
+))]
+main!(library_benchmark_groups = md5_group, crc32_group);
+
+#[cfg(not(target_arch = "x86_64"))]
+main!(library_benchmark_groups = crc32_group);
diff --git a/benches/iai_simd.rs b/benches/iai_simd.rs
index a148bdf0..eec732a7 100644
--- a/benches/iai_simd.rs
+++ b/benches/iai_simd.rs
@@ -1,11 +1,50 @@
use iai_callgrind::{library_benchmark, library_benchmark_group, main};
+use par2rs::reed_solomon::alloc_aligned_vec;
use par2rs::reed_solomon::codec::build_split_mul_table;
use par2rs::reed_solomon::galois::Galois16;
use par2rs::reed_solomon::simd::process_slice_multiply_add_portable_simd;
#[cfg(target_arch = "x86_64")]
+use par2rs::reed_solomon::simd::{
+ prepare_xor_jit_bitplane_chunks, XorJitBitplaneKernel, XorJitPreparedCoeff,
+};
+#[cfg(target_arch = "x86_64")]
use par2rs::reed_solomon::simd::{process_slice_multiply_add_simd, SimdLevel};
use std::hint::black_box;
+#[cfg(target_arch = "x86_64")]
+const BITPLANE_BLOCK_BYTES: usize = 512;
+
+#[cfg(target_arch = "x86_64")]
+struct BitplaneFixture {
+ inputs: Vec>,
+ output: Vec,
+ kernels: Vec,
+}
+
+#[cfg(target_arch = "x86_64")]
+fn bitplane_fixture(batch_len: usize, segment_len: usize) -> BitplaneFixture {
+ let mut inputs = Vec::with_capacity(batch_len);
+ let mut kernels = Vec::with_capacity(batch_len);
+
+ for input_idx in 0..batch_len {
+ let source = (0..segment_len)
+ .map(|byte_idx| (input_idx * 31 + byte_idx * 17 + byte_idx / 7) as u8)
+ .collect::>();
+ let mut prepared = alloc_aligned_vec(segment_len.next_multiple_of(BITPLANE_BLOCK_BYTES));
+ prepare_xor_jit_bitplane_chunks(&mut prepared, &source);
+ inputs.push(prepared);
+
+ let coeff = XorJitPreparedCoeff::new((0x100b + input_idx * 37) as u16);
+ kernels.push(XorJitBitplaneKernel::new(&coeff).expect("bitplane kernel"));
+ }
+
+ BitplaneFixture {
+ inputs,
+ output: alloc_aligned_vec(segment_len.next_multiple_of(BITPLANE_BLOCK_BYTES)),
+ kernels,
+ }
+}
+
#[library_benchmark]
#[bench::small(vec![0xAAu8; 528])]
#[bench::medium(vec![0xAAu8; 4096])]
@@ -69,9 +108,48 @@ fn bench_scalar_baseline(input: Vec) -> Vec {
output
}
+#[cfg(target_arch = "x86_64")]
+#[library_benchmark]
+#[bench::batch12_128k(bitplane_fixture(12, 128 * 1024))]
+fn bench_xor_jit_bitplane_batch_first(mut fixture: BitplaneFixture) -> Vec {
+ for (input, kernel) in fixture.inputs.iter().zip(fixture.kernels.iter()) {
+ kernel.multiply_add_chunks(black_box(input), black_box(&mut fixture.output));
+ }
+
+ fixture.output
+}
+
+#[cfg(target_arch = "x86_64")]
+#[library_benchmark]
+#[bench::batch12_128k(bitplane_fixture(12, 128 * 1024))]
+fn bench_xor_jit_bitplane_block_first(mut fixture: BitplaneFixture) -> Vec {
+ for block_offset in (0..fixture.output.len()).step_by(BITPLANE_BLOCK_BYTES) {
+ let output_block = &mut fixture.output[block_offset..block_offset + BITPLANE_BLOCK_BYTES];
+ for (input, kernel) in fixture.inputs.iter().zip(fixture.kernels.iter()) {
+ let input_block = &input[block_offset..block_offset + BITPLANE_BLOCK_BYTES];
+ kernel.multiply_add_block(black_box(input_block), black_box(output_block));
+ }
+ }
+
+ fixture.output
+}
+
library_benchmark_group!(
name = simd_group;
benchmarks = bench_pshufb_simd, bench_portable_simd, bench_scalar_baseline
);
+#[cfg(target_arch = "x86_64")]
+library_benchmark_group!(
+ name = xor_jit_bitplane_group;
+ benchmarks = bench_xor_jit_bitplane_batch_first, bench_xor_jit_bitplane_block_first
+);
+
+#[cfg(target_arch = "x86_64")]
+main!(
+ library_benchmark_groups = simd_group,
+ xor_jit_bitplane_group
+);
+
+#[cfg(not(target_arch = "x86_64"))]
main!(library_benchmark_groups = simd_group);
diff --git a/benches/md5x2_crc_fused.rs b/benches/md5x2_crc_fused.rs
new file mode 100644
index 00000000..0804cbe0
--- /dev/null
+++ b/benches/md5x2_crc_fused.rs
@@ -0,0 +1,269 @@
+//! End-to-end fused-hashing comparison for the par2 create path.
+//!
+//! Each variant computes the three outputs the create path needs:
+//! - file-MD5 (rolls across all blocks of a file)
+//! - block-MD5 (per source block)
+//! - block-CRC32 (per source block)
+//!
+//! 5 variants × 2 sizes (16 KiB sub-slice and 4 MiB block).
+//!
+//! All variants are sanity-checked at startup against the naive
+//! reference: any divergence aborts the run.
+//!
+//! Variants:
+//! 1. naive_seq — three independent passes (md-5 + md-5 + crc32fast).
+//! 2. tier1_subslice — current shipped helper:
+//! `update_file_md5_block_md5_crc32_fused`
+//! interleaves all three at 16 KiB sub-slice
+//! granularity (single-lane MD5s + crc32fast).
+//! 3. md5x2_crc32fast_64b — MD5x2 absorbs block+file MD5; crc32fast
+//! streamed in 64 B chunks.
+//! 4. md5x2_crcfast_64b — same, but `crc_fast::Digest` for CRC.
+//! 5. md5x2_only — lower bound: MD5x2 with no CRC.
+//! 6. md5x2_64b_then_crc_bulk — MD5x2 at 64 B granularity (block+file MD5
+//! in one walk) followed by ONE bulk
+//! `crc32fast::Hasher::update(data)` call.
+//! Tests whether crc32fast's per-call overhead
+//! (SIMD setup, alignment prologue) is what's
+//! costing variants 3/4 the gap to variant 5.
+//! 7. md5x2_clmul_64b_fused — MD5x2 + ParPar PCLMULQDQ CRC32 fused at
+//! 64 B granularity. The candidate: one
+//! cache-line read, three states updated,
+//! no per-call SIMD setup overhead.
+//!
+//! The MD5x2 lanes are fed the same 64 B for harness simplicity; this is
+//! what the real `HasherInput` will do when the file is 64 B-aligned and
+//! the staggered position offset is 0. (Stagger overhead is a separate
+//! concern for T2.c, not the backend choice here.)
+
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
+use md5::Digest as _;
+use md5::Md5;
+use par2rs::checksum::update_file_md5_block_md5_crc32_fused;
+use par2rs::parpar_hasher::{crc_clmul, md5x2_scalar};
+use std::hint::black_box;
+
+type Out = ([u8; 16], [u8; 16], u32); // (file_md5, block_md5, crc32)
+
+fn make_data(len: usize) -> Vec {
+ (0..len)
+ .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7))
+ .collect()
+}
+
+// ---------- variants ----------
+
+fn v1_naive_seq(data: &[u8]) -> Out {
+ let mut f = Md5::new();
+ f.update(data);
+ let mut b = Md5::new();
+ b.update(data);
+ let mut c = crc32fast::Hasher::new();
+ c.update(data);
+ (f.finalize().into(), b.finalize().into(), c.finalize())
+}
+
+fn v2_tier1_subslice(data: &[u8]) -> Out {
+ let mut f = Md5::new();
+ let mut b = Md5::new();
+ let mut c = crc32fast::Hasher::new();
+ update_file_md5_block_md5_crc32_fused(&mut f, &mut b, &mut c, data);
+ (f.finalize().into(), b.finalize().into(), c.finalize())
+}
+
+/// MD5x2 produces *two* MD5 results from one walk. We use lane 1 for
+/// file-MD5 and lane 2 for block-MD5; this bench feeds the same 64 B
+/// to both lanes so the two lanes' final digests are identical, but
+/// the cost profile (two MD5s + CRC in a single pass) matches what
+/// HasherInput will pay.
+fn md5x2_finalise(mut state: [u32; 8], total_len: usize) -> ([u8; 16], [u8; 16]) {
+ // Bench harness only feeds chunks_exact(64), so the partial buffer is
+ // empty at finalise time and one tail block suffices: 0x80, then zero
+ // pad to 56 mod 64, then 8-byte LE bit-length.
+ let bit_len = (total_len as u64).wrapping_mul(8);
+ let mut tail = [0u8; 64];
+ tail[0] = 0x80;
+ tail[56..64].copy_from_slice(&bit_len.to_le_bytes());
+ // Both lanes finalise in one MD5x2 step: same tail in both lanes,
+ // independent state per lane.
+ unsafe {
+ md5x2_scalar::process_block_x2_scalar(&mut state, tail.as_ptr(), tail.as_ptr());
+ }
+ let mut out1 = [0u8; 16];
+ let mut out2 = [0u8; 16];
+ for i in 0..4 {
+ out1[i * 4..i * 4 + 4].copy_from_slice(&state[i].to_le_bytes());
+ out2[i * 4..i * 4 + 4].copy_from_slice(&state[i + 4].to_le_bytes());
+ }
+ (out1, out2)
+}
+
+fn v3_md5x2_crc32fast(data: &[u8]) -> Out {
+ let mut state = md5x2_scalar::init_state();
+ let mut crc = crc32fast::Hasher::new();
+ for chunk in data.chunks_exact(64) {
+ unsafe {
+ md5x2_scalar::process_block_x2_scalar(&mut state, chunk.as_ptr(), chunk.as_ptr());
+ }
+ crc.update(chunk);
+ }
+ let (lane1, lane2) = md5x2_finalise(state, data.len());
+ (lane1, lane2, crc.finalize())
+}
+
+fn v4_md5x2_crcfast(data: &[u8]) -> Out {
+ use crc_fast::{CrcAlgorithm, Digest};
+ let mut state = md5x2_scalar::init_state();
+ let mut crc = Digest::new(CrcAlgorithm::Crc32IsoHdlc);
+ for chunk in data.chunks_exact(64) {
+ unsafe {
+ md5x2_scalar::process_block_x2_scalar(&mut state, chunk.as_ptr(), chunk.as_ptr());
+ }
+ crc.update(chunk);
+ }
+ let (lane1, lane2) = md5x2_finalise(state, data.len());
+ (lane1, lane2, crc.finalize() as u32)
+}
+
+fn v5_md5x2_only(data: &[u8]) -> ([u8; 16], [u8; 16]) {
+ let mut state = md5x2_scalar::init_state();
+ for chunk in data.chunks_exact(64) {
+ unsafe {
+ md5x2_scalar::process_block_x2_scalar(&mut state, chunk.as_ptr(), chunk.as_ptr());
+ }
+ }
+ md5x2_finalise(state, data.len())
+}
+
+/// MD5x2 fused at 64 B (block+file MD5 in one walk), but CRC32 is called
+/// ONCE on the whole buffer afterwards so crc32fast hits its big-buffer
+/// fast path instead of paying per-call overhead 65 536× per 4 MiB block.
+///
+/// The buffer is L2-resident from the MD5x2 walk, so the second pass is
+/// a hot read. Tests whether that's cheaper than 64 B-granularity CRC.
+fn v6_md5x2_then_crc_bulk(data: &[u8]) -> Out {
+ let mut state = md5x2_scalar::init_state();
+ for chunk in data.chunks_exact(64) {
+ unsafe {
+ md5x2_scalar::process_block_x2_scalar(&mut state, chunk.as_ptr(), chunk.as_ptr());
+ }
+ }
+ let (lane1, lane2) = md5x2_finalise(state, data.len());
+ let mut crc = crc32fast::Hasher::new();
+ crc.update(data);
+ (lane1, lane2, crc.finalize())
+}
+
+/// MD5x2 + ParPar PCLMULQDQ CRC32 fused at 64 B granularity. Both states
+/// advance from the same cache-line read; no per-call SIMD setup cost.
+///
+/// Bench harness only feeds 16 KiB / 4 MiB inputs (both 64-aligned), so
+/// the CRC `finish` path's tail handling never triggers here — the loop
+/// is the steady-state cost the create path will see for full blocks.
+fn v7_md5x2_clmul_64b_fused(data: &[u8]) -> Out {
+ let mut md5_state = md5x2_scalar::init_state();
+ let mut crc_state = crc_clmul::State::new();
+ unsafe {
+ crc_clmul::init(&mut crc_state);
+ let mut p = data.as_ptr();
+ let mut remaining = data.len();
+ while remaining >= 64 {
+ md5x2_scalar::process_block_x2_scalar(&mut md5_state, p, p);
+ crc_clmul::process_block(&mut crc_state, p);
+ p = p.add(64);
+ remaining -= 64;
+ }
+ let crc = crc_clmul::finish(&mut crc_state, p, remaining);
+ let (lane1, lane2) = md5x2_finalise(md5_state, data.len());
+ (lane1, lane2, crc)
+ }
+}
+
+// ---------- bench ----------
+
+fn bench(c: &mut Criterion) {
+ let sizes = [16 * 1024usize, 4 * 1024 * 1024];
+ let mut group = c.benchmark_group("create_path_hashing");
+
+ // Sanity-check correctness at every size before benching.
+ for &len in &sizes {
+ assert!(
+ len % 64 == 0,
+ "bench sizes must be 64-aligned for MD5x2 harness"
+ );
+ let d = make_data(len);
+ let r1 = v1_naive_seq(&d);
+ let r2 = v2_tier1_subslice(&d);
+ let r3 = v3_md5x2_crc32fast(&d);
+ let r4 = v4_md5x2_crcfast(&d);
+ let r5 = v5_md5x2_only(&d);
+ let r6 = v6_md5x2_then_crc_bulk(&d);
+ let r7 = v7_md5x2_clmul_64b_fused(&d);
+
+ // CRC: variants 1, 2, 3, 6, 7 must match (all CRC32 IEEE);
+ // variant 4 same value via crc-fast.
+ assert_eq!(r1.2, r2.2);
+ assert_eq!(r1.2, r3.2);
+ assert_eq!(r1.2, r4.2, "crc-fast disagrees with crc32fast at len={len}");
+ assert_eq!(r1.2, r6.2, "v6 bulk crc disagrees at len={len}");
+ assert_eq!(r1.2, r7.2, "v7 clmul crc disagrees at len={len}");
+
+ // Naive single-lane MD5 == MD5x2 lane (since both lanes get the
+ // same input, MD5x2 lane1 == MD5x2 lane2 == single-lane MD5).
+ assert_eq!(
+ r1.0, r3.0,
+ "MD5x2 lane1 != single-lane MD5 at len={len} (v3)"
+ );
+ assert_eq!(
+ r1.0, r3.1,
+ "MD5x2 lane2 != single-lane MD5 at len={len} (v3)"
+ );
+ assert_eq!(r3.0, r4.0);
+ assert_eq!(r3.0, r5.0);
+ assert_eq!(r3.1, r5.1);
+ assert_eq!(r3.0, r6.0);
+ assert_eq!(r3.1, r6.1);
+ assert_eq!(r3.0, r7.0);
+ assert_eq!(r3.1, r7.1);
+ }
+
+ for &len in &sizes {
+ let data = make_data(len);
+ group.throughput(Throughput::Bytes(len as u64));
+
+ group.bench_with_input(BenchmarkId::new("1_naive_seq", len), &data, |b, d| {
+ b.iter(|| black_box(v1_naive_seq(black_box(d))));
+ });
+ group.bench_with_input(BenchmarkId::new("2_tier1_subslice", len), &data, |b, d| {
+ b.iter(|| black_box(v2_tier1_subslice(black_box(d))));
+ });
+ group.bench_with_input(BenchmarkId::new("3_md5x2+crc32fast", len), &data, |b, d| {
+ b.iter(|| black_box(v3_md5x2_crc32fast(black_box(d))))
+ });
+ group.bench_with_input(BenchmarkId::new("4_md5x2+crc-fast", len), &data, |b, d| {
+ b.iter(|| black_box(v4_md5x2_crcfast(black_box(d))));
+ });
+ group.bench_with_input(BenchmarkId::new("5_md5x2_only", len), &data, |b, d| {
+ b.iter(|| black_box(v5_md5x2_only(black_box(d))));
+ });
+ group.bench_with_input(
+ BenchmarkId::new("6_md5x2_then_crc_bulk", len),
+ &data,
+ |b, d| {
+ b.iter(|| black_box(v6_md5x2_then_crc_bulk(black_box(d))));
+ },
+ );
+ group.bench_with_input(
+ BenchmarkId::new("7_md5x2_clmul_64b_fused", len),
+ &data,
+ |b, d| {
+ b.iter(|| black_box(v7_md5x2_clmul_64b_fused(black_box(d))));
+ },
+ );
+ }
+
+ group.finish();
+}
+
+criterion_group!(benches, bench);
+criterion_main!(benches);
diff --git a/benches/par2verify_compare.rs b/benches/par2verify_compare.rs
new file mode 100644
index 00000000..074afd34
--- /dev/null
+++ b/benches/par2verify_compare.rs
@@ -0,0 +1,51 @@
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
+use std::process::Command;
+use std::time::Duration;
+
+#[path = "common/par2verify_fixture.rs"]
+mod par2verify_fixture;
+
+fn run_invocation(invocation: &par2verify_fixture::Invocation) {
+ let output = Command::new(&invocation.program)
+ .current_dir(&invocation.current_dir)
+ .args(&invocation.args)
+ .output()
+ .unwrap_or_else(|err| {
+ panic!(
+ "failed to run {}: {err}",
+ invocation.program.to_string_lossy()
+ )
+ });
+
+ assert!(
+ output.status.success(),
+ "verify command failed: program={} status={:?} stderr={}",
+ invocation.program.display(),
+ output.status.code(),
+ String::from_utf8_lossy(&output.stderr)
+ );
+}
+
+fn bench_par2verify_compare(c: &mut Criterion) {
+ let fixture = par2verify_fixture::criterion_fixture();
+ let mut group = c.benchmark_group("par2verify_compare");
+ group.sample_size(10);
+ group.warm_up_time(Duration::from_secs(2));
+ group.measurement_time(Duration::from_secs(20));
+ group.throughput(Throughput::Bytes(fixture.protected_bytes));
+
+ let par2rs = par2verify_fixture::par2rs_verify_invocation(fixture);
+ group.bench_function(BenchmarkId::new("par2rs", fixture.case_name), |b| {
+ b.iter(|| run_invocation(&par2rs))
+ });
+
+ let turbo = par2verify_fixture::turbo_verify_invocation(fixture);
+ group.bench_function(BenchmarkId::new("turbo", fixture.case_name), |b| {
+ b.iter(|| run_invocation(&turbo))
+ });
+
+ group.finish();
+}
+
+criterion_group!(benches, bench_par2verify_compare);
+criterion_main!(benches);
diff --git a/benches/parpar_hasher_input.rs b/benches/parpar_hasher_input.rs
new file mode 100644
index 00000000..de372620
--- /dev/null
+++ b/benches/parpar_hasher_input.rs
@@ -0,0 +1,285 @@
+//! Benchmark for the ported ParPar `HasherInput` fused driver (T2.c).
+//!
+//! Compares three implementations of the create-path workload —
+//! producing per-file MD5 + per-block MD5 + per-block CRC32 over a
+//! sequence of source bytes:
+//!
+//! 1. `naive_seq` — three independent passes (file MD5, block MD5,
+//! block CRC32) using `md-5` + `crc32fast`. The
+//! upper bound on naive throughput.
+//! 2. `tier1_helper` — the currently-shipped Tier-1 sub-slice helper
+//! `update_file_md5_block_md5_crc32_fused` driving
+//! three single-lane hashers from one walk.
+//! 3. `hasher_input` — the new ParPar HasherInput port: MD5x2
+//! (block-MD5 + file-MD5 in one walk via ILP) +
+//! PCLMULQDQ CRC32, fused at 64 B granularity.
+//!
+//! Two scenarios:
+//!
+//! * `single_block_` — exactly one PAR2 block, no zero-pad.
+//! Driver is reset per iteration. Sizes: 16 KiB, 4 MiB.
+//! * `multi_block_x` — N blocks of `block_size` bytes
+//! plus a `block_size/2` short tail. Exercises the steady-state
+//! loop AND the staggered-offset get_block path. Default: 4×4 MiB.
+//!
+//! Correctness: every variant is sanity-checked once at startup
+//! against the naive reference; any mismatch panics before timing.
+
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
+use md5::Digest as _;
+use md5::Md5;
+use par2rs::checksum::update_file_md5_block_md5_crc32_fused;
+use par2rs::parpar_hasher::hasher_input::HasherInput;
+use par2rs::parpar_hasher::md5x2::Md5x2;
+use par2rs::parpar_hasher::md5x2_avx512::Avx512;
+use par2rs::parpar_hasher::md5x2_bmi1::Bmi1;
+use par2rs::parpar_hasher::md5x2_scalar::Scalar;
+use par2rs::parpar_hasher::md5x2_sse2::Sse2;
+use std::hint::black_box;
+
+/// (file_md5, Vec<(block_md5, block_crc32)>)
+type MultiBlockOut = ([u8; 16], Vec<([u8; 16], u32)>);
+
+fn make_data(len: usize) -> Vec {
+ (0..len)
+ .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7))
+ .collect()
+}
+
+// ----------------------------- Variants -----------------------------
+
+fn v1_naive_seq_blocks(data: &[u8], block_size: usize) -> MultiBlockOut {
+ let mut file = Md5::new();
+ file.update(data);
+ let file_md5: [u8; 16] = file.finalize().into();
+
+ let mut blocks = Vec::new();
+ let mut off = 0;
+ while off < data.len() {
+ let end = (off + block_size).min(data.len());
+ let real = &data[off..end];
+ let pad = block_size - real.len();
+
+ let mut bm = Md5::new();
+ bm.update(real);
+ if pad > 0 {
+ let zeros = vec![0u8; pad];
+ bm.update(&zeros);
+ }
+ let bmd5: [u8; 16] = bm.finalize().into();
+
+ let mut bc = crc32fast::Hasher::new();
+ bc.update(real);
+ if pad > 0 {
+ let zeros = vec![0u8; pad];
+ bc.update(&zeros);
+ }
+ blocks.push((bmd5, bc.finalize()));
+ off = end;
+ }
+ (file_md5, blocks)
+}
+
+fn v2_tier1_helper_blocks(data: &[u8], block_size: usize) -> MultiBlockOut {
+ let mut file = Md5::new();
+ let mut blocks = Vec::new();
+ let mut off = 0;
+ while off < data.len() {
+ let end = (off + block_size).min(data.len());
+ let real = &data[off..end];
+ let pad = block_size - real.len();
+
+ let mut bm = Md5::new();
+ let mut bc = crc32fast::Hasher::new();
+ update_file_md5_block_md5_crc32_fused(&mut file, &mut bm, &mut bc, real);
+ if pad > 0 {
+ // Pad block-only (file MD5 already absorbed real bytes only,
+ // matching naive: file is over data, NOT over zero-padded blocks).
+ let zeros = vec![0u8; pad];
+ bm.update(&zeros);
+ bc.update(&zeros);
+ }
+ let bmd5: [u8; 16] = bm.finalize().into();
+ blocks.push((bmd5, bc.finalize()));
+ off = end;
+ }
+ let file_md5: [u8; 16] = file.finalize().into();
+ (file_md5, blocks)
+}
+
+fn hasher_input_blocks(data: &[u8], block_size: usize) -> MultiBlockOut {
+ let mut h: HasherInput = HasherInput::new();
+ let mut blocks = Vec::new();
+ let mut off = 0;
+ let mut written_in_block = 0usize;
+
+ while off < data.len() {
+ let block_remaining = block_size - written_in_block;
+ let take = block_remaining.min(data.len() - off);
+ h.update(&data[off..off + take]);
+ off += take;
+ written_in_block += take;
+ if written_in_block == block_size {
+ let bh = h.get_block(0);
+ blocks.push((bh.md5, bh.crc32));
+ written_in_block = 0;
+ }
+ }
+ if written_in_block > 0 {
+ let pad = (block_size - written_in_block) as u64;
+ let bh = h.get_block(pad);
+ blocks.push((bh.md5, bh.crc32));
+ }
+ let file_md5 = h.end();
+ (file_md5, blocks)
+}
+
+fn v3_hasher_input_scalar(data: &[u8], block_size: usize) -> MultiBlockOut {
+ hasher_input_blocks::(data, block_size)
+}
+
+fn v4_hasher_input_sse2(data: &[u8], block_size: usize) -> MultiBlockOut {
+ hasher_input_blocks::(data, block_size)
+}
+
+fn v5_hasher_input_bmi1(data: &[u8], block_size: usize) -> MultiBlockOut {
+ hasher_input_blocks::(data, block_size)
+}
+
+fn v6_hasher_input_avx512(data: &[u8], block_size: usize) -> MultiBlockOut {
+ hasher_input_blocks::(data, block_size)
+}
+
+fn avx512_supported() -> bool {
+ std::is_x86_feature_detected!("avx512f")
+ && std::is_x86_feature_detected!("avx512vl")
+ && std::is_x86_feature_detected!("pclmulqdq")
+ && std::is_x86_feature_detected!("sse4.1")
+}
+
+// ----------------------------- Bench -----------------------------
+
+fn bench(c: &mut Criterion) {
+ // Scenario A: a single full block. Highlights steady-state cost
+ // without the get_block / cross-block plumbing.
+ let single_sizes = [16 * 1024usize, 4 * 1024 * 1024];
+
+ // Scenario B: a multi-block "file": four full blocks + one short
+ // tail (half a block) that exercises zero-pad + carry between
+ // get_block calls.
+ let multi_block_size = 4 * 1024 * 1024;
+ let multi_blocks = 4usize;
+ let multi_total = multi_block_size * multi_blocks + multi_block_size / 2;
+
+ // ---- Correctness sanity check (panics on divergence) ----
+ let bmi1_ok = std::is_x86_feature_detected!("bmi1");
+ let avx512_ok = avx512_supported();
+ for &len in &single_sizes {
+ let data = make_data(len);
+ let r1 = v1_naive_seq_blocks(&data, len);
+ let r2 = v2_tier1_helper_blocks(&data, len);
+ let r3 = v3_hasher_input_scalar(&data, len);
+ let r4 = v4_hasher_input_sse2(&data, len);
+ assert_eq!(r1, r2, "tier1 helper diverges from naive at len={len}");
+ assert_eq!(
+ r1, r3,
+ "HasherInput diverges from naive at len={len}"
+ );
+ assert_eq!(r1, r4, "HasherInput diverges from naive at len={len}");
+ if bmi1_ok {
+ let r5 = v5_hasher_input_bmi1(&data, len);
+ assert_eq!(r1, r5, "HasherInput diverges from naive at len={len}");
+ }
+ if avx512_ok {
+ let r6 = v6_hasher_input_avx512(&data, len);
+ assert_eq!(
+ r1, r6,
+ "HasherInput diverges from naive at len={len}"
+ );
+ }
+ }
+ {
+ let data = make_data(multi_total);
+ let r1 = v1_naive_seq_blocks(&data, multi_block_size);
+ let r2 = v2_tier1_helper_blocks(&data, multi_block_size);
+ let r3 = v3_hasher_input_scalar(&data, multi_block_size);
+ let r4 = v4_hasher_input_sse2(&data, multi_block_size);
+ assert_eq!(r1, r2, "tier1 helper multi-block diverges");
+ assert_eq!(r1, r3, "HasherInput multi-block diverges");
+ assert_eq!(r1, r4, "HasherInput multi-block diverges");
+ if bmi1_ok {
+ let r5 = v5_hasher_input_bmi1(&data, multi_block_size);
+ assert_eq!(r1, r5, "HasherInput multi-block diverges");
+ }
+ if avx512_ok {
+ let r6 = v6_hasher_input_avx512(&data, multi_block_size);
+ assert_eq!(r1, r6, "HasherInput multi-block diverges");
+ }
+ }
+
+ // ---- Single-block group ----
+ let mut g = c.benchmark_group("hasher_input_single_block");
+ for &len in &single_sizes {
+ let data = make_data(len);
+ g.throughput(Throughput::Bytes(len as u64));
+ g.bench_with_input(BenchmarkId::new("1_naive_seq", len), &data, |b, d| {
+ b.iter(|| black_box(v1_naive_seq_blocks(black_box(d), len)))
+ });
+ g.bench_with_input(BenchmarkId::new("2_tier1_helper", len), &data, |b, d| {
+ b.iter(|| black_box(v2_tier1_helper_blocks(black_box(d), len)))
+ });
+ g.bench_with_input(BenchmarkId::new("3_hasher_scalar", len), &data, |b, d| {
+ b.iter(|| black_box(v3_hasher_input_scalar(black_box(d), len)))
+ });
+ g.bench_with_input(BenchmarkId::new("4_hasher_sse2", len), &data, |b, d| {
+ b.iter(|| black_box(v4_hasher_input_sse2(black_box(d), len)))
+ });
+ if bmi1_ok {
+ g.bench_with_input(BenchmarkId::new("5_hasher_bmi1", len), &data, |b, d| {
+ b.iter(|| black_box(v5_hasher_input_bmi1(black_box(d), len)))
+ });
+ }
+ if avx512_ok {
+ g.bench_with_input(BenchmarkId::new("6_hasher_avx512", len), &data, |b, d| {
+ b.iter(|| black_box(v6_hasher_input_avx512(black_box(d), len)))
+ });
+ }
+ }
+ g.finish();
+
+ // ---- Multi-block group ----
+ let mut g = c.benchmark_group("hasher_input_multi_block");
+ let data = make_data(multi_total);
+ let label = format!("{multi_blocks}x{multi_block_size}+half");
+ g.throughput(Throughput::Bytes(multi_total as u64));
+ g.bench_with_input(BenchmarkId::new("1_naive_seq", &label), &data, |b, d| {
+ b.iter(|| black_box(v1_naive_seq_blocks(black_box(d), multi_block_size)))
+ });
+ g.bench_with_input(BenchmarkId::new("2_tier1_helper", &label), &data, |b, d| {
+ b.iter(|| black_box(v2_tier1_helper_blocks(black_box(d), multi_block_size)))
+ });
+ g.bench_with_input(
+ BenchmarkId::new("3_hasher_scalar", &label),
+ &data,
+ |b, d| b.iter(|| black_box(v3_hasher_input_scalar(black_box(d), multi_block_size))),
+ );
+ g.bench_with_input(BenchmarkId::new("4_hasher_sse2", &label), &data, |b, d| {
+ b.iter(|| black_box(v4_hasher_input_sse2(black_box(d), multi_block_size)))
+ });
+ if bmi1_ok {
+ g.bench_with_input(BenchmarkId::new("5_hasher_bmi1", &label), &data, |b, d| {
+ b.iter(|| black_box(v5_hasher_input_bmi1(black_box(d), multi_block_size)))
+ });
+ }
+ if avx512_ok {
+ g.bench_with_input(
+ BenchmarkId::new("6_hasher_avx512", &label),
+ &data,
+ |b, d| b.iter(|| black_box(v6_hasher_input_avx512(black_box(d), multi_block_size))),
+ );
+ }
+ g.finish();
+}
+
+criterion_group!(benches, bench);
+criterion_main!(benches);
diff --git a/build.rs b/build.rs
new file mode 100644
index 00000000..2cb2da25
--- /dev/null
+++ b/build.rs
@@ -0,0 +1,118 @@
+use std::env;
+use std::path::PathBuf;
+
+fn main() {
+ println!("cargo:rustc-check-cfg=cfg(parpar_compare_embedded)");
+
+ if env::var_os("CARGO_FEATURE_PARPAR_COMPARE").is_none() {
+ return;
+ }
+
+ let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
+ let parpar = manifest_dir.join("par2cmdline-turbo/parpar");
+ let hasher = parpar.join("hasher");
+ let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default();
+ let md5_final = hasher.join("md5-final.c");
+
+ if target_arch != "x86_64" {
+ println!(
+ "cargo:warning=parpar-compare requested for unsupported target arch `{target_arch}`; skipping embedded ParPar build"
+ );
+ return;
+ }
+
+ if !md5_final.exists() {
+ println!(
+ "cargo:warning=parpar-compare requested but vendored ParPar sources are unavailable at {}; skipping embedded ParPar build",
+ parpar.display()
+ );
+ return;
+ }
+
+ println!("cargo:rustc-cfg=parpar_compare_embedded");
+
+ for path in [
+ "src/ffi/wrapper.cpp",
+ "par2cmdline-turbo/parpar/hasher/hasher_input.cpp",
+ "par2cmdline-turbo/parpar/hasher/hasher_md5crc.cpp",
+ "par2cmdline-turbo/parpar/hasher/hasher_scalar.cpp",
+ "par2cmdline-turbo/parpar/hasher/hasher_sse.cpp",
+ "par2cmdline-turbo/parpar/hasher/hasher_bmi1.cpp",
+ "par2cmdline-turbo/parpar/hasher/hasher_clmul.cpp",
+ "par2cmdline-turbo/parpar/hasher/tables.cpp",
+ "par2cmdline-turbo/parpar/hasher/crc_zeropad.c",
+ "par2cmdline-turbo/parpar/hasher/hasher_input_impl.h",
+ "par2cmdline-turbo/parpar/hasher/hasher_input.h",
+ "par2cmdline-turbo/parpar/hasher/hasher_md5crc.h",
+ "par2cmdline-turbo/parpar/hasher/hasher_md5crc_impl.h",
+ "par2cmdline-turbo/parpar/hasher/hasher_input_base.h",
+ "par2cmdline-turbo/parpar/hasher/md5x2-base.h",
+ "par2cmdline-turbo/parpar/hasher/md5x2-scalar.h",
+ "par2cmdline-turbo/parpar/hasher/md5x2-sse.h",
+ "par2cmdline-turbo/parpar/hasher/crc_slice4.h",
+ "par2cmdline-turbo/parpar/hasher/crc_clmul.h",
+ "par2cmdline-turbo/parpar/hasher/crc_zeropad.h",
+ "par2cmdline-turbo/parpar/src/platform.h",
+ "par2cmdline-turbo/parpar/src/hedley.h",
+ ] {
+ println!("cargo:rerun-if-changed={path}");
+ }
+
+ let mut c_build = cc::Build::new();
+ c_build.include(&manifest_dir);
+ c_build.include(&parpar);
+ c_build.include(&hasher);
+ c_build.file(&md5_final);
+ c_build.file(hasher.join("crc_zeropad.c"));
+ c_build.compile("parpar_helpers");
+
+ let mut cpp_build = cc::Build::new();
+ cpp_build.cpp(true);
+ cpp_build.std("c++17");
+ cpp_build.define("PARPAR_ENABLE_HASHER_MD5CRC", None);
+ cpp_build.include(&manifest_dir);
+ cpp_build.include(&parpar);
+ cpp_build.include(&hasher);
+
+ if target_arch == "x86_64" {
+ cpp_build.flag_if_supported("-msse2");
+ cpp_build.flag_if_supported("-mssse3");
+ cpp_build.flag_if_supported("-msse4.1");
+ cpp_build.flag_if_supported("-mpclmul");
+ cpp_build.flag_if_supported("-mbmi");
+ cpp_build.flag_if_supported("-mbmi2");
+ cpp_build.flag_if_supported("-mavx");
+ }
+
+ cpp_build
+ .file(manifest_dir.join("src/ffi/wrapper.cpp"))
+ .file(hasher.join("hasher_input.cpp"))
+ .file(hasher.join("hasher_md5crc.cpp"))
+ .file(hasher.join("hasher_scalar.cpp"))
+ .file(hasher.join("hasher_sse.cpp"))
+ .file(hasher.join("hasher_bmi1.cpp"))
+ .file(hasher.join("hasher_clmul.cpp"))
+ .file(hasher.join("tables.cpp"))
+ .compile("parpar_wrapper");
+
+ let mut avx512_build = cc::Build::new();
+ avx512_build.cpp(true);
+ avx512_build.std("c++17");
+ avx512_build.include(&manifest_dir);
+ avx512_build.include(&parpar);
+ avx512_build.include(&hasher);
+ avx512_build.flag_if_supported("-msse2");
+ avx512_build.flag_if_supported("-mssse3");
+ avx512_build.flag_if_supported("-msse4.1");
+ avx512_build.flag_if_supported("-mpclmul");
+ avx512_build.flag_if_supported("-mbmi");
+ avx512_build.flag_if_supported("-mavx512f");
+ avx512_build.flag_if_supported("-mavx512vl");
+ avx512_build.flag_if_supported("-mavx512bw");
+ avx512_build.flag_if_supported("-mavx512dq");
+ avx512_build.flag_if_supported("-mbmi2");
+ avx512_build
+ .file(hasher.join("hasher_avx512.cpp"))
+ .file(hasher.join("hasher_avx512vl.cpp"))
+ .compile("parpar_avx512");
+}
diff --git a/flake.nix b/flake.nix
index 52aee1cd..7bbb1635 100644
--- a/flake.nix
+++ b/flake.nix
@@ -102,11 +102,11 @@
inferno
bc # for benchmark averaging calculations
gnuplot # for criterion benchmark graphs
- valgrind # for iai-callgrind benchmarks
gh # GitHub CLI
git-filter-repo # for rewriting git history
]
++ pkgs.lib.optionals pkgs.stdenv.isLinux [
+ valgrind # for iai-callgrind benchmarks
heaptrack
perf
];
diff --git a/scripts/benchmark_create_perf.sh b/scripts/benchmark_create_perf.sh
index 594803d7..9368a611 100755
--- a/scripts/benchmark_create_perf.sh
+++ b/scripts/benchmark_create_perf.sh
@@ -1,30 +1,49 @@
#!/usr/bin/env bash
-# Compare par2rs create against par2cmdline-turbo using instruction counts.
+# Compare par2rs create against par2cmdline-turbo.
#
-# This harness is designed for loaded machines: wall clock is captured only as
-# context, while Linux perf hardware counters are the primary comparison signal.
+# Wall time is the pass/fail signal for the JIT-vs-turbo requirement. Linux perf
+# hardware counters are captured as diagnostics to explain timing differences.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+PLATFORM="$(uname -s)"
+ARCH="$(uname -m)"
ITERATIONS="${ITERATIONS:-30}"
WARMUP_RUNS="${WARMUP_RUNS:-3}"
REDUNDANCY="${REDUNDANCY:-10}"
-RECOVERY_FILES="${RECOVERY_FILES:-1}"
+RECOVERY_FILES="${RECOVERY_FILES:-8}"
+FIRST_RECOVERY_BLOCK="${FIRST_RECOVERY_BLOCK:-}"
THREADS="${THREADS:-1}"
PAR2CMD_BIN="${PAR2CMD_BIN:-par2}"
PAR2RS_BIN="${PAR2RS_BIN:-$PROJECT_ROOT/target/release/par2}"
RESULTS_ROOT="${RESULTS_ROOT:-$PROJECT_ROOT/target/perf-results/create}"
KEEP_WORK="${KEEP_WORK:-0}"
RUN_FLAMEGRAPH="${RUN_FLAMEGRAPH:-1}"
+PROFILE_CASE="${PROFILE_CASE:-}"
+PROFILE_TOOL="${PROFILE_TOOL:-}"
PROFILE_FREQUENCY="${PROFILE_FREQUENCY:-997}"
PERF_EVENTS="${PERF_EVENTS:-instructions,cycles,branches,branch-misses,cache-references,cache-misses,task-clock,context-switches,cpu-migrations,page-faults}"
+CACHE_PROFILE="${CACHE_PROFILE:-0}"
+CACHE_PROFILE_TOOL="${CACHE_PROFILE_TOOL:-}"
+CACHE_PROFILE_JIT_DUMPS="${CACHE_PROFILE_JIT_DUMPS:-1}"
+CACHE_LOAD_LATENCY="${CACHE_LOAD_LATENCY:-30}"
+CACHE_STAT_EVENTS="${CACHE_STAT_EVENTS:-instructions,cycles,L1-dcache-loads,L1-dcache-load-misses,LLC-loads,LLC-load-misses,dTLB-loads,dTLB-load-misses,cache-references,cache-misses}"
VERIFY_OUTPUTS="${VERIFY_OUTPUTS:-1}"
+VERIFY_REPAIR="${VERIFY_REPAIR:-smoke}"
+INCLUDE_PSHUFB="${INCLUDE_PSHUFB:-0}"
+DEFAULT_BENCHMARK_TOOLS=(
+ par2rs-xor-jit
+ turbo-auto
+)
+if [[ "$INCLUDE_PSHUFB" == "1" ]]; then
+ DEFAULT_BENCHMARK_TOOLS=(par2rs-pshufb "${DEFAULT_BENCHMARK_TOOLS[@]}")
+fi
# label:file_count:file_size_mib:block_size_bytes
-DEFAULT_CASES="single_16m:1:16:262144,single_128m:1:128:1048576,multi_128m:32:4:262144"
+DEFAULT_CASES="single_256m:1:256:1048576,multi_1g:64:16:1048576,single_5g:1:5120:1048576"
CASES="${CASES:-$DEFAULT_CASES}"
usage() {
@@ -39,15 +58,37 @@ Environment:
THREADS=N pass -t N to both tools; 0 omits -t (default: $THREADS)
REDUNDANCY=N create redundancy percentage (default: $REDUNDANCY)
RECOVERY_FILES=N recovery file count via -n (default: $RECOVERY_FILES)
+ FIRST_RECOVERY_BLOCK=N
+ pass -f N to both tools; default unset
PAR2CMD_BIN=PATH par2cmdline-turbo binary (default: $PAR2CMD_BIN)
+ INCLUDE_PSHUFB=1 include the par2rs PSHUFB baseline (default: $INCLUDE_PSHUFB)
+ WORK_ROOT=DIR generated corpus and PAR2 work directory
+ default: result run directory/work
+ CORPUS_ROOT=DIR reusable corpus root, separate from per-run work/output dirs
+ default: under WORK_ROOT//corpus
VERIFY_OUTPUTS=0 skip cross-tool verification after each create
- RUN_FLAMEGRAPH=0 skip par2rs perf/inferno flamegraph generation
- PROFILE_FREQUENCY=N perf sampling frequency for flamegraph (default: $PROFILE_FREQUENCY)
+ VERIFY_REPAIR=MODE run destructive cross-tool repair validation: smoke, 1, or 0
+ default: $VERIFY_REPAIR
+ Required tools depend on platform/arch.
+ Forced JIT runs set PAR2RS_CREATE_XOR_JIT_FALLBACK=error.
+ RUN_FLAMEGRAPH=0 skip par2rs flamegraph generation
+ PROFILE_CASE=LABEL profile this CASES label; default profiles the last case
+ PROFILE_TOOL=TOOL par2rs tool variant to profile (default: platform-specific)
+ PROFILE_FREQUENCY=N sampling frequency for flamegraph (default: $PROFILE_FREQUENCY)
+ CACHE_PROFILE=1 run Linux cache-miss attribution for the profiled case
+ CACHE_PROFILE_TOOL=TOOL par2rs variant for cache profile (default: PROFILE_TOOL)
+ CACHE_PROFILE_JIT_DUMPS=0 skip xor-jit byte dumps during cache profiles (default: $CACHE_PROFILE_JIT_DUMPS)
+ also disables per-overwrite coefficient perf-map labels
+ CACHE_LOAD_LATENCY=N minimum load latency for precise mem-load sampling (default: $CACHE_LOAD_LATENCY)
+ CACHE_STAT_EVENTS=EVENTS comma-separated cache counter list
KEEP_WORK=1 keep generated benchmark work directory
Example:
- nix develop --command env ITERATIONS=40 THREADS=16 \\
- CASES='single_256m:1:256:1048576,multi_1g:64:16:1048576' \\
+ nix develop --command env ITERATIONS=10 THREADS=16 PROFILE_CASE=single_5g \\
+ WORK_ROOT="\$HOME/uncompressed/par2rs-create-perf/manual-run" \\
+ CORPUS_ROOT="\$HOME/uncompressed/par2rs-create-perf/corpus-cache" \\
+ RECOVERY_FILES=8 FIRST_RECOVERY_BLOCK=1 \\
+ CASES='single_256m:1:256:1048576,multi_1g:64:16:1048576,single_5g:1:5120:1048576' \\
scripts/benchmark_create_perf.sh
EOF
}
@@ -65,29 +106,88 @@ need_tool() {
fi
}
-need_tool perf
+tool_supported() {
+ local tool="$1"
+
+ case "$tool" in
+ turbo-auto|par2rs-auto|par2rs-scalar)
+ return 0
+ ;;
+ par2rs-pshufb|par2rs-xor-jit|par2rs-xor-jit-port)
+ [[ "$ARCH" == "x86_64" ]]
+ return
+ ;;
+ *)
+ return 1
+ ;;
+ esac
+}
+
+BENCHMARK_TOOLS=()
+for tool in "${DEFAULT_BENCHMARK_TOOLS[@]}"; do
+ if tool_supported "$tool"; then
+ BENCHMARK_TOOLS+=("$tool")
+ fi
+done
+
+if [[ "${#BENCHMARK_TOOLS[@]}" == "1" && "${BENCHMARK_TOOLS[0]}" == "turbo-auto" ]]; then
+ BENCHMARK_TOOLS=(par2rs-auto turbo-auto)
+fi
+
+if [[ -z "$PROFILE_TOOL" ]]; then
+ if tool_supported "par2rs-xor-jit"; then
+ PROFILE_TOOL="par2rs-xor-jit"
+ else
+ for tool in "${BENCHMARK_TOOLS[@]}"; do
+ if [[ "$tool" == par2rs-* ]]; then
+ PROFILE_TOOL="$tool"
+ break
+ fi
+ done
+ fi
+fi
+
+if ! tool_supported "$PROFILE_TOOL"; then
+ echo "error: PROFILE_TOOL '$PROFILE_TOOL' is unsupported on $PLATFORM/$ARCH" >&2
+ exit 1
+fi
+
+if [[ -z "$CACHE_PROFILE_TOOL" ]]; then
+ CACHE_PROFILE_TOOL="$PROFILE_TOOL"
+fi
+if ! tool_supported "$CACHE_PROFILE_TOOL"; then
+ echo "error: CACHE_PROFILE_TOOL '$CACHE_PROFILE_TOOL' is unsupported on $PLATFORM/$ARCH" >&2
+ exit 1
+fi
+
need_tool python3
need_tool dd
+need_tool cmp
need_tool "$PAR2CMD_BIN"
-if ! perf stat -x, -e instructions -- true >/dev/null 2>&1; then
- echo "error: perf cannot read the instructions counter." >&2
- echo "Check kernel.perf_event_paranoid or run in an environment with perf permissions." >&2
- exit 1
+if [[ "$PLATFORM" == "Linux" ]]; then
+ need_tool perf
+ if ! perf stat -x, -e instructions -- true >/dev/null 2>&1; then
+ echo "error: perf cannot read the instructions counter." >&2
+ echo "Check kernel.perf_event_paranoid or run in an environment with perf permissions." >&2
+ exit 1
+ fi
fi
mkdir -p "$RESULTS_ROOT"
TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
RUN_ROOT="$RESULTS_ROOT/run-$TIMESTAMP"
-WORK_ROOT="$RUN_ROOT/work"
+WORK_ROOT="${WORK_ROOT:-$RUN_ROOT/work}"
+CORPUS_ROOT="${CORPUS_ROOT:-}"
RAW_CSV="$RUN_ROOT/raw.csv"
SUMMARY_MD="$RUN_ROOT/summary.md"
SMOKE_CSV="$RUN_ROOT/smoke.csv"
SMOKE_SUMMARY_MD="$RUN_ROOT/smoke-summary.md"
-mkdir -p "$WORK_ROOT"
+PRESERVE_WORK=0
+mkdir -p "$RUN_ROOT" "$WORK_ROOT"
cleanup() {
- if [[ "$KEEP_WORK" != "1" ]]; then
+ if [[ "$KEEP_WORK" != "1" && "$PRESERVE_WORK" != "1" ]]; then
rm -rf "$WORK_ROOT"
fi
}
@@ -95,36 +195,109 @@ trap cleanup EXIT
echo "=== par2rs create perf benchmark ==="
echo "results: $RUN_ROOT"
+echo "work: $WORK_ROOT"
+if [[ -n "$CORPUS_ROOT" ]]; then
+ echo "corpus: $CORPUS_ROOT"
+fi
echo "iterations: $ITERATIONS measured, $WARMUP_RUNS warmup"
echo "cases: $CASES"
echo "threads: $THREADS"
+echo "recovery files: $RECOVERY_FILES"
+if [[ -n "$FIRST_RECOVERY_BLOCK" ]]; then
+ echo "first recovery block: $FIRST_RECOVERY_BLOCK"
+fi
+echo "platform: $PLATFORM/$ARCH"
+echo "tools: ${BENCHMARK_TOOLS[*]}"
+echo "profile tool: $PROFILE_TOOL"
+if [[ "$CACHE_PROFILE" == "1" ]]; then
+ echo "cache profile tool: $CACHE_PROFILE_TOOL"
+fi
echo
+probe_turbo_method() {
+ local probe_dir="$RUN_ROOT/turbo-method-probe"
+ local stdout_file="$RUN_ROOT/turbo-method-probe.stdout"
+ local stderr_file="$RUN_ROOT/turbo-method-probe.stderr"
+ rm -rf "$probe_dir"
+ mkdir -p "$probe_dir"
+
+ python3 - "$probe_dir/file.bin" <<'PY'
+import sys
+
+with open(sys.argv[1], "wb") as f:
+ for idx in range(1024):
+ f.write(bytes([(idx * 17 + 3) & 0xFF]) * 1024)
+PY
+
+ local probe_recovery_files="$RECOVERY_FILES"
+ if [[ -z "$probe_recovery_files" || "$probe_recovery_files" -lt 1 ]]; then
+ probe_recovery_files=1
+ fi
+ local -a probe_cmd=("$PAR2CMD_BIN" c -v -s1048576 -r10 "-n$probe_recovery_files")
+ if [[ -n "$FIRST_RECOVERY_BLOCK" ]]; then
+ probe_cmd+=("-f$FIRST_RECOVERY_BLOCK")
+ fi
+ if [[ "$THREADS" != "0" ]]; then
+ probe_cmd+=("-t$THREADS")
+ fi
+
+ if ! (cd "$probe_dir" && "${probe_cmd[@]}" out.par2 file.bin >"$stdout_file" 2>"$stderr_file"); then
+ rm -rf "$probe_dir"
+ printf 'unknown'
+ return 0
+ fi
+
+ local method
+ method="$(awk -F': ' '/Multiply method:/ {print $2; exit}' "$stdout_file" "$stderr_file")"
+ rm -rf "$probe_dir"
+ printf '%s' "${method:-unknown}"
+}
+
echo "Building par2rs release binary..."
(cd "$PROJECT_ROOT" && cargo build --release --bin par2 --quiet)
write_csv_header() {
local csv_file="$1"
- printf 'case,tool,run,status,validation_status,instructions,cycles,branches,branch_misses,cache_references,cache_misses,task_clock_msec,context_switches,cpu_migrations,page_faults,wall_seconds\n' > "$csv_file"
+ printf 'case,tool,selected_method,run,status,validation_status,instructions,cycles,branches,branch_misses,cache_references,cache_misses,task_clock_msec,context_switches,cpu_migrations,page_faults,wall_seconds\n' > "$csv_file"
}
write_csv_header "$RAW_CSV"
write_csv_header "$SMOKE_CSV"
+TURBO_OBSERVED_METHOD="$(probe_turbo_method)"
+echo "turbo observed method: $TURBO_OBSERVED_METHOD"
+echo
+
+now_ns() {
+ python3 - <<'PY'
+import time
+print(time.time_ns())
+PY
+}
+
split_cases() {
tr ',' '\n' <<< "$CASES"
}
+corpus_dir_for_case() {
+ local label="$1"
+ if [[ -n "$CORPUS_ROOT" ]]; then
+ printf '%s/%s/corpus\n' "$CORPUS_ROOT" "$label"
+ else
+ printf '%s/%s/corpus\n' "$WORK_ROOT" "$label"
+ fi
+}
+
make_corpus() {
- local case_dir="$1"
+ local corpus_dir="$1"
local file_count="$2"
local file_size_mib="$3"
- mkdir -p "$case_dir/corpus"
+ mkdir -p "$corpus_dir"
local i
for i in $(seq 1 "$file_count"); do
local file
- file="$case_dir/corpus/file_$(printf '%04d' "$i").bin"
+ file="$corpus_dir/file_$(printf '%04d' "$i").bin"
if [[ ! -f "$file" ]]; then
python3 - "$file" "$i" "$file_size_mib" <<'PY'
import sys
@@ -160,6 +333,9 @@ link_corpus() {
perf_value() {
local perf_file="$1"
local event="$2"
+ if [[ ! -f "$perf_file" ]]; then
+ return 0
+ fi
awk -F, -v event="$event" '
{
parsed_event = $3
@@ -192,26 +368,61 @@ run_create() {
link_corpus "$corpus_dir" "$run_dir"
local -a cmd
- if [[ "$tool" == "par2rs" ]]; then
+ local selected_method="$tool"
+ local -a env_prefix=()
+ if [[ "$tool" == par2rs-* ]]; then
+ case "$tool" in
+ par2rs-auto)
+ selected_method="auto"
+ env_prefix=()
+ ;;
+ par2rs-pshufb)
+ selected_method="pshufb"
+ env_prefix=(env PAR2RS_CREATE_GF16=pshufb)
+ ;;
+ par2rs-xor-jit|par2rs-xor-jit-port)
+ selected_method="xor-jit"
+ env_prefix=(env PAR2RS_CREATE_GF16=xor-jit PAR2RS_CREATE_XOR_JIT_FALLBACK=error)
+ ;;
+ par2rs-scalar)
+ selected_method="scalar"
+ env_prefix=(env PAR2RS_CREATE_GF16=scalar)
+ ;;
+ *)
+ echo "error: unknown par2rs variant: $tool" >&2
+ return 2
+ ;;
+ esac
cmd=("$PAR2RS_BIN" c -q -q "-s$block_size" "-r$REDUNDANCY" "-n$RECOVERY_FILES")
else
+ selected_method="auto"
cmd=("$PAR2CMD_BIN" c -q -q "-s$block_size" "-r$REDUNDANCY" "-n$RECOVERY_FILES")
fi
+ if [[ -n "$FIRST_RECOVERY_BLOCK" ]]; then
+ cmd+=("-f$FIRST_RECOVERY_BLOCK")
+ fi
if [[ "$THREADS" != "0" ]]; then
cmd+=("-t$THREADS")
fi
-
local start_ns end_ns status validation_status wall_seconds
- start_ns="$(date +%s%N)"
+ start_ns="$(now_ns)"
set +e
- (
- cd "$run_dir"
- sources=(./*.bin)
- perf stat -x, -o "$perf_file" -e "$PERF_EVENTS" -- "${cmd[@]}" out.par2 "${sources[@]}" >/dev/null
- )
+ if [[ "$PLATFORM" == "Linux" ]]; then
+ (
+ cd "$run_dir"
+ sources=(./*.bin)
+ perf stat -x, -o "$perf_file" -e "$PERF_EVENTS" -- "${env_prefix[@]}" "${cmd[@]}" out.par2 "${sources[@]}" >/dev/null
+ )
+ else
+ (
+ cd "$run_dir"
+ sources=(./*.bin)
+ "${env_prefix[@]}" "${cmd[@]}" out.par2 "${sources[@]}" >/dev/null
+ )
+ fi
status="$?"
set -e
- end_ns="$(date +%s%N)"
+ end_ns="$(now_ns)"
wall_seconds="$(awk -v s="$start_ns" -v e="$end_ns" 'BEGIN { printf "%.6f", (e - s) / 1000000000 }')"
validation_status=0
@@ -220,7 +431,7 @@ run_create() {
verifier_stdout="$RUN_ROOT/verify-$case_label-$tool-$run_number.stdout"
verifier_stderr="$RUN_ROOT/verify-$case_label-$tool-$run_number.stderr"
set +e
- if [[ "$tool" == "par2rs" ]]; then
+ if [[ "$tool" == par2rs-* ]]; then
(cd "$run_dir" && "$PAR2CMD_BIN" v -q -q out.par2 >"$verifier_stdout" 2>"$verifier_stderr")
else
(cd "$run_dir" && "$PAR2RS_BIN" verify -q -q out.par2 >"$verifier_stdout" 2>"$verifier_stderr")
@@ -233,7 +444,7 @@ run_create() {
echo "quiet validation failed with status $validation_status; rerunning verbosely"
} >> "$verifier_stderr"
set +e
- if [[ "$tool" == "par2rs" ]]; then
+ if [[ "$tool" == par2rs-* ]]; then
(cd "$run_dir" && "$PAR2CMD_BIN" v out.par2 >>"$verifier_stdout" 2>>"$verifier_stderr")
else
(cd "$run_dir" && "$PAR2RS_BIN" verify out.par2 >>"$verifier_stdout" 2>>"$verifier_stderr")
@@ -246,6 +457,24 @@ run_create() {
else
rm -f "$verifier_stdout" "$verifier_stderr"
fi
+
+ if [[ "$validation_status" == "0" ]] && should_repair_validate "$run_number"; then
+ local repair_stdout repair_stderr
+ repair_stdout="$RUN_ROOT/repair-$case_label-$tool-$run_number.stdout"
+ repair_stderr="$RUN_ROOT/repair-$case_label-$tool-$run_number.stderr"
+ set +e
+ validate_repair_output "$tool" "$run_dir" "$corpus_dir" "$block_size" "$repair_stdout" "$repair_stderr"
+ validation_status="$?"
+ set -e
+ if [[ "$validation_status" != "0" ]]; then
+ echo "error: cross-tool repair validation failed for $case_label/$tool run $run_number" >&2
+ echo "stdout: $repair_stdout" >&2
+ echo "stderr: $repair_stderr" >&2
+ echo "work dir: $run_dir" >&2
+ else
+ rm -f "$repair_stdout" "$repair_stderr"
+ fi
+ fi
fi
if [[ "$measured" == "1" ]]; then
@@ -261,8 +490,8 @@ run_create() {
context_switches="$(perf_value "$perf_file" context-switches)"
cpu_migrations="$(perf_value "$perf_file" cpu-migrations)"
page_faults="$(perf_value "$perf_file" page-faults)"
- printf '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' \
- "$case_label" "$tool" "$run_number" "$status" "$validation_status" \
+ printf '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' \
+ "$case_label" "$tool" "$selected_method" "$run_number" "$status" "$validation_status" \
"$instructions" "$cycles" "$branches" "$branch_misses" \
"$cache_references" "$cache_misses" "$task_clock" \
"$context_switches" "$cpu_migrations" "$page_faults" "$wall_seconds" >> "$output_csv"
@@ -271,6 +500,7 @@ run_create() {
if [[ "$status" == "0" && "$validation_status" == "0" ]]; then
rm -rf "$run_dir"
elif [[ "$KEEP_WORK" != "1" ]]; then
+ PRESERVE_WORK=1
echo "preserving failed run directory: $run_dir" >&2
fi
if [[ "$status" != "0" ]]; then
@@ -279,6 +509,66 @@ run_create() {
return "$validation_status"
}
+should_repair_validate() {
+ local run_number="$1"
+ case "$VERIFY_REPAIR" in
+ 1|true|yes)
+ return 0
+ ;;
+ smoke)
+ [[ "$run_number" == "smoke" ]]
+ return
+ ;;
+ *)
+ return 1
+ ;;
+ esac
+}
+
+validate_repair_output() {
+ local tool="$1"
+ local run_dir="$2"
+ local corpus_dir="$3"
+ local block_size="$4"
+ local repair_stdout="$5"
+ local repair_stderr="$6"
+
+ local target base
+ target="$(find "$run_dir" -maxdepth 1 -type f -name '*.bin' | sort | head -n 1)"
+ if [[ -z "$target" ]]; then
+ echo "no source file found to damage" > "$repair_stderr"
+ return 97
+ fi
+ base="$(basename "$target")"
+
+ cp -p "$target" "$target.detached"
+ mv "$target.detached" "$target"
+ python3 - "$target" "$block_size" <<'PY'
+import sys
+
+path = sys.argv[1]
+block_size = int(sys.argv[2])
+
+with open(path, "r+b") as f:
+ data = f.read(block_size)
+ if not data:
+ sys.exit(2)
+ f.seek(0)
+ f.write(bytes(b ^ 0xFF for b in data))
+PY
+
+ if [[ "$tool" == par2rs-* ]]; then
+ (cd "$run_dir" && "$PAR2CMD_BIN" r -q -q out.par2 >"$repair_stdout" 2>"$repair_stderr")
+ else
+ (cd "$run_dir" && "$PAR2RS_BIN" repair -q -q out.par2 >"$repair_stdout" 2>"$repair_stderr")
+ fi || return "$?"
+
+ if ! cmp -s "$corpus_dir/$base" "$run_dir/$base"; then
+ echo "repaired file differs from original corpus file: $base" >> "$repair_stderr"
+ return 98
+ fi
+}
+
summarize_results() {
local raw_csv="${1:-$RAW_CSV}"
local summary_md="${2:-$SUMMARY_MD}"
@@ -286,14 +576,14 @@ summarize_results() {
local warmups="${4:-$WARMUP_RUNS}"
local title="${5:-PAR2 Create Perf Benchmark}"
- python3 - "$raw_csv" "$summary_md" "$iterations" "$warmups" "$THREADS" "$REDUNDANCY" "$RECOVERY_FILES" "$title" <<'PY'
+ python3 - "$raw_csv" "$summary_md" "$iterations" "$warmups" "$THREADS" "$REDUNDANCY" "$RECOVERY_FILES" "$FIRST_RECOVERY_BLOCK" "$title" "$TURBO_OBSERVED_METHOD" <<'PY'
import csv
import math
import statistics
import sys
from collections import defaultdict
-raw_csv, summary_md, iterations, warmups, threads, redundancy, recovery_files, title = sys.argv[1:]
+raw_csv, summary_md, iterations, warmups, threads, redundancy, recovery_files, first_recovery_block, title, turbo_observed_method = sys.argv[1:]
metrics = [
("instructions", "instructions"),
("cycles", "cycles"),
@@ -340,6 +630,16 @@ def fmt(value):
return f"{value:,.2f}"
return f"{value:.4f}"
+def metric_mean(case, tool, key):
+ group = groups.get((case, tool), [])
+ s = stats(number(r, key) for r in group)
+ return s[0] if s else None
+
+def ratio(num, den):
+ if num is None or den is None or den == 0:
+ return None
+ return num / den
+
cases = sorted({row["case"] for row in rows})
with open(summary_md, "w") as out:
@@ -349,29 +649,49 @@ with open(summary_md, "w") as out:
out.write(f"- threads: {threads}\n")
out.write(f"- redundancy: {redundancy}%\n")
out.write(f"- recovery files: {recovery_files}\n")
- out.write("- validation: par2rs output verified by turbo; turbo output verified by par2rs\n")
- out.write("- primary signal: Linux perf `instructions`\n\n")
+ out.write(f"- first recovery block: {first_recovery_block or 'default'}\n")
+ out.write(f"- turbo observed method: {turbo_observed_method}\n")
+ out.write("- validation: par2rs output verified/repaired by turbo; turbo output verified/repaired by par2rs\n")
+ out.write("- primary signal: wall seconds; Linux perf counters are recorded for diagnostics when available\n\n")
out.write("## Relative Summary\n\n")
- out.write("| case | par2rs instructions | turbo instructions | par2rs/turbo | par2rs cycles | turbo cycles | par2rs/turbo cycles |\n")
- out.write("|---|---:|---:|---:|---:|---:|---:|\n")
+ out.write("| case | tool | method | instructions | cycles | wall seconds | effective CPU |\n")
+ out.write("|---|---|---|---:|---:|---:|---:|\n")
+ for case in cases:
+ case_tools = sorted({row["tool"] for row in rows if row["case"] == case})
+ for tool in case_tools:
+ group = groups[(case, tool)]
+ method = group[0].get("selected_method", tool) if group else tool
+ instr = stats(number(r, "instructions") for r in group)
+ cycles = stats(number(r, "cycles") for r in group)
+ wall = stats(number(r, "wall_seconds") for r in group)
+ task_clock = stats(number(r, "task_clock_msec") for r in group)
+ effective_cpu = task_clock[0] / (wall[0] * 1000) if task_clock and wall and wall[0] else None
+ out.write(
+ f"| {case} | {tool} | {method} | {fmt(instr[0] if instr else None)} | "
+ f"{fmt(cycles[0] if cycles else None)} | {fmt(wall[0] if wall else None)} | {fmt(effective_cpu)} |\n"
+ )
+
+ out.write("\n## XOR-JIT Ratios\n\n")
+ out.write("| case | wall / turbo | instructions / turbo | cache misses / turbo |\n")
+ out.write("|---|---:|---:|---:|\n")
for case in cases:
- p_instr = stats(number(r, "instructions") for r in groups[(case, "par2rs")])
- t_instr = stats(number(r, "instructions") for r in groups[(case, "turbo")])
- p_cycles = stats(number(r, "cycles") for r in groups[(case, "par2rs")])
- t_cycles = stats(number(r, "cycles") for r in groups[(case, "turbo")])
- instr_ratio = p_instr[0] / t_instr[0] if p_instr and t_instr and t_instr[0] else None
- cycle_ratio = p_cycles[0] / t_cycles[0] if p_cycles and t_cycles and t_cycles[0] else None
+ xor_wall = metric_mean(case, "par2rs-xor-jit", "wall_seconds")
+ turbo_wall = metric_mean(case, "turbo-auto", "wall_seconds")
+ xor_instr = metric_mean(case, "par2rs-xor-jit", "instructions")
+ turbo_instr = metric_mean(case, "turbo-auto", "instructions")
+ xor_cache = metric_mean(case, "par2rs-xor-jit", "cache_misses")
+ turbo_cache = metric_mean(case, "turbo-auto", "cache_misses")
out.write(
- f"| {case} | {fmt(p_instr[0] if p_instr else None)} | {fmt(t_instr[0] if t_instr else None)} | "
- f"{fmt(instr_ratio)} | {fmt(p_cycles[0] if p_cycles else None)} | {fmt(t_cycles[0] if t_cycles else None)} | {fmt(cycle_ratio)} |\n"
+ f"| {case} | {fmt(ratio(xor_wall, turbo_wall))} | "
+ f"{fmt(ratio(xor_instr, turbo_instr))} | {fmt(ratio(xor_cache, turbo_cache))} |\n"
)
out.write("\n## Detailed Stats\n\n")
out.write("| case | tool | metric | n | mean | stddev | cv % | min | max |\n")
out.write("|---|---|---|---:|---:|---:|---:|---:|---:|\n")
for case in cases:
- for tool in ("par2rs", "turbo"):
+ for tool in sorted({row["tool"] for row in rows if row["case"] == case}):
group = groups[(case, tool)]
failures = [r for r in group if r["status"] != "0"]
validation_failures = [r for r in group if r.get("validation_status", "0") != "0"]
@@ -389,6 +709,82 @@ with open(summary_md, "w") as out:
PY
}
+enforce_pass_criteria() {
+ local raw_csv="${1:-$RAW_CSV}"
+
+ if ! tool_supported "par2rs-xor-jit"; then
+ echo "Skipping XOR-JIT pass criteria on $PLATFORM/$ARCH."
+ return 0
+ fi
+
+ python3 - "$raw_csv" <<'PY'
+import csv
+import math
+import statistics
+import sys
+from collections import defaultdict
+
+raw_csv = sys.argv[1]
+required = ["par2rs-xor-jit", "turbo-auto"]
+groups = defaultdict(list)
+failures = []
+
+with open(raw_csv, newline="") as f:
+ for row in csv.DictReader(f):
+ if row.get("status") != "0" or row.get("validation_status") != "0":
+ failures.append(row)
+ groups[(row["case"], row["tool"])].append(row)
+
+def mean_wall(rows):
+ values = []
+ for row in rows:
+ try:
+ value = float(row.get("wall_seconds", ""))
+ except ValueError:
+ continue
+ if math.isfinite(value):
+ values.append(value)
+ return statistics.fmean(values) if values else None
+
+errors = []
+if failures:
+ for row in failures:
+ errors.append(
+ f"{row['case']}/{row['tool']} run {row['run']} status={row.get('status')} "
+ f"validation={row.get('validation_status')}"
+ )
+
+cases = sorted({case for case, _tool in groups})
+for case in cases:
+ missing = [tool for tool in required if not groups.get((case, tool))]
+ if missing:
+ errors.append(f"{case}: missing required tools: {', '.join(missing)}")
+ continue
+
+ turbo = mean_wall(groups[(case, "turbo-auto")])
+ if turbo is None:
+ errors.append(f"{case}: missing turbo-auto wall_seconds")
+ continue
+
+ tool = "par2rs-xor-jit"
+ wall = mean_wall(groups[(case, tool)])
+ if wall is None:
+ errors.append(f"{case}: missing {tool} wall_seconds")
+ elif wall > turbo:
+ errors.append(
+ f"{case}: {tool} mean wall {wall:.6f}s is slower than turbo-auto {turbo:.6f}s"
+ )
+
+if errors:
+ print("error: XOR-JIT performance pass criteria failed", file=sys.stderr)
+ for error in errors:
+ print(f" - {error}", file=sys.stderr)
+ sys.exit(1)
+
+print("XOR-JIT performance pass criteria passed.")
+PY
+}
+
run_smoke_benchmarks() {
echo
echo "Smoke benchmark: one measured, cross-validated run for each configured case/tool"
@@ -401,15 +797,15 @@ run_smoke_benchmarks() {
fi
local case_dir="$WORK_ROOT/$label"
- local corpus_dir="$case_dir/corpus"
+ local corpus_dir
+ corpus_dir="$(corpus_dir_for_case "$label")"
echo "Smoke preparing '$label': ${file_count}x${file_size_mib}MiB, block size ${block_size}"
- make_corpus "$case_dir" "$file_count" "$file_size_mib"
-
- echo "Smoke measured: $label / par2rs"
- run_create "$label" "par2rs" "smoke" "$block_size" "$corpus_dir" 1 "$SMOKE_CSV"
+ make_corpus "$corpus_dir" "$file_count" "$file_size_mib"
- echo "Smoke measured: $label / turbo"
- run_create "$label" "turbo" "smoke" "$block_size" "$corpus_dir" 1 "$SMOKE_CSV"
+ for tool in "${BENCHMARK_TOOLS[@]}"; do
+ echo "Smoke measured: $label / $tool"
+ run_create "$label" "$tool" "smoke" "$block_size" "$corpus_dir" 1 "$SMOKE_CSV"
+ done
done < <(split_cases)
summarize_results "$SMOKE_CSV" "$SMOKE_SUMMARY_MD" 1 0 "PAR2 Create Perf Smoke Benchmark"
@@ -476,29 +872,76 @@ for name, count in inclusive.most_common(40):
'
}
+par2rs_tool_env() {
+ local tool="$1"
+ local -a env_args=(env)
+
+ case "$tool" in
+ par2rs-auto)
+ ;;
+ par2rs-pshufb)
+ env_args+=(PAR2RS_CREATE_GF16=pshufb)
+ ;;
+ par2rs-xor-jit|par2rs-xor-jit-port)
+ env_args+=(PAR2RS_CREATE_GF16=xor-jit PAR2RS_CREATE_XOR_JIT_FALLBACK=error)
+ ;;
+ par2rs-scalar)
+ env_args+=(PAR2RS_CREATE_GF16=scalar)
+ ;;
+ *)
+ echo "error: PROFILE_TOOL must be a par2rs variant, got: $tool" >&2
+ return 1
+ ;;
+ esac
+ env_args+=(PAR2RS_XOR_JIT_PERF_MAP=1)
+ printf '%s\0' "${env_args[@]}"
+}
+
+par2rs_tool_env_args() {
+ local tool="$1"
+ local -a env_args=()
+
+ case "$tool" in
+ par2rs-auto)
+ ;;
+ par2rs-pshufb)
+ env_args+=(PAR2RS_CREATE_GF16=pshufb)
+ ;;
+ par2rs-xor-jit|par2rs-xor-jit-port)
+ env_args+=(PAR2RS_CREATE_GF16=xor-jit PAR2RS_CREATE_XOR_JIT_FALLBACK=error)
+ ;;
+ par2rs-scalar)
+ env_args+=(PAR2RS_CREATE_GF16=scalar)
+ ;;
+ *)
+ echo "error: PROFILE_TOOL must be a par2rs variant, got: $tool" >&2
+ return 1
+ ;;
+ esac
+ env_args+=(PAR2RS_XOR_JIT_PERF_MAP=1)
+ if [[ "${#env_args[@]}" -gt 0 ]]; then
+ printf '%s\0' "${env_args[@]}"
+ fi
+}
+
generate_flamegraph() {
local case_spec="$1"
IFS=: read -r label file_count file_size_mib block_size <<< "$case_spec"
local case_dir="$WORK_ROOT/$label"
- local corpus_dir="$case_dir/corpus"
- local flamegraph_file="$RUN_ROOT/par2rs-create-$label-flamegraph.svg"
+ local corpus_dir
+ corpus_dir="$(corpus_dir_for_case "$label")"
+ local flamegraph_file="$RUN_ROOT/par2rs-create-$label-$PROFILE_TOOL-flamegraph.svg"
local output_file="$case_dir/flamegraph-out.par2"
if [[ "$RUN_FLAMEGRAPH" != "1" ]]; then
return 0
fi
- if ! command -v inferno-collapse-perf >/dev/null 2>&1 || ! command -v inferno-flamegraph >/dev/null 2>&1; then
- if command -v cargo >/dev/null 2>&1; then
- echo "inferno not found; installing with cargo install inferno..."
- cargo install inferno
- else
- echo "warning: inferno is unavailable; skipping flamegraph" >&2
- return 0
- fi
- fi
rm -f "$case_dir"/flamegraph-out*.par2
local -a args=(c -q -q "-s$block_size" "-r$REDUNDANCY" "-n$RECOVERY_FILES")
+ if [[ -n "$FIRST_RECOVERY_BLOCK" ]]; then
+ args+=("-f$FIRST_RECOVERY_BLOCK")
+ fi
if [[ "$THREADS" != "0" ]]; then
args+=("-t$THREADS")
fi
@@ -508,8 +951,169 @@ generate_flamegraph() {
args+=("$file")
done
+ case "$PLATFORM" in
+ Linux)
+ if ! command -v inferno-collapse-perf >/dev/null 2>&1 || ! command -v inferno-flamegraph >/dev/null 2>&1; then
+ if command -v cargo >/dev/null 2>&1; then
+ echo "inferno not found; installing with cargo install inferno..."
+ cargo install inferno
+ else
+ echo "warning: inferno is unavailable; skipping flamegraph" >&2
+ return 0
+ fi
+ fi
+
+ echo
+ echo "Building par2rs profiling binary with frame pointers..."
+ (
+ cd "$PROJECT_ROOT"
+ RUSTFLAGS="-C target-cpu=native -C force-frame-pointers=yes" \
+ cargo build --profile profiling --bin par2 --quiet
+ )
+
+ local profiling_bin="$PROJECT_ROOT/target/profiling/par2"
+ local perf_data="$RUN_ROOT/par2rs-create-$label-$PROFILE_TOOL.perf.data"
+ local perf_summary="$RUN_ROOT/par2rs-create-$label-$PROFILE_TOOL.perf-report.txt"
+ local hotspots="$RUN_ROOT/par2rs-create-$label-$PROFILE_TOOL.hotspots.txt"
+ local -a env_prefix
+ mapfile -d '' -t env_prefix < <(par2rs_tool_env "$PROFILE_TOOL")
+
+ echo "Recording par2rs flamegraph for case '$label' / $PROFILE_TOOL..."
+ if ! perf record -g --call-graph fp -F "$PROFILE_FREQUENCY" -o "$perf_data" -- "${env_prefix[@]}" "$profiling_bin" "${args[@]}" >/dev/null; then
+ echo "warning: flamegraph generation failed; benchmark results are still available" >&2
+ return 0
+ fi
+
+ echo "Rendering flamegraph SVG..."
+ perf script -i "$perf_data" 2>/dev/null | inferno-collapse-perf | inferno-flamegraph > "$flamegraph_file"
+
+ echo "Generating perf report text..."
+ perf report -i "$perf_data" --stdio --sort comm,dso,symbol > "$perf_summary" 2>/dev/null || true
+
+ echo "Generating hotspot summary..."
+ perf script -i "$perf_data" 2>/dev/null | summarize_perf_script > "$hotspots" || true
+
+ echo "flamegraph: $flamegraph_file"
+ echo "perf data: $perf_data"
+ echo "perf text: $perf_summary"
+ echo "hotspots: $hotspots"
+ ;;
+ Darwin)
+ if ! command -v xctrace >/dev/null 2>&1; then
+ echo "warning: xctrace is unavailable; skipping flamegraph" >&2
+ return 0
+ fi
+ if ! command -v inferno-collapse-xctrace >/dev/null 2>&1; then
+ echo "warning: inferno-collapse-xctrace is unavailable; skipping flamegraph" >&2
+ return 0
+ fi
+ if ! command -v inferno-flamegraph >/dev/null 2>&1; then
+ echo "warning: inferno-flamegraph is unavailable; skipping flamegraph" >&2
+ return 0
+ fi
+
+ local -a env_vars
+ local xctrace_bin xctrace_dir developer_dir
+ local trace_file xml_file folded_file
+ mapfile -d '' -t env_vars < <(par2rs_tool_env_args "$PROFILE_TOOL")
+ xctrace_bin="$(command -v xctrace || true)"
+ if [[ -z "$xctrace_bin" ]]; then
+ xctrace_bin="$(xcrun --find xctrace 2>/dev/null || true)"
+ fi
+ if [[ -z "$xctrace_bin" ]]; then
+ echo "warning: xctrace is unavailable; skipping flamegraph" >&2
+ return 0
+ fi
+ xctrace_dir="$(dirname "$xctrace_bin")"
+ developer_dir="/Applications/Xcode.app/Contents/Developer"
+ if [[ ! -d "$developer_dir" ]]; then
+ developer_dir="$(dirname "$(dirname "$xctrace_dir")")"
+ fi
+ trace_file="$RUN_ROOT/par2rs-create-$label-$PROFILE_TOOL.trace"
+ xml_file="$RUN_ROOT/par2rs-create-$label-$PROFILE_TOOL.xml"
+ folded_file="$RUN_ROOT/par2rs-create-$label-$PROFILE_TOOL.folded"
+
+ echo
+ echo "Building par2rs profiling binary with frame pointers..."
+ (
+ cd "$PROJECT_ROOT"
+ RUSTFLAGS="-C target-cpu=native -C force-frame-pointers=yes" \
+ cargo build --profile profiling --bin par2 --quiet
+ ) || {
+ echo "warning: flamegraph generation failed; benchmark results are still available" >&2
+ return 0
+ }
+
+ local profiling_bin="$PROJECT_ROOT/target/profiling/par2"
+ local -a xctrace_cmd=(
+ "$xctrace_bin" record
+ --template "Time Profiler"
+ --output "$trace_file"
+ --target-stdout /dev/null
+ --launch --
+ "$profiling_bin"
+ )
+ local env_var
+ for env_var in "${env_vars[@]}"; do
+ xctrace_cmd+=(--env "$env_var")
+ done
+ xctrace_cmd+=("${args[@]}")
+
+ echo "Recording par2rs flamegraph for case '$label' / $PROFILE_TOOL with xctrace..."
+ env DEVELOPER_DIR="$developer_dir" PATH="$xctrace_dir:$PATH" \
+ "${xctrace_cmd[@]}" >/dev/null || {
+ echo "warning: xctrace recording failed; benchmark results are still available" >&2
+ return 0
+ }
+
+ echo "Exporting xctrace profile..."
+ env DEVELOPER_DIR="$developer_dir" PATH="$xctrace_dir:$PATH" \
+ xctrace export --input "$trace_file" \
+ --xpath '/trace-toc/*/data/table[@schema="time-profile"]' > "$xml_file" || {
+ echo "warning: xctrace export failed; benchmark results are still available" >&2
+ return 0
+ }
+
+ echo "Collapsing xctrace stacks..."
+ if ! inferno-collapse-xctrace "$xml_file" > "$folded_file"; then
+ echo "warning: xctrace stack collapse failed; benchmark results are still available" >&2
+ return 0
+ fi
+
+ echo "Rendering flamegraph SVG..."
+ if ! inferno-flamegraph "$folded_file" > "$flamegraph_file"; then
+ echo "warning: inferno flamegraph rendering failed; benchmark results are still available" >&2
+ return 0
+ fi
+
+ echo "flamegraph: $flamegraph_file"
+ echo "trace: $trace_file"
+ echo "xml: $xml_file"
+ echo "folded: $folded_file"
+ ;;
+ *)
+ echo "warning: flamegraph generation is unsupported on $PLATFORM; skipping flamegraph" >&2
+ ;;
+ esac
+}
+
+generate_cache_profile() {
+ local case_spec="$1"
+ IFS=: read -r label file_count file_size_mib block_size <<< "$case_spec"
+ local case_dir="$WORK_ROOT/$label"
+ local corpus_dir
+ corpus_dir="$(corpus_dir_for_case "$label")"
+
+ if [[ "$CACHE_PROFILE" != "1" ]]; then
+ return 0
+ fi
+ if [[ "$PLATFORM" != "Linux" ]]; then
+ echo "warning: cache profiling is currently Linux-only; skipping cache profile" >&2
+ return 0
+ fi
+
echo
- echo "Building par2rs profiling binary with frame pointers..."
+ echo "Preparing cache-miss attribution for case '$label' / $CACHE_PROFILE_TOOL..."
(
cd "$PROJECT_ROOT"
RUSTFLAGS="-C target-cpu=native -C force-frame-pointers=yes" \
@@ -517,32 +1121,118 @@ generate_flamegraph() {
)
local profiling_bin="$PROJECT_ROOT/target/profiling/par2"
- local perf_data="$RUN_ROOT/par2rs-create-$label.perf.data"
- local perf_summary="$RUN_ROOT/par2rs-create-$label.perf-report.txt"
- local hotspots="$RUN_ROOT/par2rs-create-$label.hotspots.txt"
+ local output_file="$case_dir/cache-profile-out.par2"
+ local cache_stat="$RUN_ROOT/par2rs-create-$label-$CACHE_PROFILE_TOOL.cache-stat.txt"
+ local cache_miss_data="$RUN_ROOT/par2rs-create-$label-$CACHE_PROFILE_TOOL.cache-misses.data"
+ local cache_miss_report="$RUN_ROOT/par2rs-create-$label-$CACHE_PROFILE_TOOL.cache-misses-report.txt"
+ local cache_miss_script="$RUN_ROOT/par2rs-create-$label-$CACHE_PROFILE_TOOL.cache-misses-script.txt"
+ local perf_mem_data="$RUN_ROOT/par2rs-create-$label-$CACHE_PROFILE_TOOL.perf-mem.data"
+ local perf_mem_report="$RUN_ROOT/par2rs-create-$label-$CACHE_PROFILE_TOOL.perf-mem-report.txt"
+ local precise_data="$RUN_ROOT/par2rs-create-$label-$CACHE_PROFILE_TOOL.mem-loads.data"
+ local precise_report="$RUN_ROOT/par2rs-create-$label-$CACHE_PROFILE_TOOL.mem-loads-report.txt"
+ local ibs_data="$RUN_ROOT/par2rs-create-$label-$CACHE_PROFILE_TOOL.ibs-op.data"
+ local ibs_report="$RUN_ROOT/par2rs-create-$label-$CACHE_PROFILE_TOOL.ibs-op-report.txt"
+ local ibs_script="$RUN_ROOT/par2rs-create-$label-$CACHE_PROFILE_TOOL.ibs-op-script.txt"
+ local -a env_prefix
+ local -a args=(c -q -q "-s$block_size" "-r$REDUNDANCY" "-n$RECOVERY_FILES")
+ if [[ -n "$FIRST_RECOVERY_BLOCK" ]]; then
+ args+=("-f$FIRST_RECOVERY_BLOCK")
+ fi
- echo "Recording par2rs flamegraph for case '$label'..."
- if ! perf record -g --call-graph fp -F "$PROFILE_FREQUENCY" -o "$perf_data" -- "$profiling_bin" "${args[@]}" >/dev/null; then
- echo "warning: flamegraph generation failed; benchmark results are still available" >&2
- return 0
+ mapfile -d '' -t env_prefix < <(par2rs_tool_env "$CACHE_PROFILE_TOOL")
+ if [[ "$CACHE_PROFILE_JIT_DUMPS" == "1" ]]; then
+ env_prefix+=(PAR2RS_XOR_JIT_DUMP_DIR="$RUN_ROOT/xor-jit-dumps")
+ else
+ env_prefix+=(PAR2RS_XOR_JIT_PERF_COEFF_LABELS=0)
+ fi
+ if [[ "$THREADS" != "0" ]]; then
+ args+=("-t$THREADS")
+ fi
+ args+=("$output_file")
+ local file
+ for file in "$corpus_dir"/*.bin; do
+ args+=("$file")
+ done
+
+ echo "Collecting targeted cache counters..."
+ rm -f "$case_dir"/cache-profile-out*.par2
+ if ! perf stat -o "$cache_stat" -d -d -d -e "$CACHE_STAT_EVENTS" -- "${env_prefix[@]}" "$profiling_bin" "${args[@]}" >/dev/null; then
+ echo "warning: cache perf stat failed; see $cache_stat" >&2
+ fi
+
+ echo "Recording cache-miss callgraph samples..."
+ rm -f "$case_dir"/cache-profile-out*.par2
+ if perf record -g --call-graph fp -e cache-misses:u -o "$cache_miss_data" -- "${env_prefix[@]}" "$profiling_bin" "${args[@]}" >/dev/null; then
+ perf report -i "$cache_miss_data" --stdio --sort comm,dso,symbol > "$cache_miss_report" 2>/dev/null || true
+ perf script -i "$cache_miss_data" > "$cache_miss_script" 2>/dev/null || true
+ echo "cache-misses data: $cache_miss_data"
+ echo "cache-misses text: $cache_miss_report"
+ echo "cache-misses script:$cache_miss_script"
+ else
+ echo "warning: cache-miss callgraph sampling failed; see perf permissions/counter availability" >&2
+ rm -f "$cache_miss_data"
+ fi
+
+ echo "Recording data-source samples with perf mem..."
+ rm -f "$case_dir"/cache-profile-out*.par2
+ if perf mem record -o "$perf_mem_data" --call-graph fp -- "${env_prefix[@]}" "$profiling_bin" "${args[@]}" >/dev/null; then
+ echo "Generating perf mem report..."
+ perf mem report -i "$perf_mem_data" --stdio --sort symbol,dso,mem,local_weight > "$perf_mem_report" 2>/dev/null || true
+ echo "perf mem data: $perf_mem_data"
+ echo "perf mem report: $perf_mem_report"
+ else
+ echo "warning: perf mem record failed; falling back to precise mem-load sampling" >&2
+ rm -f "$perf_mem_data"
fi
- echo "Rendering flamegraph SVG..."
- perf script -i "$perf_data" 2>/dev/null | inferno-collapse-perf | inferno-flamegraph > "$flamegraph_file"
+ echo "Recording high-latency load samples..."
+ rm -f "$case_dir"/cache-profile-out*.par2
+ if perf record -g --call-graph fp -e "cpu/mem-loads,ldlat=${CACHE_LOAD_LATENCY}/P" -o "$precise_data" -- "${env_prefix[@]}" "$profiling_bin" "${args[@]}" >/dev/null; then
+ echo "Generating high-latency load report..."
+ perf report -i "$precise_data" --stdio --sort comm,dso,symbol > "$precise_report" 2>/dev/null || true
+ echo "mem-loads data: $precise_data"
+ echo "mem-loads text: $precise_report"
+ else
+ echo "warning: precise mem-load sampling failed; trying AMD IBS op sampling" >&2
+ rm -f "$precise_data"
+ if [[ "$(perf list 2>/dev/null)" == *ibs_op* ]]; then
+ local old_perf_paranoid=""
+ if [[ -r /proc/sys/kernel/perf_event_paranoid ]]; then
+ old_perf_paranoid="$(cat /proc/sys/kernel/perf_event_paranoid)"
+ fi
+ if [[ -n "$old_perf_paranoid" && "$old_perf_paranoid" != "-1" ]]; then
+ if sudo -n true >/dev/null 2>&1; then
+ sudo -n sysctl -w kernel.perf_event_paranoid=-1 >/dev/null || true
+ else
+ echo "warning: AMD IBS needs system-wide perf access; sudo -n is unavailable" >&2
+ fi
+ fi
- echo "Generating perf report text..."
- perf report -i "$perf_data" --stdio --sort comm,dso,symbol > "$perf_summary" 2>/dev/null || true
+ rm -f "$case_dir"/cache-profile-out*.par2
+ if perf record -g --call-graph fp -e ibs_op// -o "$ibs_data" -- "${env_prefix[@]}" "$profiling_bin" "${args[@]}" >/dev/null; then
+ perf report -i "$ibs_data" --stdio --sort comm,dso,symbol > "$ibs_report" 2>/dev/null || true
+ perf script -i "$ibs_data" > "$ibs_script" 2>/dev/null || true
+ echo "ibs-op data: $ibs_data"
+ echo "ibs-op text: $ibs_report"
+ echo "ibs-op script: $ibs_script"
+ else
+ echo "warning: AMD IBS op sampling failed; see perf permissions and IBS PMU support" >&2
+ rm -f "$ibs_data"
+ fi
- echo "Generating hotspot summary..."
- perf script -i "$perf_data" 2>/dev/null | summarize_perf_script > "$hotspots" || true
+ if [[ -n "$old_perf_paranoid" && "$(cat /proc/sys/kernel/perf_event_paranoid 2>/dev/null || true)" != "$old_perf_paranoid" ]]; then
+ sudo -n sysctl -w kernel.perf_event_paranoid="$old_perf_paranoid" >/dev/null || true
+ fi
+ else
+ echo "warning: precise mem-load sampling failed; event may be unsupported on this CPU/kernel" >&2
+ fi
+ fi
- echo "flamegraph: $flamegraph_file"
- echo "perf data: $perf_data"
- echo "perf text: $perf_summary"
- echo "hotspots: $hotspots"
+ echo "cache stat: $cache_stat"
}
last_case=""
+profile_case_spec=""
run_smoke_benchmarks
while IFS= read -r case_spec; do
@@ -554,12 +1244,15 @@ while IFS= read -r case_spec; do
fi
last_case="$case_spec"
+ if [[ -n "$PROFILE_CASE" && ( "$PROFILE_CASE" == "$label" || "$PROFILE_CASE" == "$case_spec" ) ]]; then
+ profile_case_spec="$case_spec"
+ fi
case_dir="$WORK_ROOT/$label"
- corpus_dir="$case_dir/corpus"
+ corpus_dir="$(corpus_dir_for_case "$label")"
echo "Preparing case '$label': ${file_count}x${file_size_mib}MiB, block size ${block_size}"
- make_corpus "$case_dir" "$file_count" "$file_size_mib"
+ make_corpus "$corpus_dir" "$file_count" "$file_size_mib"
- for tool in par2rs turbo; do
+ for tool in "${BENCHMARK_TOOLS[@]}"; do
echo "Warmup: $label / $tool"
for run in $(seq 1 "$WARMUP_RUNS"); do
run_create "$label" "$tool" "warmup-$run" "$block_size" "$corpus_dir" 0
@@ -575,9 +1268,18 @@ while IFS= read -r case_spec; do
done < <(split_cases)
summarize_results
-if [[ -n "$last_case" ]]; then
- generate_flamegraph "$last_case"
+if [[ -n "$PROFILE_CASE" && -z "$profile_case_spec" ]]; then
+ echo "error: PROFILE_CASE '$PROFILE_CASE' did not match any CASES label or spec" >&2
+ exit 1
+fi
+if [[ -z "$profile_case_spec" ]]; then
+ profile_case_spec="$last_case"
+fi
+if [[ -n "$profile_case_spec" ]]; then
+ generate_flamegraph "$profile_case_spec"
+ generate_cache_profile "$profile_case_spec"
fi
+enforce_pass_criteria "$RAW_CSV"
echo
echo "raw data: $RAW_CSV"
diff --git a/scripts/profile_create_slow_paths.sh b/scripts/profile_create_slow_paths.sh
new file mode 100755
index 00000000..26416380
--- /dev/null
+++ b/scripts/profile_create_slow_paths.sh
@@ -0,0 +1,358 @@
+#!/usr/bin/env bash
+# Profile representative par2rs create workloads against ParPar-backed comparison paths.
+#
+# The default comparison target is par2cmdline-turbo, which embeds ParPar's
+# create backend. Set PARPAR_BIN to also run a standalone ParPar-compatible CLI
+# if one is available in your environment.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
+PROJECT_ROOT="${PROJECT_ROOT:-$(git -C "$SCRIPT_DIR/.." rev-parse --show-toplevel)}"
+PAR2RS_BIN="${PAR2RS_BIN:-$PROJECT_ROOT/target/release/par2}"
+PAR2CMD_BIN="${PAR2CMD_BIN:-par2}"
+PARPAR_BIN="${PARPAR_BIN:-}"
+RESULTS_ROOT="${RESULTS_ROOT:-$PROJECT_ROOT/target/perf-results/create-slow-paths}"
+WORK_ROOT="${WORK_ROOT:-}"
+ITERATIONS="${ITERATIONS:-3}"
+WARMUP_RUNS="${WARMUP_RUNS:-1}"
+THREADS="${THREADS:-1}"
+RECOVERY_FILES="${RECOVERY_FILES:-4}"
+KEEP_WORK="${KEEP_WORK:-0}"
+RUN_KERNEL_BENCH="${RUN_KERNEL_BENCH:-0}"
+TRACE_IO="${TRACE_IO:-0}"
+PERF_EVENTS="${PERF_EVENTS:-instructions,cache-misses,branches,branch-misses,task-clock,context-switches}"
+DATE_BIN="${DATE_BIN:-date}"
+
+# label:file_count:file_size_bytes:block_size:recovery_percent:memory_mb
+DEFAULT_WORKLOADS="small_full_block:1:8388608:1048576:10:0,large_capped:1:100663296:67108864:10:0,many_small_files:128:65536:65536:10:0,one_large_file:1:268435456:1048576:10:0,high_recovery_count:1:33554432:1048576:50:0,forced_slim_low_memory:1:33554432:8388608:10:16"
+WORKLOADS="${WORKLOADS:-$DEFAULT_WORKLOADS}"
+
+usage() {
+ cat </dev/null 2>&1; then
+ echo "error: required tool not found: $tool" >&2
+ exit 1
+ fi
+}
+
+need_tool perl
+need_tool cargo
+need_tool perf
+need_tool "$PAR2CMD_BIN"
+
+if ! perf stat -x, -e instructions -- true >/dev/null 2>&1; then
+ echo "error: perf cannot read hardware counters in this environment" >&2
+ exit 1
+fi
+
+mkdir -p "$RESULTS_ROOT"
+RUN_ID="$("$DATE_BIN" +%Y%m%d_%H%M%S)"
+RUN_ROOT="$RESULTS_ROOT/run-$RUN_ID"
+WORK_ROOT="${WORK_ROOT:-$RUN_ROOT/work}"
+RAW_CSV="$RUN_ROOT/raw.csv"
+COUNTERS_CSV="$RUN_ROOT/counters.csv"
+PHASE_CSV="$RUN_ROOT/phase.csv"
+SUMMARY_MD="$RUN_ROOT/summary.md"
+mkdir -p "$RUN_ROOT" "$WORK_ROOT"
+
+cleanup() {
+ if [[ "$KEEP_WORK" != "1" ]]; then
+ rm -rf "$WORK_ROOT"
+ fi
+}
+trap cleanup EXIT
+
+echo "Building par2rs release binary..."
+cargo build --manifest-path "$PROJECT_ROOT/Cargo.toml" --release --bin par2 --quiet
+
+printf 'case,tool,iteration,kind,wall_seconds,exit_status,perf_log,profile_log,io_log,work_dir\n' >"$RAW_CSV"
+printf 'case,tool,iteration,wall_seconds,instructions,cache_misses,branches,branch_misses,task_clock_ms,context_switches\n' >"$COUNTERS_CSV"
+printf 'case,tool,iteration,section,name,value\n' >"$PHASE_CSV"
+
+make_corpus() {
+ local corpus_dir="$1"
+ local file_count="$2"
+ local file_size="$3"
+ rm -rf "$corpus_dir"
+ mkdir -p "$corpus_dir"
+ perl - "$corpus_dir" "$file_count" "$file_size" <<'PL'
+use strict;
+use warnings;
+use bytes;
+
+my ($root, $file_count, $file_size) = @ARGV;
+my $chunk_size = 1024 * 1024;
+
+for my $file_idx (0 .. $file_count - 1) {
+ my $path = sprintf "%s/file_%05d.bin", $root, $file_idx;
+ open my $fh, ">:raw", $path or die "open $path: $!";
+ my $remaining = $file_size;
+ my $offset = 0;
+ while ($remaining > 0) {
+ my $n = $remaining < $chunk_size ? $remaining : $chunk_size;
+ my $data = pack "C*", map { ($file_idx * 131 + $offset + $_) & 0xFF } 0 .. $n - 1;
+ print {$fh} $data or die "write $path: $!";
+ $remaining -= $n;
+ $offset += $n;
+ }
+ close $fh or die "close $path: $!";
+}
+PL
+}
+
+source_files_for() {
+ local corpus_dir="$1"
+ find "$corpus_dir" -type f -name '*.bin' -print | sort
+}
+
+run_create() {
+ local case_label="$1"
+ local tool="$2"
+ local iteration="$3"
+ local kind="$4"
+ local output_dir="$5"
+ local corpus_dir="$6"
+ local block_size="$7"
+ local redundancy="$8"
+ local memory_mb="$9"
+
+ rm -rf "$output_dir"
+ mkdir -p "$output_dir"
+ local output_base="$output_dir/out.par2"
+ local perf_log="$output_dir/perf.log"
+ local profile_log="$output_dir/profile.log"
+ local io_log="$output_dir/io.log"
+ local -a sources
+ mapfile -t sources < <(source_files_for "$corpus_dir")
+
+ local -a cmd env_prefix
+ env_prefix=()
+ case "$tool" in
+ par2rs)
+ env_prefix=(env PAR2RS_CREATE_PROFILE=1)
+ cmd=("$PAR2RS_BIN" c -q "-s$block_size" "-r$redundancy" "-n$RECOVERY_FILES")
+ ;;
+ turbo-parpar)
+ cmd=("$PAR2CMD_BIN" c -q "-s$block_size" "-r$redundancy" "-n$RECOVERY_FILES")
+ ;;
+ parpar)
+ if [[ -z "$PARPAR_BIN" ]]; then
+ return 0
+ fi
+ cmd=("$PARPAR_BIN" c -q "-s$block_size" "-r$redundancy" "-n$RECOVERY_FILES")
+ ;;
+ *)
+ echo "error: unknown tool '$tool'" >&2
+ exit 1
+ ;;
+ esac
+ if [[ "$THREADS" != "0" ]]; then
+ cmd+=("-t$THREADS")
+ fi
+ if [[ "$memory_mb" != "0" ]]; then
+ cmd+=("-m$memory_mb")
+ fi
+ cmd+=("$output_base")
+ cmd+=("${sources[@]}")
+
+ local start_ns end_ns status wall
+ start_ns="$("$DATE_BIN" +%s%N)"
+ set +e
+ "${env_prefix[@]}" perf stat -x, -e "$PERF_EVENTS" -o "$perf_log" -- "${cmd[@]}" >"$output_dir/stdout.log" 2>"$profile_log"
+ status=$?
+ set -e
+ end_ns="$("$DATE_BIN" +%s%N)"
+ wall="$(perl -e 'printf "%.9f\n", ($ARGV[1] - $ARGV[0]) / 1_000_000_000' "$start_ns" "$end_ns")"
+
+ if [[ "$TRACE_IO" == "1" && "$kind" == "measured" ]] && command -v strace >/dev/null 2>&1; then
+ rm -rf "$output_dir/io-sample"
+ mkdir -p "$output_dir/io-sample"
+ local io_output="$output_dir/io-sample/out.par2"
+ local -a io_cmd=("${cmd[@]}")
+ local output_idx=$(( ${#io_cmd[@]} - ${#sources[@]} - 1 ))
+ io_cmd[$output_idx]="$io_output"
+ strace -qq -c -e trace=read,pread64,lseek,write,pwrite64 -o "$io_log" "${env_prefix[@]}" "${io_cmd[@]}" >/dev/null 2>&1 || true
+ fi
+
+ printf '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' \
+ "$case_label" "$tool" "$iteration" "$kind" "$wall" "$status" \
+ "$perf_log" "$profile_log" "$io_log" "$output_dir" >>"$RAW_CSV"
+
+ if [[ "$kind" == "measured" ]]; then
+ perl - "$case_label" "$tool" "$iteration" "$wall" "$perf_log" "$profile_log" "$COUNTERS_CSV" "$PHASE_CSV" <<'PL'
+use strict;
+use warnings;
+
+my ($case_label, $tool, $iteration, $wall, $perf_log, $profile_log, $counters_csv, $phase_csv) = @ARGV;
+my %events = map { $_ => "" } qw(instructions cache-misses branches branch-misses task-clock context-switches);
+
+if (open my $perf, "<", $perf_log) {
+ while (my $line = <$perf>) {
+ chomp $line;
+ my @row = split /,/, $line;
+ next unless @row >= 3;
+ if (exists $events{$row[2]}) {
+ $row[0] =~ s/,//g;
+ $events{$row[2]} = $row[0];
+ }
+ }
+ close $perf;
+}
+
+open my $counters, ">>", $counters_csv or die "open $counters_csv: $!";
+print {$counters} join(",", $case_label, $tool, $iteration, $wall,
+ @events{qw(instructions cache-misses branches branch-misses task-clock context-switches)}
+) . "\n";
+close $counters;
+
+if (open my $profile, "<", $profile_log) {
+ open my $phase, ">>", $phase_csv or die "open $phase_csv: $!";
+ my $in_profile = 0;
+ my $section = "";
+ while (my $line = <$profile>) {
+ chomp $line;
+ if ($line eq "PAR2RS_CREATE_PROFILE_BEGIN") {
+ $in_profile = 1;
+ next;
+ }
+ last if $line eq "PAR2RS_CREATE_PROFILE_END";
+ next unless $in_profile;
+ if ($line eq "phase,seconds") {
+ $section = "phase";
+ next;
+ }
+ if ($line eq "counter,value") {
+ $section = "counter";
+ next;
+ }
+ next unless index($line, ",") >= 0;
+ my ($name, $value) = split /,/, $line, 2;
+ print {$phase} join(",", $case_label, $tool, $iteration, $section, $name, $value) . "\n";
+ }
+ close $phase;
+ close $profile;
+}
+PL
+ fi
+}
+
+IFS=',' read -r -a workload_specs <<<"$WORKLOADS"
+tools=(par2rs turbo-parpar)
+if [[ -n "$PARPAR_BIN" ]]; then
+ tools+=(parpar)
+fi
+
+for spec in "${workload_specs[@]}"; do
+ IFS=':' read -r label file_count file_size block_size redundancy memory_mb <<<"$spec"
+ corpus_dir="$WORK_ROOT/$label/corpus"
+ echo "Preparing workload $label..."
+ make_corpus "$corpus_dir" "$file_count" "$file_size"
+
+ for tool in "${tools[@]}"; do
+ for ((i = 1; i <= WARMUP_RUNS; i++)); do
+ run_create "$label" "$tool" "$i" "warmup" "$WORK_ROOT/$label/$tool/warmup-$i" "$corpus_dir" "$block_size" "$redundancy" "$memory_mb"
+ done
+ for ((i = 1; i <= ITERATIONS; i++)); do
+ echo "Running $label / $tool / iteration $i..."
+ run_create "$label" "$tool" "$i" "measured" "$WORK_ROOT/$label/$tool/run-$i" "$corpus_dir" "$block_size" "$redundancy" "$memory_mb"
+ done
+ done
+done
+
+perl - "$COUNTERS_CSV" "$PHASE_CSV" "$SUMMARY_MD" <<'PL'
+use strict;
+use warnings;
+
+my ($counters_csv, $phase_csv, $summary_md) = @ARGV;
+my %wall;
+my %phase;
+
+sub median {
+ my @values = sort { $a <=> $b } @_;
+ return 0 unless @values;
+ my $mid = int(@values / 2);
+ return @values % 2 ? $values[$mid] : ($values[$mid - 1] + $values[$mid]) / 2;
+}
+
+if (open my $counters, "<", $counters_csv) {
+ my $header = <$counters>;
+ while (my $line = <$counters>) {
+ chomp $line;
+ my @row = split /,/, $line;
+ next unless @row >= 4 && $row[3] ne "";
+ push @{ $wall{"$row[0]\0$row[1]"} }, $row[3] + 0;
+ }
+ close $counters;
+}
+
+if (open my $ph, "<", $phase_csv) {
+ my $header = <$ph>;
+ while (my $line = <$ph>) {
+ chomp $line;
+ my @row = split /,/, $line;
+ next unless @row >= 6;
+ next unless $row[3] eq "phase" && $row[1] eq "par2rs";
+ push @{ $phase{"$row[0]\0$row[4]"} }, $row[5] + 0;
+ }
+ close $ph;
+}
+
+open my $out, ">", $summary_md or die "open $summary_md: $!";
+print {$out} "# par2rs create slow-path profile\n\n";
+print {$out} "## Median wall time\n\n";
+print {$out} "| workload | tool | median seconds |\n";
+print {$out} "| --- | ---: | ---: |\n";
+for my $key (sort keys %wall) {
+ my ($case, $tool) = split /\0/, $key;
+ printf {$out} "| %s | %s | %.6f |\n", $case, $tool, median(@{ $wall{$key} });
+}
+print {$out} "\n## par2rs median phase time\n\n";
+print {$out} "| workload | phase | median seconds |\n";
+print {$out} "| --- | ---: | ---: |\n";
+for my $key (sort keys %phase) {
+ my ($case, $name) = split /\0/, $key;
+ printf {$out} "| %s | %s | %.6f |\n", $case, $name, median(@{ $phase{$key} });
+}
+close $out;
+PL
+
+if [[ "$RUN_KERNEL_BENCH" == "1" ]]; then
+ cargo bench --manifest-path "$PROJECT_ROOT/Cargo.toml" --features parpar-compare --bench compare_with_parpar -- --save-baseline parpar-create-slow-paths
+fi
+
+echo "Results written to $RUN_ROOT"
+echo "Summary: $SUMMARY_MD"
diff --git a/scripts/turbo_dump_xor_finish_packed_avx2.c b/scripts/turbo_dump_xor_finish_packed_avx2.c
new file mode 100644
index 00000000..7e4eb184
--- /dev/null
+++ b/scripts/turbo_dump_xor_finish_packed_avx2.c
@@ -0,0 +1,50 @@
+#include
+#include
+#include
+#include
+
+#define PARPAR_INCLUDE_BASIC_OPS
+#include "../../par2cmdline-turbo/parpar/gf16/gf16_xor_avx2.c"
+
+static void fill_pattern(uint8_t* dst, size_t len) {
+ for(size_t i = 0; i < len; i++) {
+ dst[i] = (uint8_t)((i * 29u + 7u) & 0xffu);
+ }
+}
+
+int main(int argc, char** argv) {
+ const char* output_path = argc > 1 ? argv[1] : NULL;
+ size_t slice_len = argc > 2 ? strtoull(argv[2], NULL, 0) : 1024 * 1024;
+ size_t chunk_len = argc > 3 ? strtoull(argv[3], NULL, 0) : 128 * 1024;
+ unsigned num_outputs = argc > 4 ? (unsigned)strtoul(argv[4], NULL, 0) : 7;
+ unsigned output_num = argc > 5 ? (unsigned)strtoul(argv[5], NULL, 0) : 3;
+
+ size_t segment_count = (slice_len + chunk_len - 1) / chunk_len;
+ size_t prepared_len = segment_count * num_outputs * chunk_len;
+ uint8_t* prepared = (uint8_t*)calloc(prepared_len, 1);
+ uint8_t* input = (uint8_t*)malloc(slice_len);
+ uint8_t* output = (uint8_t*)malloc(slice_len);
+ if(!prepared || !input || !output) {
+ fprintf(stderr, "allocation failed\n");
+ return 1;
+ }
+
+ fill_pattern(input, slice_len);
+ gf16_xor_prepare_packed_avx2(prepared, input, slice_len, slice_len, num_outputs, output_num, chunk_len);
+
+ gf16_xor_finish_packed_avx2(output, prepared, slice_len, num_outputs, output_num, chunk_len);
+
+ FILE* fp = output_path ? fopen(output_path, "wb") : stdout;
+ if(!fp) {
+ perror("fopen");
+ return 1;
+ }
+ fwrite(output, 1, slice_len, fp);
+ if(output_path)
+ fclose(fp);
+
+ free(output);
+ free(input);
+ free(prepared);
+ return 0;
+}
diff --git a/scripts/turbo_dump_xor_prepare_packed_avx2.c b/scripts/turbo_dump_xor_prepare_packed_avx2.c
new file mode 100644
index 00000000..95b82ff3
--- /dev/null
+++ b/scripts/turbo_dump_xor_prepare_packed_avx2.c
@@ -0,0 +1,48 @@
+#include
+#include
+#include
+#include
+
+#define PARPAR_INCLUDE_BASIC_OPS
+#include "../../par2cmdline-turbo/parpar/gf16/gf16_xor_avx2.c"
+
+static void fill_pattern(uint8_t* dst, size_t len) {
+ for(size_t i = 0; i < len; i++) {
+ dst[i] = (uint8_t)((i * 37u + 11u) & 0xffu);
+ }
+}
+
+int main(int argc, char** argv) {
+ const char* output_path = argc > 1 ? argv[1] : NULL;
+ size_t src_len = argc > 2 ? strtoull(argv[2], NULL, 0) : 1024 * 1024;
+ size_t slice_len = argc > 3 ? strtoull(argv[3], NULL, 0) : 1024 * 1024;
+ size_t chunk_len = argc > 4 ? strtoull(argv[4], NULL, 0) : 128 * 1024;
+ unsigned input_pack_size = argc > 5 ? (unsigned)strtoul(argv[5], NULL, 0) : 12;
+ unsigned input_num = argc > 6 ? (unsigned)strtoul(argv[6], NULL, 0) : 5;
+
+ size_t segment_count = (slice_len + chunk_len - 1) / chunk_len;
+ size_t dst_len = segment_count * input_pack_size * chunk_len;
+
+ uint8_t* src = (uint8_t*)malloc(src_len);
+ uint8_t* dst = (uint8_t*)calloc(dst_len, 1);
+ if(!src || !dst) {
+ fprintf(stderr, "allocation failed\n");
+ return 1;
+ }
+
+ fill_pattern(src, src_len);
+ gf16_xor_prepare_packed_avx2(dst, src, src_len, slice_len, input_pack_size, input_num, chunk_len);
+
+ FILE* fp = output_path ? fopen(output_path, "wb") : stdout;
+ if(!fp) {
+ perror("fopen");
+ return 1;
+ }
+ fwrite(dst, 1, dst_len, fp);
+ if(output_path)
+ fclose(fp);
+
+ free(dst);
+ free(src);
+ return 0;
+}
diff --git a/scripts/turbo_dump_xorjit_body_avx2.c b/scripts/turbo_dump_xorjit_body_avx2.c
new file mode 100644
index 00000000..88ac235e
--- /dev/null
+++ b/scripts/turbo_dump_xorjit_body_avx2.c
@@ -0,0 +1,51 @@
+#include
+#include
+#include
+
+#include "../../par2cmdline-turbo/parpar/gf16/gf16_xor_avx2.c"
+
+int main(int argc, char** argv) {
+ uint16_t coefficient = (uint16_t)strtoul(argc > 1 ? argv[1] : "0xc814", NULL, 0);
+ int prefetch = argc > 2 ? atoi(argv[2]) : 0;
+ const char* output_path = argc > 3 ? argv[3] : NULL;
+
+ struct gf16_xor_scratch* scratch =
+ (struct gf16_xor_scratch*)gf16_xor_jit_init_avx2(0x1100b, GF16_XOR_JIT_STRAT_NONE);
+ jit_wx_pair* jit = (jit_wx_pair*)gf16_xor_jit_init_mut_avx2();
+ if(!scratch || !jit) {
+ fprintf(stderr, "failed to initialize turbo xor-jit scratch\n");
+ return 1;
+ }
+
+ uint8_t* body_start = (uint8_t*)jit->w + scratch->codeStart;
+ uint8_t* end =
+ xor_write_jit_avx(scratch, body_start, coefficient, XORDEP_JIT_MODE_MULADD, prefetch ? _MM_HINT_T1 : 0);
+ write32(end, (int32_t)((uint8_t*)jit->w - end - 4));
+ end[4] = 0xC3;
+ end += 5;
+
+ size_t total_len = (size_t)(end - (uint8_t*)jit->w);
+ size_t dynamic_len = (size_t)(end - body_start);
+ fprintf(
+ stderr,
+ "turbo xor-jit coeff=%#06x prefetch=%d code_start=%u total_len=%zu dynamic_len=%zu\n",
+ coefficient,
+ prefetch,
+ (unsigned)scratch->codeStart,
+ total_len,
+ dynamic_len
+ );
+
+ FILE* fp = output_path ? fopen(output_path, "wb") : stdout;
+ if(!fp) {
+ perror("fopen");
+ return 1;
+ }
+ fwrite(jit->w, 1, total_len, fp);
+ if(output_path)
+ fclose(fp);
+
+ jit_free(jit);
+ ALIGN_FREE(scratch);
+ return 0;
+}
diff --git a/src/checksum.rs b/src/checksum.rs
index 422cecb9..b91c3867 100644
--- a/src/checksum.rs
+++ b/src/checksum.rs
@@ -82,15 +82,15 @@ pub fn compute_crc32_padded(data: &[u8], block_size: usize) -> Crc32Value {
// Combined MD5 + CRC32 (Performance Optimization)
// ============================================================================
-/// Compute both MD5 hash and CRC32 checksum in a single pass
+/// Compute both MD5 hash and CRC32 checksum with cache-local chunking.
///
/// This is more efficient than calling `compute_md5()` and `compute_crc32()`
-/// separately, as it only reads the data once and processes it in parallel.
+/// separately, as each chunk is fed to both hashers while still hot in cache.
///
/// PAR2 frequently needs both checksums for block verification (CRC32 for
/// fast pre-screening, MD5 for cryptographic verification).
///
-/// Uses simultaneous computation for optimal performance (40-60% faster than separate calls).
+/// Uses paired chunk updates for better cache locality than separate calls.
#[inline]
pub fn compute_block_checksums(data: &[u8]) -> (Md5Hash, Crc32Value) {
compute_md5_crc32_simultaneous(data)
@@ -107,35 +107,88 @@ pub fn compute_block_checksums_padded(data: &[u8], block_size: usize) -> (Md5Has
compute_md5_crc32_simultaneous_padded(data, block_size)
}
-/// Compute MD5 and CRC32 simultaneously in a single pass (par2cmdline style)
+/// Sub-slice size used by the fused MD5+CRC32 stream updater.
///
-/// This is the most efficient way to compute both checksums as it:
-/// - Reads data only once (50% less memory bandwidth)
-/// - Processes data while still in CPU cache
-/// - Updates both hash states in the same loop
+/// Sized to fit comfortably in L1d (typical 32-48 KiB) while remaining large
+/// enough to amortize the per-slice call overhead of the underlying SIMD
+/// compress routines in `md-5` and `crc32fast`.
+const FUSED_HASH_CHUNK: usize = 16 * 1024;
+
+/// Update an MD5 hasher and a CRC32 hasher over the same buffer with
+/// cache-resident sub-slices.
///
-/// Based on par2cmdline-turbo's MD5CRC_Calc implementation which showed
-/// ~40-60% performance improvement over separate computation.
+/// Both `md-5` and `crc32fast` use vectorized compress loops internally, so we
+/// can't literally interleave their inner instructions from Rust. What we *can*
+/// do is feed each cache-line-sized region to both hashers back-to-back, so the
+/// CRC pass reads bytes that the MD5 pass just pulled into L1 (or vice versa).
+/// This avoids the worst case where a large `data` buffer is scanned end-to-end
+/// by MD5 (evicting itself from L1), then scanned end-to-end again by CRC32 and
+/// re-fetched from L2/L3.
///
-/// # Performance
+/// Mirrors the design intent of par2cmdline-turbo's `HasherInput::update`,
+/// which fuses MD5 and CRC32 updates per 64-byte block via hand-tuned ASM.
+/// Without 2-lane MD5 we can't match the ILP, but we can still kill the
+/// double-scan memory traffic.
+#[inline]
+pub fn update_md5_crc32_fused(
+ md5_hasher: &mut D,
+ crc_hasher: &mut crc32fast::Hasher,
+ data: &[u8],
+) {
+ for chunk in data.chunks(FUSED_HASH_CHUNK) {
+ md5_hasher.update(chunk);
+ crc_hasher.update(chunk);
+ }
+}
+
+/// Update three hashers (file-MD5, block-MD5, block-CRC32) over the same
+/// buffer with cache-resident sub-slices.
+///
+/// Used by the create path's encode_and_hash_files where each chunk is
+/// simultaneously contributing to a per-file rolling MD5 and the current
+/// block's MD5+CRC32. Without a 2-lane MD5 (md5x2) we still pay two MD5
+/// compress passes, but we keep the data hot between all three hash updates.
+///
+/// Reference: par2cmdline-turbo/parpar/hasher/hasher_input_base.h
+/// (HasherInput::update fuses block-MD5 lane, file-MD5 lane and CRC32 per 64B)
+#[inline]
+pub fn update_file_md5_block_md5_crc32_fused(
+ file_md5_hasher: &mut D,
+ block_md5_hasher: &mut D,
+ crc_hasher: &mut crc32fast::Hasher,
+ data: &[u8],
+) {
+ for chunk in data.chunks(FUSED_HASH_CHUNK) {
+ block_md5_hasher.update(chunk);
+ crc_hasher.update(chunk);
+ file_md5_hasher.update(chunk);
+ }
+}
+
+/// Compute MD5 and CRC32 from one source-buffer traversal using cache-local chunking.
+///
+/// This helper operates on an in-memory slice, not directly on a file handle.
+/// The create path can still use it as part of a single-read file pipeline, but
+/// at this layer we are simply feeding two independent hashers from the same
+/// chunk stream while the data is still hot in cache.
///
-/// For 1MB blocks:
-/// - Separate: 2 passes through data, ~poor cache reuse
-/// - Simultaneous: 1 pass through data, excellent cache reuse
+/// Based on par2cmdline-turbo's MD5CRC_Calc-style chunking approach.
///
-/// Expected speedup: 1.5-2x for block verification workloads
+/// # Performance
+///
+/// For large blocks:
+/// - Separate whole-buffer passes: weaker cache locality between hashers
+/// - Chunked paired updates: better cache reuse across both hash states
#[inline]
pub fn compute_md5_crc32_simultaneous(data: &[u8]) -> (Md5Hash, Crc32Value) {
use crc32fast::Hasher as Crc32Hasher;
- use md5::Digest;
let mut md5_hasher = Md5::new();
let mut crc_hasher = Crc32Hasher::new();
- // Process data in a single pass, updating both hash states
- // This keeps data hot in CPU cache for both operations
- md5_hasher.update(data);
- crc_hasher.update(data);
+ // Process data in cache-resident sub-slices so neither hasher evicts
+ // the other's working set from L1.
+ update_md5_crc32_fused(&mut md5_hasher, &mut crc_hasher, data);
(
Md5Hash::new(md5_hasher.finalize().into()),
@@ -143,13 +196,14 @@ pub fn compute_md5_crc32_simultaneous(data: &[u8]) -> (Md5Hash, Crc32Value) {
)
}
-/// Compute MD5 and CRC32 simultaneously with zero-padding (par2cmdline style)
+/// Compute MD5 and CRC32 over one padded buffer using the same chunked updates.
///
/// For partial blocks at the end of files, this computes both checksums
-/// with zero-padding to the target block size, all in a single pass.
+/// with zero-padding to the target block size from one temporary buffer.
///
/// This is significantly more efficient than the padded separate version
-/// because it only allocates and processes the padded buffer once.
+/// because it only allocates the padded buffer once and then reuses cache-local
+/// chunking for both hashers.
///
/// # Arguments
///
@@ -159,17 +213,14 @@ pub fn compute_md5_crc32_simultaneous(data: &[u8]) -> (Md5Hash, Crc32Value) {
/// # Performance
///
/// For partial blocks:
-/// - Old: allocate padding, compute MD5, compute CRC32 (3 operations)
-/// - New: allocate padding, compute both simultaneously (2 operations)
-///
-/// Expected speedup: 1.3-1.5x for partial block processing
+/// - Separate padded hashing: allocate once, then run distinct whole-buffer hashes
+/// - Chunked paired updates: allocate once, then update both hashers per chunk
#[inline]
pub fn compute_md5_crc32_simultaneous_padded(
data: &[u8],
target_size: usize,
) -> (Md5Hash, Crc32Value) {
use crc32fast::Hasher as Crc32Hasher;
- use md5::Digest;
if data.len() >= target_size {
// No padding needed, use direct simultaneous computation
@@ -183,9 +234,8 @@ pub fn compute_md5_crc32_simultaneous_padded(
let mut md5_hasher = Md5::new();
let mut crc_hasher = Crc32Hasher::new();
- // Single pass through padded data
- md5_hasher.update(&padded);
- crc_hasher.update(&padded);
+ // Cache-resident sub-slice walk over the padded buffer.
+ update_md5_crc32_fused(&mut md5_hasher, &mut crc_hasher, &padded);
(
Md5Hash::new(md5_hasher.finalize().into()),
diff --git a/src/create/backend.rs b/src/create/backend.rs
index 7a266f9b..04e9f977 100644
--- a/src/create/backend.rs
+++ b/src/create/backend.rs
@@ -5,56 +5,428 @@ use crate::reed_solomon::codec::{
};
use crate::reed_solomon::galois::Galois16;
use crate::reed_solomon::AlignedVec;
-use rayon::prelude::*;
+use std::sync::{
+ atomic::{AtomicUsize, Ordering},
+ Arc, Condvar, Mutex,
+};
+use std::thread::JoinHandle;
#[cfg(target_arch = "x86_64")]
use crate::reed_solomon::simd::{
- detect_simd_support, prepare_avx2_coeff, process_slice_multiply_add_prepared_avx2,
- process_slices_multiply_add_prepared_avx2_x2, Avx2PreparedCoeff, SimdLevel,
+ detect_simd_support, finish_xor_jit_bitplane_packed_output_cksum, prepare_avx2_coeff,
+ prepare_xor_jit_bitplane_packed_input_cksum, process_slice_multiply_add_prepared_avx2,
+ process_slice_multiply_add_xor_jit, process_slices_multiply_add_prepared_avx2_x2,
+ process_slices_multiply_add_prepared_avx2_x4, process_slices_multiply_add_xor_jit_x2,
+ process_slices_multiply_add_xor_jit_x4,
+ process_slices_multiply_add_xor_jit_x4_inputs_x2_outputs,
+ process_slices_multiply_add_xor_jit_x4_inputs_x4_outputs, xor_jit_create_avx2_method_info,
+ xor_jit_create_prefetch_plan, xor_packed_multi_region_v16i1_ptr, Avx2PreparedCoeff, SimdLevel,
+ XorJitBitplaneScratch, XorJitCreateMethodInfo, XorJitFlavor, XorJitPreparedCoeff,
+ XorJitPreparedCoeffCache,
};
-const DEFAULT_INPUT_BATCH_SIZE: usize = 12;
+const DEFAULT_INPUT_GROUPING: usize = 12;
const TRANSFER_BUFFER_COUNT: usize = 2;
+const CREATE_SEGMENT_SIZE: usize = 256 * 1024;
+// The prepared x1 PSHUFB path currently retires fewer instructions on the
+// large-file create proxy than the x2/x4 packed kernels on this CPU.
+const PSHUFB_PACKED_INPUTS: usize = 1;
+const AVX2_ALIGNMENT: usize = 32;
+#[cfg(target_arch = "x86_64")]
+const XOR_JIT_SEGMENT_LEN_ENV: &str = "PAR2RS_CREATE_XOR_JIT_SEGMENT_BYTES";
+#[cfg(target_arch = "x86_64")]
+const XOR_JIT_PREFETCH_MAX_FACTOR: usize = 3;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum CreateGf16Method {
+ Auto,
+ Avx2PshufbPrepared,
+ #[cfg(target_arch = "x86_64")]
+ Avx2XorJit,
+ Scalar,
+}
+
+impl CreateGf16Method {
+ fn from_env() -> Self {
+ match std::env::var("PAR2RS_CREATE_GF16") {
+ Ok(value) => match value.to_ascii_lowercase().as_str() {
+ "auto" => Self::Auto,
+ "pshufb" | "avx2-pshufb" | "avx2_pshufb" => Self::Avx2PshufbPrepared,
+ "xor-jit" | "xor_jit" | "xorit" | "avx2-xor-jit" | "xor-jit-port"
+ | "xor_jit_port" | "avx2-xor-jit-port" => {
+ #[cfg(target_arch = "x86_64")]
+ {
+ return Self::Avx2XorJit;
+ }
+ #[cfg(not(target_arch = "x86_64"))]
+ Self::Auto
+ }
+ "scalar" => Self::Scalar,
+ _ => Self::Auto,
+ },
+ Err(_) => Self::Auto,
+ }
+ }
+
+ fn resolve(self) -> Self {
+ match self {
+ Self::Auto => {
+ #[cfg(target_arch = "x86_64")]
+ {
+ if matches!(detect_simd_support(), SimdLevel::Avx2) {
+ return Self::Avx2PshufbPrepared;
+ }
+ }
+ Self::Scalar
+ }
+ Self::Avx2PshufbPrepared => {
+ #[cfg(target_arch = "x86_64")]
+ {
+ if matches!(detect_simd_support(), SimdLevel::Avx2) {
+ return Self::Avx2PshufbPrepared;
+ }
+ }
+ Self::Scalar
+ }
+ #[cfg(target_arch = "x86_64")]
+ Self::Avx2XorJit => {
+ if matches!(detect_simd_support(), SimdLevel::Avx2)
+ && is_x86_feature_detected!("vpclmulqdq")
+ {
+ return self;
+ }
+ panic!("XOR-JIT create backend requires x86_64 AVX2 and VPCLMULQDQ support");
+ }
+ Self::Scalar => Self::Scalar,
+ }
+ }
+
+ #[inline]
+ fn ideal_input_multiple(self) -> usize {
+ #[cfg(target_arch = "x86_64")]
+ if let Some(info) = self.xor_jit_create_method_info() {
+ return info.ideal_input_multiple;
+ }
+
+ match self {
+ Self::Auto | Self::Avx2PshufbPrepared => 4,
+ Self::Scalar => 1,
+ #[cfg(target_arch = "x86_64")]
+ Self::Avx2XorJit => unreachable!("handled by xor_jit_create_method_info"),
+ }
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[inline]
+ fn xor_jit_create_method_info(self) -> Option {
+ match self {
+ Self::Avx2XorJit => Some(xor_jit_create_avx2_method_info()),
+ _ => None,
+ }
+ }
+
+ #[inline]
+ fn ideal_segment_len(self) -> usize {
+ match self {
+ Self::Auto | Self::Avx2PshufbPrepared => CREATE_SEGMENT_SIZE,
+ #[cfg(target_arch = "x86_64")]
+ Self::Avx2XorJit => xor_jit_segment_len_override().unwrap_or(128 * 1024),
+ Self::Scalar => CREATE_SEGMENT_SIZE / 2,
+ }
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[inline]
+ fn xor_jit_flavor(self) -> Option {
+ match self {
+ Self::Avx2XorJit => Some(XorJitFlavor::Jit),
+ _ => None,
+ }
+ }
+}
+
+#[cfg(target_arch = "x86_64")]
+fn xor_jit_segment_len_override() -> Option {
+ match std::env::var(XOR_JIT_SEGMENT_LEN_ENV) {
+ Ok(value) => {
+ let parsed = value.parse::().unwrap_or_else(|_| {
+ panic!(
+ "{XOR_JIT_SEGMENT_LEN_ENV} must be a positive integer byte count, got {value:?}"
+ )
+ });
+ assert!(
+ parsed > 0,
+ "{XOR_JIT_SEGMENT_LEN_ENV} must be greater than zero"
+ );
+ Some(parsed)
+ }
+ Err(std::env::VarError::NotPresent) => None,
+ Err(std::env::VarError::NotUnicode(_)) => {
+ panic!("{XOR_JIT_SEGMENT_LEN_ENV} must be valid UTF-8")
+ }
+ }
+}
+
+#[cfg(not(target_arch = "x86_64"))]
+fn xor_jit_segment_len_override() -> Option {
+ None
+}
/// Prepared coefficient for one `(recovery output, source input)` pair.
-pub struct Gf16Coeff {
+pub struct CreateCoeff {
pub value: u16,
pub split: SplitMulTable,
#[cfg(target_arch = "x86_64")]
pub avx2: Option,
+ #[cfg(target_arch = "x86_64")]
+ pub xor_jit: Option,
+ #[cfg(target_arch = "x86_64")]
+ pub xor_jit_bitplane: Option,
}
-impl Gf16Coeff {
+pub type Gf16Coeff = CreateCoeff;
+
+impl CreateCoeff {
#[inline]
- fn new(value: u16) -> Self {
+ #[cfg(target_arch = "x86_64")]
+ fn new(
+ value: u16,
+ prepare_pshufb: bool,
+ prepare_bitplane: bool,
+ xor_jit_cache: &mut XorJitPreparedCoeffCache,
+ ) -> Self {
let split = build_split_mul_table(Galois16::new(value));
- #[cfg(target_arch = "x86_64")]
- let avx2 = Some(prepare_avx2_coeff(&split));
+ let avx2 = prepare_pshufb.then(|| prepare_avx2_coeff(&split));
+ let xor_jit = prepare_bitplane.then(|| {
+ let prepared = xor_jit_cache.prepare(value);
+ prepared.ensure_bitplane_emitted();
+ xor_jit_cache.cache_bitplane_handle(value, prepared.bitplane_handle());
+ prepared
+ });
+ let xor_jit_bitplane = xor_jit.clone();
Self {
value,
split,
- #[cfg(target_arch = "x86_64")]
avx2,
+ xor_jit,
+ xor_jit_bitplane,
+ }
+ }
+
+ #[cfg(not(target_arch = "x86_64"))]
+ fn new(value: u16, _prepare_pshufb: bool) -> Self {
+ Self {
+ value,
+ split: build_split_mul_table(Galois16::new(value)),
+ }
+ }
+}
+
+pub struct StagingArea {
+ inputs: AlignedVec,
+ source_indices: Vec,
+ #[cfg(target_arch = "x86_64")]
+ xor_jit_coeffs: Vec,
+ #[cfg(target_arch = "x86_64")]
+ xor_jit_out_non_zero: Vec,
+ #[cfg(target_arch = "x86_64")]
+ proc_refs: AtomicUsize,
+ batch_len: usize,
+}
+
+impl StagingArea {
+ #[cfg(target_arch = "x86_64")]
+ fn new(input_grouping: usize, input_storage_len: usize, recovery_count: usize) -> Self {
+ Self {
+ inputs: AlignedVec::new_zeroed(input_storage_len),
+ source_indices: vec![0; input_grouping],
+ xor_jit_coeffs: vec![0; input_grouping * recovery_count],
+ xor_jit_out_non_zero: vec![0; recovery_count],
+ proc_refs: AtomicUsize::new(0),
+ batch_len: 0,
+ }
+ }
+
+ #[cfg(not(target_arch = "x86_64"))]
+ fn new(input_grouping: usize, input_storage_len: usize) -> Self {
+ Self {
+ inputs: AlignedVec::new_zeroed(input_storage_len),
+ source_indices: vec![0; input_grouping],
+ batch_len: 0,
+ }
+ }
+
+ #[inline]
+ fn slot_mut(&mut self, slot: usize, aligned_chunk_len: usize, chunk_len: usize) -> &mut [u8] {
+ let start = slot * aligned_chunk_len;
+ let end = start + chunk_len;
+ &mut self.inputs[start..end]
+ }
+}
+
+#[cfg(target_arch = "x86_64")]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+struct XorJitBitplaneLayout {
+ aligned_slice_len: usize,
+ stride: usize,
+ chunk_len: usize,
+ input_batch_size: usize,
+ recovery_count: usize,
+}
+
+#[cfg(target_arch = "x86_64")]
+impl XorJitBitplaneLayout {
+ fn new(
+ max_slice_len: usize,
+ chunk_len: usize,
+ input_batch_size: usize,
+ recovery_count: usize,
+ method_info: XorJitCreateMethodInfo,
+ ) -> Self {
+ let stride = method_info.stride;
+ let aligned_slice_len = align_up(max_slice_len, stride) + stride;
+ debug_assert!(chunk_len > 0);
+ debug_assert!(chunk_len.is_multiple_of(method_info.stride));
+ debug_assert!(aligned_slice_len.is_multiple_of(method_info.stride));
+ Self {
+ aligned_slice_len,
+ stride,
+ chunk_len,
+ input_batch_size,
+ recovery_count,
}
}
+
+ #[inline]
+ fn align_to_stride(&self, len: usize) -> usize {
+ align_up(len, self.stride)
+ }
+
+ #[inline]
+ fn aligned_current_slice_size(&self, slice_len: usize) -> usize {
+ self.align_to_stride(slice_len) + self.stride
+ }
+
+ #[inline]
+ fn segment_count_for(&self, aligned_current_slice_size: usize) -> usize {
+ aligned_current_slice_size.div_ceil(self.chunk_len)
+ }
+
+ #[inline]
+ #[cfg_attr(not(test), allow(dead_code))]
+ fn input_offset(&self, slice_offset: usize, batch_idx: usize, proc_size: usize) -> usize {
+ debug_assert!(batch_idx < self.input_batch_size);
+ let offset = slice_offset * self.input_batch_size + batch_idx * proc_size;
+ debug_assert!(offset.is_multiple_of(self.stride));
+ offset
+ }
+
+ #[inline]
+ #[cfg_attr(not(test), allow(dead_code))]
+ fn output_offset(&self, slice_offset: usize, recovery_idx: usize, proc_size: usize) -> usize {
+ debug_assert!(recovery_idx < self.recovery_count);
+ let offset = slice_offset * self.recovery_count + recovery_idx * proc_size;
+ debug_assert!(offset.is_multiple_of(self.stride));
+ offset
+ }
+
+ #[inline]
+ #[cfg_attr(not(test), allow(dead_code))]
+ fn slice_offset(&self, segment_idx: usize) -> usize {
+ segment_idx * self.chunk_len
+ }
+
+ #[inline]
+ #[cfg_attr(not(test), allow(dead_code))]
+ fn segment_len_for(&self, slice_offset: usize, aligned_current_slice_size: usize) -> usize {
+ debug_assert!(slice_offset < aligned_current_slice_size);
+ let len = (aligned_current_slice_size - slice_offset).min(self.chunk_len);
+ debug_assert!(len.is_multiple_of(self.stride));
+ len
+ }
+
+ #[inline]
+ fn input_storage_len(&self) -> usize {
+ self.input_batch_size * self.aligned_slice_len
+ }
+
+ #[inline]
+ fn output_storage_len(&self) -> usize {
+ self.recovery_count * self.aligned_slice_len
+ }
+}
+
+#[cfg(target_arch = "x86_64")]
+#[inline]
+fn round_div(a: usize, b: usize) -> usize {
+ (a + (b >> 1)) / b
+}
+
+#[cfg(target_arch = "x86_64")]
+fn xor_jit_runtime_chunk_len(
+ slice_len: usize,
+ worker_count: usize,
+ method: CreateGf16Method,
+) -> usize {
+ let method_info = method
+ .xor_jit_create_method_info()
+ .expect("XOR-JIT runtime chunk length requires XOR-JIT method info");
+ let aligned_current_slice_size = align_up(slice_len, method_info.stride) + method_info.stride;
+ let ideal_chunk_len = align_down(
+ method
+ .ideal_segment_len()
+ .min(aligned_current_slice_size)
+ .max(method_info.stride),
+ method_info.stride,
+ )
+ .max(method_info.stride);
+ let target_thread_chunk = aligned_current_slice_size.div_ceil(worker_count.max(1));
+
+ let mut num_chunks = if target_thread_chunk <= ideal_chunk_len / 2 {
+ round_div(aligned_current_slice_size, ideal_chunk_len).max(1)
+ } else {
+ round_div(target_thread_chunk, ideal_chunk_len).max(1) * worker_count.max(1)
+ };
+
+ let chunk_len = align_up(
+ aligned_current_slice_size.div_ceil(num_chunks),
+ method_info.stride,
+ );
+ num_chunks = aligned_current_slice_size.div_ceil(chunk_len);
+ debug_assert!(num_chunks > 0);
+ chunk_len
}
/// Create-side recovery backend with all hot-path storage owned up front.
pub struct CreateRecoveryBackend {
- pub recovery_exponents: Vec,
pub source_count: usize,
+ pub recovery_exponents: Vec,
+ pub max_chunk_len: usize,
pub chunk_len: usize,
- pub output_chunks: Vec,
- pub input_staging: Vec,
- pub coeffs: Vec,
+ pub method: CreateGf16Method,
+ pub input_grouping: usize,
#[cfg(target_arch = "x86_64")]
- pub batch_coeffs: Vec,
- transfer_buffers: Vec,
- batch_source_indices: Vec,
- batch_len: usize,
+ xor_jit_bitplane: bool,
+ #[cfg(target_arch = "x86_64")]
+ xor_jit_layout: Option,
+ transfer_buffers: [AlignedVec; TRANSFER_BUFFER_COUNT],
+ pub staging: Vec,
+ pub output_chunks: AlignedVec,
+ pub coeffs: Vec,
+ workers: CreateWorkerPool,
+ aligned_chunk_len: usize,
+ active_staging: usize,
+ compute_in_flight: bool,
#[cfg(target_arch = "x86_64")]
- simd_level: SimdLevel,
+ processing_add: bool,
+ job_storage: Vec,
+ #[cfg(target_arch = "x86_64")]
+ xor_jit_req_storage: Vec,
+ #[cfg(target_arch = "x86_64")]
+ xor_jit_req_thread_starts: Vec,
+ #[cfg(target_arch = "x86_64")]
+ xor_jit_req_thread_counts: Vec,
}
impl CreateRecoveryBackend {
@@ -63,12 +435,55 @@ impl CreateRecoveryBackend {
first_recovery_block: u32,
recovery_count: usize,
max_chunk_len: usize,
+ thread_count: usize,
) -> Self {
+ let requested_method = CreateGf16Method::from_env();
+ let method = requested_method.resolve();
let source_count = base_values.len();
+ #[cfg(target_arch = "x86_64")]
+ let xor_jit_bitplane = method.xor_jit_flavor().is_some();
+ #[cfg(target_arch = "x86_64")]
+ let chunk_alignment = method
+ .xor_jit_create_method_info()
+ .map(|info| info.alignment)
+ .unwrap_or(AVX2_ALIGNMENT);
+ #[cfg(not(target_arch = "x86_64"))]
+ let chunk_alignment = AVX2_ALIGNMENT;
+ let aligned_chunk_len = align_up(max_chunk_len, chunk_alignment);
+ let input_grouping = input_grouping(source_count, method);
+ let worker_count = thread_count.max(1);
+ #[cfg(target_arch = "x86_64")]
+ let xor_jit_layout = xor_jit_bitplane.then(|| {
+ let method_info = method
+ .xor_jit_create_method_info()
+ .expect("xor-jit bitplane layout requires XOR-JIT method info");
+ let chunk_len = xor_jit_runtime_chunk_len(max_chunk_len, worker_count, method);
+ XorJitBitplaneLayout::new(
+ aligned_chunk_len,
+ chunk_len,
+ input_grouping,
+ recovery_count,
+ method_info,
+ )
+ });
+ #[cfg(target_arch = "x86_64")]
+ let staging_storage_len = xor_jit_layout
+ .map(|layout| layout.input_storage_len())
+ .unwrap_or(input_grouping * aligned_chunk_len);
+ #[cfg(not(target_arch = "x86_64"))]
+ let staging_storage_len = input_grouping * aligned_chunk_len;
+ #[cfg(target_arch = "x86_64")]
+ let output_storage_len = xor_jit_layout
+ .map(|layout| layout.output_storage_len())
+ .unwrap_or(recovery_count * aligned_chunk_len);
+ #[cfg(not(target_arch = "x86_64"))]
+ let output_storage_len = recovery_count * aligned_chunk_len;
let recovery_exponents = (0..recovery_count)
.map(|offset| (first_recovery_block + offset as u32) as u16)
.collect::>();
-
+ let prepare_pshufb = matches!(method, CreateGf16Method::Avx2PshufbPrepared);
+ #[cfg(target_arch = "x86_64")]
+ let mut xor_jit_cache = XorJitPreparedCoeffCache::new();
let coeffs = recovery_exponents
.iter()
.flat_map(|&exponent| {
@@ -76,117 +491,322 @@ impl CreateRecoveryBackend {
.iter()
.map(move |&base| Galois16::new(base).pow(exponent).value())
})
- .map(Gf16Coeff::new)
+ .map(|value| {
+ #[cfg(target_arch = "x86_64")]
+ {
+ CreateCoeff::new(value, prepare_pshufb, xor_jit_bitplane, &mut xor_jit_cache)
+ }
+ #[cfg(not(target_arch = "x86_64"))]
+ {
+ CreateCoeff::new(value, prepare_pshufb)
+ }
+ })
.collect::>();
-
+ let max_job_count = max_compute_jobs(
+ aligned_chunk_len.max(max_chunk_len),
+ recovery_count,
+ worker_count,
+ method,
+ );
+ let mut staging = vec![
+ {
+ #[cfg(target_arch = "x86_64")]
+ {
+ StagingArea::new(input_grouping, staging_storage_len, recovery_count)
+ }
+ #[cfg(not(target_arch = "x86_64"))]
+ {
+ StagingArea::new(input_grouping, staging_storage_len)
+ }
+ },
+ {
+ #[cfg(target_arch = "x86_64")]
+ {
+ StagingArea::new(input_grouping, staging_storage_len, recovery_count)
+ }
+ #[cfg(not(target_arch = "x86_64"))]
+ {
+ StagingArea::new(input_grouping, staging_storage_len)
+ }
+ },
+ ];
#[cfg(target_arch = "x86_64")]
- let batch_coeffs = coeffs
- .iter()
- .filter_map(|coeff| coeff.avx2.clone())
- .collect::>();
+ if xor_jit_bitplane {
+ let out_non_zero = recovery_exponents
+ .iter()
+ .map(|&exponent| u8::from(exponent != 0))
+ .collect::>();
+ staging.iter_mut().for_each(|staging| {
+ staging.xor_jit_out_non_zero.copy_from_slice(&out_non_zero);
+ });
+ }
Self {
- recovery_exponents,
source_count,
+ recovery_exponents,
+ max_chunk_len,
chunk_len: 0,
- output_chunks: (0..recovery_count)
- .map(|_| AlignedVec::new_zeroed(max_chunk_len))
- .collect(),
- input_staging: (0..DEFAULT_INPUT_BATCH_SIZE)
- .map(|_| AlignedVec::new_zeroed(max_chunk_len))
- .collect(),
+ method,
+ input_grouping,
+ #[cfg(target_arch = "x86_64")]
+ xor_jit_bitplane,
+ #[cfg(target_arch = "x86_64")]
+ xor_jit_layout,
+ transfer_buffers: [
+ AlignedVec::new_zeroed(aligned_chunk_len),
+ AlignedVec::new_zeroed(aligned_chunk_len),
+ ],
+ staging,
+ output_chunks: AlignedVec::new_zeroed(output_storage_len),
coeffs,
+ workers: CreateWorkerPool::new(worker_count, max_job_count),
+ aligned_chunk_len,
+ active_staging: 0,
+ compute_in_flight: false,
#[cfg(target_arch = "x86_64")]
- batch_coeffs,
- transfer_buffers: (0..TRANSFER_BUFFER_COUNT)
- .map(|_| AlignedVec::new_zeroed(max_chunk_len))
- .collect(),
- batch_source_indices: vec![0; DEFAULT_INPUT_BATCH_SIZE],
- batch_len: 0,
+ processing_add: false,
+ job_storage: vec![ComputeJob::default(); max_job_count],
+ #[cfg(target_arch = "x86_64")]
+ xor_jit_req_storage: vec![XorJitComputeReq::default(); max_job_count],
#[cfg(target_arch = "x86_64")]
- simd_level: detect_simd_support(),
+ xor_jit_req_thread_starts: vec![0; worker_count],
+ #[cfg(target_arch = "x86_64")]
+ xor_jit_req_thread_counts: vec![0; worker_count],
}
}
#[inline]
pub fn begin_chunk(&mut self, chunk_len: usize) {
self.chunk_len = chunk_len;
- self.batch_len = 0;
+ self.active_staging = 0;
+ debug_assert!(!self.compute_in_flight);
+ #[cfg(target_arch = "x86_64")]
+ {
+ self.processing_add = false;
+ if self.xor_jit_bitplane {
+ let method_info = self
+ .method
+ .xor_jit_create_method_info()
+ .expect("xor-jit layout refresh requires XOR-JIT method info");
+ let chunk_len =
+ xor_jit_runtime_chunk_len(chunk_len, self.workers.worker_count(), self.method);
+ self.xor_jit_layout = Some(XorJitBitplaneLayout::new(
+ self.aligned_chunk_len,
+ chunk_len,
+ self.input_grouping,
+ self.recovery_exponents.len(),
+ method_info,
+ ));
+ }
+ }
+ self.staging.iter_mut().for_each(|staging| {
+ staging.batch_len = 0;
+ #[cfg(target_arch = "x86_64")]
+ staging.proc_refs.store(0, Ordering::Relaxed);
+ });
+ debug_assert!(chunk_len <= self.max_chunk_len);
debug_assert_eq!(
self.coeffs.len(),
self.recovery_exponents.len() * self.source_count
);
- #[cfg(target_arch = "x86_64")]
- debug_assert_eq!(self.batch_coeffs.len(), self.coeffs.len());
- debug_assert!(self
- .input_staging
- .iter()
- .all(|buffer| (buffer.as_ptr() as usize).is_multiple_of(32)));
debug_assert!(self
- .output_chunks
+ .staging
.iter()
- .all(|buffer| chunk_len <= buffer.len()));
+ .all(|staging| (staging.inputs.as_ptr() as usize).is_multiple_of(AVX2_ALIGNMENT)));
+ debug_assert!((self.output_chunks.as_ptr() as usize).is_multiple_of(AVX2_ALIGNMENT));
+ #[cfg(target_arch = "x86_64")]
+ if let Some(layout) = self.xor_jit_layout {
+ debug_assert!(self.output_chunks.len() >= layout.output_storage_len());
+ } else {
+ debug_assert!(
+ self.output_chunks.len() >= self.recovery_exponents.len() * self.aligned_chunk_len
+ );
+ }
+ #[cfg(not(target_arch = "x86_64"))]
+ debug_assert!(
+ self.output_chunks.len() >= self.recovery_exponents.len() * self.aligned_chunk_len
+ );
+ debug_assert!(self.workers.capacity() >= self.job_storage.len());
+ #[cfg(target_arch = "x86_64")]
+ debug_assert!(self.workers.capacity() >= self.xor_jit_req_storage.len());
+ #[cfg(target_arch = "x86_64")]
+ debug_assert!(
+ self.method != CreateGf16Method::Avx2PshufbPrepared
+ || self.coeffs.iter().all(|c| c.avx2.is_some())
+ );
+ #[cfg(target_arch = "x86_64")]
+ debug_assert!(
+ !self.xor_jit_bitplane || self.coeffs.iter().all(|c| c.xor_jit_bitplane.is_some())
+ );
- self.output_chunks
- .iter_mut()
- .for_each(|chunk| chunk[..chunk_len].fill(0));
+ #[cfg(target_arch = "x86_64")]
+ if !self.xor_jit_bitplane {
+ self.output_chunks.fill(0);
+ }
+ #[cfg(not(target_arch = "x86_64"))]
+ self.output_chunks.fill(0);
}
#[inline]
- pub fn prepare_transfer_buffer(&mut self, ring_index: usize) -> &mut [u8] {
- let idx = ring_index % self.transfer_buffers.len();
+ pub fn transfer_buffer(&mut self, ring_index: usize) -> &mut [u8] {
+ let idx = ring_index % TRANSFER_BUFFER_COUNT;
let chunk = &mut self.transfer_buffers[idx][..self.chunk_len];
chunk.fill(0);
chunk
}
+ #[inline]
+ pub fn prepare_transfer_buffer(&mut self, ring_index: usize) -> &mut [u8] {
+ self.transfer_buffer(ring_index)
+ }
+
#[inline]
pub fn add_input(&mut self, source_idx: usize, input_chunk: &[u8]) {
debug_assert!(source_idx < self.source_count);
debug_assert_eq!(input_chunk.len(), self.chunk_len);
- debug_assert!(self.batch_len < self.input_staging.len());
+ let staging_idx = self.active_staging;
+ let staging = &mut self.staging[staging_idx];
+ debug_assert!(staging.batch_len < self.input_grouping);
- let slot = self.batch_len;
- self.input_staging[slot][..self.chunk_len].copy_from_slice(input_chunk);
- self.batch_source_indices[slot] = source_idx;
- self.batch_len += 1;
+ let slot = staging.batch_len;
+ #[cfg(target_arch = "x86_64")]
+ if self.xor_jit_bitplane {
+ let layout = self
+ .xor_jit_layout
+ .expect("XOR-JIT bitplane layout initialized");
+ prepare_xor_jit_bitplane_staging(layout, staging, slot, self.chunk_len, input_chunk);
+ } else {
+ staging
+ .slot_mut(slot, self.aligned_chunk_len, self.chunk_len)
+ .copy_from_slice(input_chunk);
+ }
+ #[cfg(not(target_arch = "x86_64"))]
+ staging
+ .slot_mut(slot, self.aligned_chunk_len, self.chunk_len)
+ .copy_from_slice(input_chunk);
+ staging.source_indices[slot] = source_idx;
+ #[cfg(target_arch = "x86_64")]
+ if self.xor_jit_bitplane {
+ pack_xor_jit_bitplane_coeffs(
+ &self.coeffs,
+ self.source_count,
+ self.recovery_exponents.len(),
+ self.input_grouping,
+ staging,
+ slot,
+ source_idx,
+ );
+ }
+ staging.batch_len += 1;
- if self.batch_len == self.input_staging.len() {
- self.flush_batch();
+ if staging.batch_len == self.input_grouping {
+ self.flush_active_staging();
}
}
#[inline]
pub fn add_transfer_input(&mut self, source_idx: usize, ring_index: usize) {
- let idx = ring_index % self.transfer_buffers.len();
+ let idx = ring_index % TRANSFER_BUFFER_COUNT;
+ debug_assert!(source_idx < self.source_count);
debug_assert!(self.chunk_len <= self.transfer_buffers[idx].len());
- debug_assert!(self.batch_len < self.input_staging.len());
+ let staging_idx = self.active_staging;
+ let staging = &mut self.staging[staging_idx];
+ debug_assert!(staging.batch_len < self.input_grouping);
- let slot = self.batch_len;
- self.input_staging[slot][..self.chunk_len]
+ let slot = staging.batch_len;
+ #[cfg(target_arch = "x86_64")]
+ if self.xor_jit_bitplane {
+ let layout = self
+ .xor_jit_layout
+ .expect("XOR-JIT bitplane layout initialized");
+ prepare_xor_jit_bitplane_staging(
+ layout,
+ staging,
+ slot,
+ self.chunk_len,
+ &self.transfer_buffers[idx][..self.chunk_len],
+ );
+ } else {
+ staging
+ .slot_mut(slot, self.aligned_chunk_len, self.chunk_len)
+ .copy_from_slice(&self.transfer_buffers[idx][..self.chunk_len]);
+ }
+ #[cfg(not(target_arch = "x86_64"))]
+ staging
+ .slot_mut(slot, self.aligned_chunk_len, self.chunk_len)
.copy_from_slice(&self.transfer_buffers[idx][..self.chunk_len]);
- self.batch_source_indices[slot] = source_idx;
- self.batch_len += 1;
+ staging.source_indices[slot] = source_idx;
+ #[cfg(target_arch = "x86_64")]
+ if self.xor_jit_bitplane {
+ pack_xor_jit_bitplane_coeffs(
+ &self.coeffs,
+ self.source_count,
+ self.recovery_exponents.len(),
+ self.input_grouping,
+ staging,
+ slot,
+ source_idx,
+ );
+ }
+ staging.batch_len += 1;
- if self.batch_len == self.input_staging.len() {
- self.flush_batch();
+ if staging.batch_len == self.input_grouping {
+ self.flush_active_staging();
}
}
#[inline]
- pub fn finish_chunk(&mut self, recovery_blocks: &mut [(u16, Vec)], block_size: usize) {
- self.flush_batch();
+ pub fn end_input(&mut self) {
+ self.flush_active_staging();
+ self.wait_for_compute();
+ }
- self.output_chunks
- .iter()
- .zip(recovery_blocks.iter_mut())
- .for_each(|(output_chunk, (_, recovery_data))| {
+ #[inline]
+ pub fn finish_chunk(
+ &mut self,
+ recovery_blocks: &mut [(u16, Vec)],
+ block_size: usize,
+ ) -> bool {
+ self.end_input();
+ let mut all_checksum_ok = true;
+
+ recovery_blocks
+ .iter_mut()
+ .enumerate()
+ .for_each(|(recovery_idx, (_, recovery_data))| {
debug_assert!(recovery_data.capacity() >= block_size);
debug_assert!(recovery_data.len() + self.chunk_len <= recovery_data.capacity());
- debug_assert!(self.chunk_len <= output_chunk.len());
- recovery_data.extend_from_slice(&output_chunk[..self.chunk_len]);
+ #[cfg(target_arch = "x86_64")]
+ if self.xor_jit_bitplane {
+ let layout = self
+ .xor_jit_layout
+ .expect("XOR-JIT bitplane layout initialized");
+ let output_start = recovery_data.len();
+ recovery_data.resize(output_start + self.chunk_len, 0);
+ let output_checksum_ok = finish_xor_jit_bitplane_packed_output_cksum(
+ &mut recovery_data[output_start..output_start + self.chunk_len],
+ &self.output_chunks,
+ layout.recovery_count,
+ recovery_idx,
+ layout.chunk_len,
+ );
+ all_checksum_ok &= output_checksum_ok;
+ } else {
+ let start = recovery_idx * self.aligned_chunk_len;
+ let end = start + self.chunk_len;
+ debug_assert!(end <= self.output_chunks.len());
+ recovery_data.extend_from_slice(&self.output_chunks[start..end]);
+ }
+ #[cfg(not(target_arch = "x86_64"))]
+ {
+ let start = recovery_idx * self.aligned_chunk_len;
+ let end = start + self.chunk_len;
+ debug_assert!(end <= self.output_chunks.len());
+ recovery_data.extend_from_slice(&self.output_chunks[start..end]);
+ }
});
+ all_checksum_ok
}
#[inline]
@@ -198,138 +818,1547 @@ impl CreateRecoveryBackend {
}
#[inline]
- fn flush_batch(&mut self) {
- if self.batch_len == 0 {
+ pub fn selected_method(&self) -> CreateGf16Method {
+ self.method
+ }
+
+ #[inline]
+ fn flush_active_staging(&mut self) {
+ let staging_idx = self.active_staging;
+ if self.staging[staging_idx].batch_len == 0 {
return;
}
- let chunk_len = self.chunk_len;
- let source_count = self.source_count;
- let batch_len = self.batch_len;
- let coeffs = &self.coeffs;
- let input_staging = &self.input_staging;
- let source_indices = &self.batch_source_indices;
+ self.wait_for_compute();
#[cfg(target_arch = "x86_64")]
- let simd_level = self.simd_level;
+ if self.xor_jit_bitplane {
+ let processing_add = self.processing_add;
+ let request_count = self.build_xor_jit_requests(staging_idx, processing_add);
+ self.staging[staging_idx]
+ .proc_refs
+ .store(request_count, Ordering::Release);
+ self.workers.submit_xor_jit(
+ &self.xor_jit_req_storage[..request_count],
+ &self.xor_jit_req_thread_starts,
+ &self.xor_jit_req_thread_counts,
+ );
+ self.processing_add = true;
+ } else {
+ let job_count = self.build_compute_jobs(staging_idx);
+ self.workers.submit(&self.job_storage[..job_count]);
+ }
+ #[cfg(not(target_arch = "x86_64"))]
+ {
+ let job_count = self.build_compute_jobs(staging_idx);
+ self.workers.submit(&self.job_storage[..job_count]);
+ }
+ self.compute_in_flight = true;
+ self.staging[staging_idx].batch_len = 0;
+ self.active_staging = (self.active_staging + 1) % self.staging.len();
+ }
- self.output_chunks
- .par_iter_mut()
- .enumerate()
- .for_each(|(recovery_idx, output_chunk)| {
- let output = &mut output_chunk[..chunk_len];
- #[cfg(target_arch = "x86_64")]
- if matches!(simd_level, SimdLevel::Avx2) {
- Self::process_batch_add_avx2_x2(
- recovery_idx,
- source_count,
- batch_len,
- input_staging,
- source_indices,
- coeffs,
- output,
- );
- return;
- }
+ #[inline]
+ fn wait_for_compute(&mut self) {
+ if self.compute_in_flight {
+ self.workers.wait();
+ #[cfg(target_arch = "x86_64")]
+ debug_assert!(self
+ .staging
+ .iter()
+ .all(|staging| staging.proc_refs.load(Ordering::Acquire) == 0));
+ self.compute_in_flight = false;
+ }
+ }
- (0..batch_len).for_each(|batch_idx| {
- let source_idx = source_indices[batch_idx];
- let coeff = &coeffs[gf_coeff_index(recovery_idx, source_idx, source_count)];
- let input = &input_staging[batch_idx][..chunk_len];
- #[cfg(target_arch = "x86_64")]
- Self::process_input_add(input, output, coeff, simd_level);
- #[cfg(not(target_arch = "x86_64"))]
- Self::process_input_add(input, output, coeff);
- });
- });
+ fn build_compute_jobs(&mut self, staging_idx: usize) -> usize {
+ let worker_count = self.workers.worker_count();
+ let recovery_count = self.recovery_exponents.len();
+ let compute_len = self.chunk_len;
+ // Mirror turbo `calcChunkSize`: split the *aligned* slice across
+ // segments. Computing on the unaligned `chunk_len` lets the segment_len
+ // round down below what `max_compute_jobs` predicted, producing more
+ // segments than the preallocated job_storage can hold.
+ let segment_basis = self.aligned_chunk_len.max(compute_len);
+ let segment_len = align_down(
+ self.method
+ .ideal_segment_len()
+ .min(segment_basis)
+ .max(AVX2_ALIGNMENT),
+ AVX2_ALIGNMENT,
+ )
+ .max(AVX2_ALIGNMENT);
+ let segment_count = segment_basis.div_ceil(segment_len);
+ let output_groups = worker_count.min(recovery_count).max(1);
+ let outputs_per_group = recovery_count.div_ceil(output_groups);
+ let staging = &self.staging[staging_idx];
+ debug_assert!(staging.batch_len <= self.input_grouping);
+ debug_assert!(self.coeffs.len() == recovery_count * self.source_count);
- self.batch_len = 0;
+ let mut job_count = 0;
+ for segment_idx in 0..segment_count {
+ let start = segment_idx * segment_len;
+ if start >= compute_len {
+ break;
+ }
+ let len = (compute_len - start).min(segment_len);
+ for output_group in 0..output_groups {
+ let output_start = output_group * outputs_per_group;
+ let output_end = ((output_group + 1) * outputs_per_group).min(recovery_count);
+ if output_start == output_end {
+ continue;
+ }
+ debug_assert!(job_count < self.job_storage.len());
+ self.job_storage[job_count] = ComputeJob {
+ method: self.method,
+ input_base: staging.inputs.as_ptr() as usize,
+ output_base: self.output_chunks.as_ptr() as usize,
+ coeffs: self.coeffs.as_ptr() as usize,
+ source_indices: staging.source_indices.as_ptr() as usize,
+ source_count: self.source_count,
+ batch_len: staging.batch_len,
+ aligned_chunk_len: self.aligned_chunk_len,
+ segment_start: start,
+ segment_len: len,
+ output_start,
+ output_end,
+ };
+ job_count += 1;
+ }
+ }
+ job_count
}
#[cfg(target_arch = "x86_64")]
- #[inline]
- fn process_batch_add_avx2_x2(
- recovery_idx: usize,
- source_count: usize,
- batch_len: usize,
- input_staging: &[AlignedVec],
- source_indices: &[usize],
- coeffs: &[Gf16Coeff],
- output: &mut [u8],
- ) {
- let mut batch_idx = 0;
- while batch_idx + 1 < batch_len {
- let source_a = source_indices[batch_idx];
- let source_b = source_indices[batch_idx + 1];
- let coeff_a = &coeffs[gf_coeff_index(recovery_idx, source_a, source_count)];
- let coeff_b = &coeffs[gf_coeff_index(recovery_idx, source_b, source_count)];
-
- match (&coeff_a.avx2, &coeff_b.avx2) {
- (Some(prepared_a), Some(prepared_b)) => unsafe {
- process_slices_multiply_add_prepared_avx2_x2(
- &input_staging[batch_idx][..output.len()],
- prepared_a,
- &coeff_a.split,
- &input_staging[batch_idx + 1][..output.len()],
- prepared_b,
- &coeff_b.split,
- output,
- );
- },
- _ => {
- Self::process_input_add(
- &input_staging[batch_idx][..output.len()],
- output,
- coeff_a,
- SimdLevel::None,
- );
- Self::process_input_add(
- &input_staging[batch_idx + 1][..output.len()],
- output,
- coeff_b,
- SimdLevel::None,
- );
- }
- }
+ fn build_xor_jit_requests(&mut self, staging_idx: usize, add: bool) -> usize {
+ let staging = &self.staging[staging_idx];
+ let layout = self
+ .xor_jit_layout
+ .expect("XOR-JIT bitplane layout initialized");
+ let worker_count = self.workers.worker_count();
+ let recovery_count = self.recovery_exponents.len();
+ let aligned_current_slice_size = layout.aligned_current_slice_size(self.chunk_len);
+ let full_chunks_per_thread =
+ layout.segment_count_for(aligned_current_slice_size) / worker_count.max(1);
+ let leftover_chunks =
+ layout.segment_count_for(aligned_current_slice_size) % worker_count.max(1);
+ self.xor_jit_req_thread_counts.fill(0);
+ let mut chunk = 0usize;
+ let input_base = staging.inputs.as_ptr() as usize;
+ let output_base = self.output_chunks.as_ptr() as usize;
+ let coeffs_base = staging.xor_jit_coeffs.as_ptr() as usize;
+ let out_non_zero_base = staging.xor_jit_out_non_zero.as_ptr() as usize;
+ let proc_refs = &staging.proc_refs as *const AtomicUsize as usize;
- batch_idx += 2;
+ if leftover_chunks != 0 {
+ let threads_per_chunk = (worker_count / leftover_chunks).min(recovery_count).max(1);
+ let used_threads = threads_per_chunk * leftover_chunks;
+ self.xor_jit_req_thread_counts[..used_threads]
+ .iter_mut()
+ .for_each(|count| *count += 1);
+ }
+ if full_chunks_per_thread != 0 {
+ self.xor_jit_req_thread_counts
+ .iter_mut()
+ .for_each(|count| *count += 1);
}
- if batch_idx < batch_len {
- let source_idx = source_indices[batch_idx];
- let coeff = &coeffs[gf_coeff_index(recovery_idx, source_idx, source_count)];
- Self::process_input_add(
- &input_staging[batch_idx][..output.len()],
- output,
- coeff,
- SimdLevel::Avx2,
- );
+ let mut request_count = 0usize;
+ for (thread, count) in self.xor_jit_req_thread_counts.iter().enumerate() {
+ self.xor_jit_req_thread_starts[thread] = request_count;
+ request_count += *count;
+ }
+ let mut thread_positions = self.xor_jit_req_thread_starts.clone();
+
+ if leftover_chunks != 0 {
+ let threads_per_chunk = (worker_count / leftover_chunks).min(recovery_count).max(1);
+ let outputs_per_thread = recovery_count as f64 / threads_per_chunk as f64;
+ let mut thread = 0usize;
+
+ for _ in 0..leftover_chunks {
+ let slice_offset = chunk * layout.chunk_len;
+ let req_len = (aligned_current_slice_size - slice_offset).min(layout.chunk_len);
+ let local_input = input_base + slice_offset * self.input_grouping;
+ let local_output_base = output_base + slice_offset * recovery_count;
+ let mut output_start = 0usize;
+ for thread_chunk in 0..threads_per_chunk {
+ let output_end = (((thread_chunk + 1) as f64 * outputs_per_thread).round()
+ as usize)
+ .min(recovery_count)
+ .max(output_start + 1);
+ let num_outputs = output_end - output_start;
+ let position = thread_positions[thread];
+ thread_positions[thread] += 1;
+ debug_assert!(position < self.xor_jit_req_storage.len());
+ self.xor_jit_req_storage[position] = XorJitComputeReq {
+ method: self.method,
+ input: local_input,
+ output: local_output_base + output_start * req_len,
+ coeffs: coeffs_base
+ + output_start * self.input_grouping * std::mem::size_of::(),
+ out_non_zero: out_non_zero_base + output_start * std::mem::size_of::(),
+ proc_refs,
+ num_inputs: staging.batch_len,
+ input_grouping: self.input_grouping,
+ chunk_size: layout.chunk_len,
+ num_chunks: 1,
+ len: req_len,
+ num_outputs,
+ add,
+ };
+ output_start = output_end;
+ thread += 1;
+ }
+ debug_assert_eq!(output_start, recovery_count);
+ chunk += 1;
+ }
}
+
+ if full_chunks_per_thread != 0 {
+ for thread in 0..worker_count {
+ debug_assert!(chunk < layout.segment_count_for(aligned_current_slice_size));
+ let slice_offset = chunk * layout.chunk_len;
+ let req_len = (aligned_current_slice_size - slice_offset)
+ .min(layout.chunk_len * full_chunks_per_thread);
+ let position = thread_positions[thread];
+ thread_positions[thread] += 1;
+ debug_assert!(position < self.xor_jit_req_storage.len());
+ self.xor_jit_req_storage[position] = XorJitComputeReq {
+ method: self.method,
+ input: input_base + slice_offset * self.input_grouping,
+ output: output_base + slice_offset * recovery_count,
+ coeffs: coeffs_base,
+ out_non_zero: out_non_zero_base,
+ proc_refs,
+ num_inputs: staging.batch_len,
+ input_grouping: self.input_grouping,
+ chunk_size: layout.chunk_len,
+ num_chunks: full_chunks_per_thread,
+ len: req_len,
+ num_outputs: recovery_count,
+ add,
+ };
+ chunk += full_chunks_per_thread;
+ }
+ }
+
+ debug_assert_eq!(chunk, layout.segment_count_for(aligned_current_slice_size));
+ debug_assert!(thread_positions
+ .iter()
+ .zip(
+ self.xor_jit_req_thread_starts
+ .iter()
+ .zip(self.xor_jit_req_thread_counts.iter())
+ )
+ .all(|(position, (start, count))| *position == *start + *count));
+ debug_assert!(request_count > 0);
+ request_count
+ }
+}
+
+#[cfg(target_arch = "x86_64")]
+fn prepare_xor_jit_bitplane_staging(
+ layout: XorJitBitplaneLayout,
+ staging: &mut StagingArea,
+ slot: usize,
+ slice_len: usize,
+ input_chunk: &[u8],
+) {
+ let aligned_slice_len = layout.align_to_stride(slice_len);
+ debug_assert!(aligned_slice_len.is_multiple_of(layout.stride));
+ debug_assert!(input_chunk.len() <= slice_len);
+ prepare_xor_jit_bitplane_packed_input_cksum(
+ &mut staging.inputs,
+ input_chunk,
+ aligned_slice_len,
+ layout.input_batch_size,
+ slot,
+ layout.chunk_len,
+ );
+}
+
+#[cfg(target_arch = "x86_64")]
+#[inline]
+fn pack_xor_jit_bitplane_coeffs(
+ coeffs: &[CreateCoeff],
+ source_count: usize,
+ recovery_count: usize,
+ input_grouping: usize,
+ staging: &mut StagingArea,
+ slot: usize,
+ source_idx: usize,
+) {
+ for recovery_idx in 0..recovery_count {
+ let coeff = &coeffs[gf_coeff_index(recovery_idx, source_idx, source_count)];
+ staging.xor_jit_coeffs[recovery_idx * input_grouping + slot] = coeff.value;
}
+}
+
+#[cfg(target_arch = "x86_64")]
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+struct XorJitComputeReq {
+ method: CreateGf16Method,
+ input: usize,
+ output: usize,
+ coeffs: usize,
+ out_non_zero: usize,
+ proc_refs: usize,
+ num_inputs: usize,
+ input_grouping: usize,
+ chunk_size: usize,
+ num_chunks: usize,
+ len: usize,
+ num_outputs: usize,
+ add: bool,
+}
+
+#[cfg(target_arch = "x86_64")]
+impl XorJitComputeReq {
+ #[inline]
+ fn method_info(self) -> XorJitCreateMethodInfo {
+ self.method
+ .xor_jit_create_method_info()
+ .expect("xor-jit request requires XOR-JIT method info")
+ }
+}
+
+#[derive(Clone, Copy, Default)]
+struct ComputeJob {
+ method: CreateGf16Method,
+ input_base: usize,
+ output_base: usize,
+ coeffs: usize,
+ source_indices: usize,
+ source_count: usize,
+ batch_len: usize,
+ aligned_chunk_len: usize,
+ segment_start: usize,
+ segment_len: usize,
+ output_start: usize,
+ output_end: usize,
+}
+
+impl Default for CreateGf16Method {
+ fn default() -> Self {
+ Self::Scalar
+ }
+}
+
+struct CreateWorkerPool {
+ shared: Arc,
+ handles: Vec>,
+}
+
+struct WorkerShared {
+ state: Mutex,
+ ready: Condvar,
+ done: Condvar,
+}
+
+struct WorkerState {
+ jobs: Vec,
+ #[cfg(target_arch = "x86_64")]
+ xor_jit_worker_reqs: Vec,
+ task_kind: WorkerTaskKind,
+ task_count: usize,
+ next_task: usize,
+ remaining_tasks: usize,
+ generation: u64,
+ stop: bool,
+}
+#[derive(Clone, Copy, Default, PartialEq, Eq)]
+enum WorkerTaskKind {
+ #[default]
+ ComputeJob,
#[cfg(target_arch = "x86_64")]
+ XorJit,
+}
+
+#[cfg(target_arch = "x86_64")]
+enum WorkerTask {
+ ComputeJob(ComputeJob),
+ XorJitRequest(*const XorJitComputeReq, *const AtomicUsize, bool),
+}
+
+#[cfg(target_arch = "x86_64")]
+#[derive(Clone, Copy, Debug, Default)]
+struct XorJitWorkerReqState {
+ reqs: usize,
+ len: usize,
+ next_req: usize,
+}
+
+#[cfg(target_arch = "x86_64")]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum XorJitWorkerBranch {
+ MulAddMultiPackpf,
+ AddMultiPackpf,
+ MulAddMultiPacked,
+}
+
+#[cfg(target_arch = "x86_64")]
+#[inline]
+fn xor_jit_worker_branch(
+ final_round: bool,
+ last_local_output: bool,
+ out_non_zero: bool,
+) -> XorJitWorkerBranch {
+ if final_round && last_local_output {
+ XorJitWorkerBranch::MulAddMultiPacked
+ } else if out_non_zero {
+ XorJitWorkerBranch::MulAddMultiPackpf
+ } else {
+ XorJitWorkerBranch::AddMultiPackpf
+ }
+}
+
+#[cfg(target_arch = "x86_64")]
+#[derive(Default)]
+struct WorkerContext {
+ xor_jit_bitplane_scratch: Option,
+}
+
+impl CreateWorkerPool {
+ fn new(worker_count: usize, max_jobs: usize) -> Self {
+ let shared = Arc::new(WorkerShared {
+ state: Mutex::new(WorkerState {
+ jobs: vec![ComputeJob::default(); max_jobs.max(1)],
+ #[cfg(target_arch = "x86_64")]
+ xor_jit_worker_reqs: vec![XorJitWorkerReqState::default(); worker_count.max(1)],
+ task_kind: WorkerTaskKind::ComputeJob,
+ task_count: 0,
+ next_task: 0,
+ remaining_tasks: 0,
+ generation: 0,
+ stop: false,
+ }),
+ ready: Condvar::new(),
+ done: Condvar::new(),
+ });
+ let handles = (0..worker_count)
+ .map(|worker_id| {
+ let shared = Arc::clone(&shared);
+ std::thread::spawn(move || worker_loop(shared, worker_id))
+ })
+ .collect::>();
+
+ Self { shared, handles }
+ }
+
+ #[inline]
+ fn worker_count(&self) -> usize {
+ self.handles.len().max(1)
+ }
+
#[inline]
- fn process_input_add(
- input: &[u8],
- output: &mut [u8],
- coeff: &Gf16Coeff,
- simd_level: SimdLevel,
- ) {
- if matches!(simd_level, SimdLevel::Avx2) {
- if let Some(prepared) = &coeff.avx2 {
+ fn capacity(&self) -> usize {
+ self.shared.state.lock().unwrap().jobs.len()
+ }
+
+ fn submit(&self, jobs: &[ComputeJob]) {
+ if jobs.is_empty() {
+ return;
+ }
+
+ let mut state = self.shared.state.lock().unwrap();
+ debug_assert_eq!(state.remaining_tasks, 0);
+ debug_assert!(jobs.len() <= state.jobs.len());
+ state.jobs[..jobs.len()].copy_from_slice(jobs);
+ state.task_kind = WorkerTaskKind::ComputeJob;
+ state.task_count = jobs.len();
+ state.next_task = 0;
+ state.remaining_tasks = jobs.len();
+ state.generation = state.generation.wrapping_add(1);
+ self.shared.ready.notify_all();
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ fn submit_xor_jit(&self, reqs: &[XorJitComputeReq], starts: &[usize], counts: &[usize]) {
+ if reqs.is_empty() {
+ return;
+ }
+
+ let mut state = self.shared.state.lock().unwrap();
+ debug_assert_eq!(state.remaining_tasks, 0);
+ debug_assert_eq!(starts.len(), state.xor_jit_worker_reqs.len());
+ debug_assert_eq!(counts.len(), state.xor_jit_worker_reqs.len());
+ let mut active_workers = 0usize;
+ for worker_id in 0..state.xor_jit_worker_reqs.len() {
+ let start = starts[worker_id];
+ let count = counts[worker_id];
+ if count != 0 {
+ state.xor_jit_worker_reqs[worker_id] = XorJitWorkerReqState {
+ reqs: reqs[start..start + count].as_ptr() as usize,
+ len: count,
+ next_req: 0,
+ };
+ active_workers += 1;
+ } else {
+ state.xor_jit_worker_reqs[worker_id] = XorJitWorkerReqState::default();
+ }
+ }
+ state.task_kind = WorkerTaskKind::XorJit;
+ state.task_count = active_workers;
+ state.next_task = 0;
+ state.remaining_tasks = active_workers;
+ state.generation = state.generation.wrapping_add(1);
+ self.shared.ready.notify_all();
+ }
+
+ fn wait(&self) {
+ let mut state = self.shared.state.lock().unwrap();
+ while state.remaining_tasks != 0 {
+ state = self.shared.done.wait(state).unwrap();
+ }
+ }
+}
+
+impl Drop for CreateWorkerPool {
+ fn drop(&mut self) {
+ self.wait();
+ {
+ let mut state = self.shared.state.lock().unwrap();
+ state.stop = true;
+ state.generation = state.generation.wrapping_add(1);
+ }
+ self.shared.ready.notify_all();
+ while let Some(handle) = self.handles.pop() {
+ let _ = handle.join();
+ }
+ }
+}
+
+fn worker_loop(shared: Arc, worker_id: usize) {
+ #[cfg(target_arch = "x86_64")]
+ let mut context = WorkerContext::default();
+ let mut seen_generation = 0u64;
+ loop {
+ #[cfg(target_arch = "x86_64")]
+ let task = {
+ let mut state = shared.state.lock().unwrap();
+ loop {
+ if state.stop {
+ return;
+ }
+ if state.generation != seen_generation {
+ match state.task_kind {
+ WorkerTaskKind::ComputeJob if state.next_task < state.task_count => {
+ let task = WorkerTask::ComputeJob(state.jobs[state.next_task]);
+ state.next_task += 1;
+ break task;
+ }
+ WorkerTaskKind::ComputeJob => {
+ seen_generation = state.generation;
+ }
+ WorkerTaskKind::XorJit => {
+ let worker_reqs = state.xor_jit_worker_reqs[worker_id];
+ if worker_reqs.next_req < worker_reqs.len {
+ let req_ptr = (worker_reqs.reqs as *const XorJitComputeReq)
+ .wrapping_add(worker_reqs.next_req);
+ let req = unsafe { &*req_ptr };
+ state.xor_jit_worker_reqs[worker_id].next_req += 1;
+ let last_for_worker = worker_reqs.next_req + 1 == worker_reqs.len;
+ break WorkerTask::XorJitRequest(
+ req_ptr,
+ req.proc_refs as *const AtomicUsize,
+ last_for_worker,
+ );
+ }
+ seen_generation = state.generation;
+ }
+ }
+ }
+ state = shared.ready.wait(state).unwrap();
+ }
+ };
+ #[cfg(not(target_arch = "x86_64"))]
+ let job = {
+ let mut state = shared.state.lock().unwrap();
+ loop {
+ if state.stop {
+ return;
+ }
+ if state.generation != seen_generation && state.next_task < state.task_count {
+ let job = state.jobs[state.next_task];
+ state.next_task += 1;
+ break job;
+ }
+ if state.generation != seen_generation && state.next_task >= state.task_count {
+ seen_generation = state.generation;
+ }
+ state = shared.ready.wait(state).unwrap();
+ }
+ };
+
+ #[cfg(target_arch = "x86_64")]
+ match task {
+ WorkerTask::ComputeJob(job) => process_compute_job(job, &mut context),
+ WorkerTask::XorJitRequest(req, proc_refs, last_for_worker) => {
+ let req = unsafe { *req };
+ process_xor_jit_request(req, &mut context);
+ unsafe { (*proc_refs).fetch_sub(1, Ordering::AcqRel) };
+ if !last_for_worker {
+ continue;
+ }
+ }
+ }
+ #[cfg(not(target_arch = "x86_64"))]
+ process_compute_job(job);
+
+ let mut state = shared.state.lock().unwrap();
+ state.remaining_tasks -= 1;
+ if state.remaining_tasks == 0 {
+ seen_generation = state.generation;
+ shared.done.notify_one();
+ }
+ }
+}
+
+#[cfg(not(target_arch = "x86_64"))]
+fn process_compute_job(job: ComputeJob) {
+ for recovery_idx in job.output_start..job.output_end {
+ let output_start = recovery_idx * job.aligned_chunk_len + job.segment_start;
+ let output = unsafe {
+ std::slice::from_raw_parts_mut(
+ (job.output_base as *mut u8).add(output_start),
+ job.segment_len,
+ )
+ };
+ debug_assert!(output.as_ptr() as usize >= job.output_base);
+
+ for batch_idx in 0..job.batch_len {
+ let source_idx = unsafe { *(job.source_indices as *const usize).add(batch_idx) };
+ let coeff = unsafe {
+ &*(job.coeffs as *const CreateCoeff).add(gf_coeff_index(
+ recovery_idx,
+ source_idx,
+ job.source_count,
+ ))
+ };
+ let input_start = batch_idx * job.aligned_chunk_len + job.segment_start;
+ let input = unsafe {
+ std::slice::from_raw_parts(
+ (job.input_base as *const u8).add(input_start),
+ job.segment_len,
+ )
+ };
+ process_slice_multiply_add(input, output, &coeff.split);
+ }
+ }
+}
+
+#[cfg(target_arch = "x86_64")]
+fn process_compute_job(job: ComputeJob, context: &mut WorkerContext) {
+ #[cfg(target_arch = "x86_64")]
+ if let Some(flavor) = job.method.xor_jit_flavor() {
+ process_compute_job_xor_jit(job, flavor, context);
+ return;
+ }
+
+ for recovery_idx in job.output_start..job.output_end {
+ let output_start = recovery_idx * job.aligned_chunk_len + job.segment_start;
+ let output = unsafe {
+ std::slice::from_raw_parts_mut(
+ (job.output_base as *mut u8).add(output_start),
+ job.segment_len,
+ )
+ };
+ debug_assert!(output.as_ptr() as usize >= job.output_base);
+
+ if matches!(job.method, CreateGf16Method::Avx2PshufbPrepared) {
+ process_batch_add_avx2_pshufb(job, recovery_idx, output);
+ continue;
+ }
+
+ for batch_idx in 0..job.batch_len {
+ let source_idx = unsafe { *(job.source_indices as *const usize).add(batch_idx) };
+ let coeff = unsafe {
+ &*(job.coeffs as *const CreateCoeff).add(gf_coeff_index(
+ recovery_idx,
+ source_idx,
+ job.source_count,
+ ))
+ };
+ let input_start = batch_idx * job.aligned_chunk_len + job.segment_start;
+ let input = unsafe {
+ std::slice::from_raw_parts(
+ (job.input_base as *const u8).add(input_start),
+ job.segment_len,
+ )
+ };
+ process_slice_multiply_add(input, output, &coeff.split);
+ }
+ }
+}
+
+#[cfg(target_arch = "x86_64")]
+fn process_compute_job_xor_jit(
+ job: ComputeJob,
+ flavor: XorJitFlavor,
+ _context: &mut WorkerContext,
+) {
+ let mut recovery_idx = job.output_start;
+ while recovery_idx + 3 < job.output_end {
+ let output_a_start = recovery_idx * job.aligned_chunk_len + job.segment_start;
+ let output_b_start = (recovery_idx + 1) * job.aligned_chunk_len + job.segment_start;
+ let output_c_start = (recovery_idx + 2) * job.aligned_chunk_len + job.segment_start;
+ let output_d_start = (recovery_idx + 3) * job.aligned_chunk_len + job.segment_start;
+ let output_a = unsafe {
+ std::slice::from_raw_parts_mut(
+ (job.output_base as *mut u8).add(output_a_start),
+ job.segment_len,
+ )
+ };
+ let output_b = unsafe {
+ std::slice::from_raw_parts_mut(
+ (job.output_base as *mut u8).add(output_b_start),
+ job.segment_len,
+ )
+ };
+ let output_c = unsafe {
+ std::slice::from_raw_parts_mut(
+ (job.output_base as *mut u8).add(output_c_start),
+ job.segment_len,
+ )
+ };
+ let output_d = unsafe {
+ std::slice::from_raw_parts_mut(
+ (job.output_base as *mut u8).add(output_d_start),
+ job.segment_len,
+ )
+ };
+ process_batch_add_avx2_xor_jit_x4_outputs(
+ job,
+ recovery_idx,
+ output_a,
+ recovery_idx + 1,
+ output_b,
+ recovery_idx + 2,
+ output_c,
+ recovery_idx + 3,
+ output_d,
+ flavor,
+ );
+ recovery_idx += 4;
+ }
+
+ while recovery_idx + 1 < job.output_end {
+ let output_a_start = recovery_idx * job.aligned_chunk_len + job.segment_start;
+ let output_b_start = (recovery_idx + 1) * job.aligned_chunk_len + job.segment_start;
+ let output_a = unsafe {
+ std::slice::from_raw_parts_mut(
+ (job.output_base as *mut u8).add(output_a_start),
+ job.segment_len,
+ )
+ };
+ let output_b = unsafe {
+ std::slice::from_raw_parts_mut(
+ (job.output_base as *mut u8).add(output_b_start),
+ job.segment_len,
+ )
+ };
+ process_batch_add_avx2_xor_jit_x2_outputs(
+ job,
+ recovery_idx,
+ output_a,
+ recovery_idx + 1,
+ output_b,
+ flavor,
+ );
+ recovery_idx += 2;
+ }
+
+ if recovery_idx < job.output_end {
+ let output_start = recovery_idx * job.aligned_chunk_len + job.segment_start;
+ let output = unsafe {
+ std::slice::from_raw_parts_mut(
+ (job.output_base as *mut u8).add(output_start),
+ job.segment_len,
+ )
+ };
+ process_batch_add_avx2_xor_jit(job, recovery_idx, output, flavor);
+ }
+}
+
+#[cfg(target_arch = "x86_64")]
+fn process_xor_jit_request(req: XorJitComputeReq, context: &mut WorkerContext) {
+ let scratch = context
+ .xor_jit_bitplane_scratch
+ .get_or_insert_with(|| XorJitBitplaneScratch::new().expect("allocate xor-jit scratch"));
+ let method_info = req.method_info();
+ let (inputs_prefetched_per_invok, input_prefetch_out_offset) =
+ xor_jit_input_prefetch_schedule(req);
+ let input_base = req.input as *const u8;
+ let output_base = req.output as *mut u8;
+ let coeffs_base = req.coeffs as *const u16;
+ let out_non_zero_base = req.out_non_zero as *const u8;
+
+ for round in 0..req.num_chunks {
+ let proc_size = (req.len - round * req.chunk_size).min(req.chunk_size);
+ debug_assert!(proc_size.is_multiple_of(method_info.stride));
+ let final_round = round + 1 == req.num_chunks;
+ let input_ptr = unsafe { input_base.add(round * req.chunk_size * req.input_grouping) };
+ let round_output_base =
+ unsafe { output_base.add(round * req.num_outputs * req.chunk_size) };
+
+ for output_idx in 0..req.num_outputs {
+ let output_ptr = unsafe { round_output_base.add(output_idx * proc_size) };
+ if !req.add {
unsafe {
- process_slice_multiply_add_prepared_avx2(input, output, prepared, &coeff.split);
+ std::ptr::write_bytes(output_ptr, 0, proc_size);
}
- return;
+ }
+ let coeff_ptr = unsafe { coeffs_base.add(output_idx * req.input_grouping) };
+ let out_non_zero = unsafe { *out_non_zero_base.add(output_idx) != 0 };
+ let input_prefetch = if !final_round && output_idx >= input_prefetch_out_offset {
+ Some(unsafe {
+ input_base.add(
+ (round + 1) * req.chunk_size * req.input_grouping
+ + ((inputs_prefetched_per_invok
+ * (output_idx - input_prefetch_out_offset)
+ * proc_size)
+ >> XOR_JIT_PREFETCH_MAX_FACTOR),
+ )
+ })
+ } else {
+ None
+ };
+ let output_prefetch = xor_jit_output_prefetch_ptr(req, round, output_idx);
+
+ match xor_jit_worker_branch(
+ final_round,
+ output_idx + 1 == req.num_outputs,
+ out_non_zero,
+ ) {
+ XorJitWorkerBranch::MulAddMultiPackpf => xor_jit_mul_add_multi_packpf_ptr(
+ req,
+ coeff_ptr,
+ input_ptr,
+ output_ptr,
+ proc_size,
+ input_prefetch,
+ output_prefetch,
+ scratch,
+ ),
+ XorJitWorkerBranch::AddMultiPackpf => xor_jit_add_multi_packpf_ptr(
+ req,
+ input_ptr,
+ output_ptr,
+ proc_size,
+ input_prefetch,
+ output_prefetch,
+ ),
+ XorJitWorkerBranch::MulAddMultiPacked => xor_jit_mul_add_multi_packed_ptr(
+ req, coeff_ptr, input_ptr, output_ptr, proc_size, scratch,
+ ),
}
}
+ }
+}
- process_slice_multiply_add(input, output, &coeff.split);
+#[cfg(target_arch = "x86_64")]
+#[inline]
+fn xor_jit_input_prefetch_schedule(req: XorJitComputeReq) -> (usize, usize) {
+ let method_info = req.method_info();
+ let mut inputs_prefetched_per_invok = req.num_inputs / method_info.ideal_input_multiple;
+ let mut input_prefetch_out_offset = req.num_outputs.saturating_sub(1);
+ let prefetch_downscale = method_info.prefetch_downscale;
+
+ if inputs_prefetched_per_invok > (1usize << prefetch_downscale) {
+ inputs_prefetched_per_invok -= 1usize << prefetch_downscale;
+ inputs_prefetched_per_invok <<= XOR_JIT_PREFETCH_MAX_FACTOR - prefetch_downscale;
+ input_prefetch_out_offset =
+ (req.num_inputs << XOR_JIT_PREFETCH_MAX_FACTOR).div_ceil(inputs_prefetched_per_invok);
+ input_prefetch_out_offset = req.num_outputs.saturating_sub(input_prefetch_out_offset);
}
- #[cfg(not(target_arch = "x86_64"))]
- #[inline]
- fn process_input_add(input: &[u8], output: &mut [u8], coeff: &Gf16Coeff) {
- process_slice_multiply_add(input, output, &coeff.split);
+ (inputs_prefetched_per_invok, input_prefetch_out_offset)
+}
+
+#[cfg(target_arch = "x86_64")]
+fn xor_jit_mul_add_multi_packpf_ptr(
+ req: XorJitComputeReq,
+ coeff_ptr: *const u16,
+ input_ptr: *const u8,
+ output: *mut u8,
+ round_len: usize,
+ input_prefetch: Option<*const u8>,
+ output_prefetch: Option<*const u8>,
+ scratch: &mut XorJitBitplaneScratch,
+) {
+ unsafe {
+ xor_jit_multi_region_packpf_ptr(
+ req,
+ scratch,
+ coeff_ptr,
+ input_ptr,
+ req.num_inputs,
+ output,
+ round_len,
+ input_prefetch,
+ output_prefetch,
+ );
+ }
+}
+
+#[cfg(target_arch = "x86_64")]
+fn xor_jit_mul_add_multi_packed_ptr(
+ req: XorJitComputeReq,
+ coeff_ptr: *const u16,
+ input_ptr: *const u8,
+ output: *mut u8,
+ round_len: usize,
+ scratch: &mut XorJitBitplaneScratch,
+) {
+ unsafe {
+ xor_jit_multi_region_ptr(
+ scratch,
+ coeff_ptr,
+ input_ptr,
+ req.num_inputs,
+ output,
+ round_len,
+ );
+ }
+}
+
+#[cfg(target_arch = "x86_64")]
+unsafe fn xor_jit_multi_region_ptr(
+ scratch: &mut XorJitBitplaneScratch,
+ coeffs: *const u16,
+ src: *const u8,
+ regions: usize,
+ output: *mut u8,
+ len: usize,
+) {
+ debug_assert!(regions > 0);
+ for region in 0..regions {
+ let coefficient = unsafe { *coeffs.add(region) };
+ let input = unsafe { src.add(region * len) };
+ unsafe { scratch.multiply_add_ptr_coefficient(coefficient, input, output, len) };
+ }
+}
+
+#[cfg(target_arch = "x86_64")]
+unsafe fn xor_jit_multi_region_packpf_ptr(
+ req: XorJitComputeReq,
+ scratch: &mut XorJitBitplaneScratch,
+ coeffs: *const u16,
+ src: *const u8,
+ regions: usize,
+ output: *mut u8,
+ len: usize,
+ prefetch_in: Option<*const u8>,
+ prefetch_out: Option<*const u8>,
+) {
+ debug_assert!(regions > 0);
+
+ let prefetch_plan = xor_jit_create_prefetch_plan(req.method_info(), len);
+ let mut region = 0usize;
+ let mut prefetch_ptr = prefetch_out;
+ let mut output_pf_rounds = prefetch_plan.output_prefetch_rounds;
+ while region < regions && output_pf_rounds > 0 {
+ let coefficient = unsafe { *coeffs.add(region) };
+ let input = unsafe { src.add(region * len) };
+ let current_prefetch = prefetch_ptr.unwrap_or(std::ptr::null());
+ unsafe {
+ scratch.multiply_add_ptr_coefficient_prefetch(
+ coefficient,
+ input,
+ output,
+ len,
+ current_prefetch,
+ );
+ }
+ region += 1;
+ output_pf_rounds -= 1;
+ prefetch_ptr = prefetch_ptr.map(|ptr| ptr.wrapping_add(prefetch_plan.pf_len));
+ }
+
+ if let Some(mut prefetch_ptr) = prefetch_in {
+ while region < regions {
+ let coefficient = unsafe { *coeffs.add(region) };
+ let input = unsafe { src.add(region * len) };
+ unsafe {
+ scratch.multiply_add_ptr_coefficient_prefetch(
+ coefficient,
+ input,
+ output,
+ len,
+ prefetch_ptr,
+ );
+ }
+ region += 1;
+ prefetch_ptr = prefetch_ptr.wrapping_add(prefetch_plan.pf_len);
+ }
+ } else {
+ unsafe {
+ xor_jit_multi_region_ptr(
+ scratch,
+ coeffs.add(region),
+ src.add(region * len),
+ regions - region,
+ output,
+ len,
+ );
+ }
+ }
+}
+
+#[cfg(target_arch = "x86_64")]
+fn xor_jit_add_multi_packpf_ptr(
+ req: XorJitComputeReq,
+ input_ptr: *const u8,
+ output: *mut u8,
+ round_len: usize,
+ input_prefetch: Option<*const u8>,
+ output_prefetch: Option<*const u8>,
+) {
+ xor_packed_multi_region_v16i1_ptr(
+ input_ptr,
+ req.num_inputs,
+ output,
+ round_len,
+ req.method_info(),
+ input_prefetch,
+ output_prefetch,
+ );
+}
+
+#[cfg(target_arch = "x86_64")]
+#[cfg_attr(not(test), allow(dead_code))]
+fn xor_jit_output_prefetch_ptr(
+ req: XorJitComputeReq,
+ round: usize,
+ output_idx: usize,
+) -> Option<*const u8> {
+ if round + 1 == req.num_chunks && output_idx + 1 == req.num_outputs {
+ return None;
+ }
+
+ let proc_size = (req.len - round * req.chunk_size).min(req.chunk_size);
+ Some((req.output as *const u8).wrapping_add(
+ output_idx * proc_size + round * req.num_outputs * req.chunk_size + proc_size,
+ ))
+}
+
+#[cfg(all(target_arch = "x86_64", test))]
+fn xor_jit_input_prefetch_ptr(
+ req: XorJitComputeReq,
+ round: usize,
+ output_idx: usize,
+) -> Option<*const u8> {
+ if round + 1 >= req.num_chunks {
+ return None;
+ }
+
+ let (inputs_prefetched_per_invok, input_prefetch_out_offset) =
+ xor_jit_input_prefetch_schedule(req);
+ if output_idx < input_prefetch_out_offset {
+ return None;
+ }
+
+ let proc_size = (req.len - round * req.chunk_size).min(req.chunk_size);
+ Some((req.input as *const u8).wrapping_add(
+ (round + 1) * req.chunk_size * req.input_grouping
+ + ((inputs_prefetched_per_invok
+ * (output_idx - input_prefetch_out_offset)
+ * proc_size)
+ >> XOR_JIT_PREFETCH_MAX_FACTOR),
+ ))
+}
+
+#[cfg(target_arch = "x86_64")]
+#[allow(clippy::too_many_arguments)]
+fn process_batch_add_avx2_xor_jit_x4_outputs(
+ job: ComputeJob,
+ recovery_a: usize,
+ output_a: &mut [u8],
+ recovery_b: usize,
+ output_b: &mut [u8],
+ recovery_c: usize,
+ output_c: &mut [u8],
+ recovery_d: usize,
+ output_d: &mut [u8],
+ flavor: XorJitFlavor,
+) {
+ let mut batch_idx = 0;
+ while batch_idx + 3 < job.batch_len {
+ let source_a = unsafe { *(job.source_indices as *const usize).add(batch_idx) };
+ let source_b = unsafe { *(job.source_indices as *const usize).add(batch_idx + 1) };
+ let source_c = unsafe { *(job.source_indices as *const usize).add(batch_idx + 2) };
+ let source_d = unsafe { *(job.source_indices as *const usize).add(batch_idx + 3) };
+ let coeff_a0 = coeff_for(job, recovery_a, source_a);
+ let coeff_b0 = coeff_for(job, recovery_a, source_b);
+ let coeff_c0 = coeff_for(job, recovery_a, source_c);
+ let coeff_d0 = coeff_for(job, recovery_a, source_d);
+ let coeff_a1 = coeff_for(job, recovery_b, source_a);
+ let coeff_b1 = coeff_for(job, recovery_b, source_b);
+ let coeff_c1 = coeff_for(job, recovery_b, source_c);
+ let coeff_d1 = coeff_for(job, recovery_b, source_d);
+ let coeff_a2 = coeff_for(job, recovery_c, source_a);
+ let coeff_b2 = coeff_for(job, recovery_c, source_b);
+ let coeff_c2 = coeff_for(job, recovery_c, source_c);
+ let coeff_d2 = coeff_for(job, recovery_c, source_d);
+ let coeff_a3 = coeff_for(job, recovery_d, source_a);
+ let coeff_b3 = coeff_for(job, recovery_d, source_b);
+ let coeff_c3 = coeff_for(job, recovery_d, source_c);
+ let coeff_d3 = coeff_for(job, recovery_d, source_d);
+ let input_a = input_segment(job, batch_idx);
+ let input_b = input_segment(job, batch_idx + 1);
+ let input_c = input_segment(job, batch_idx + 2);
+ let input_d = input_segment(job, batch_idx + 3);
+
+ match (
+ &coeff_a0.xor_jit,
+ &coeff_b0.xor_jit,
+ &coeff_c0.xor_jit,
+ &coeff_d0.xor_jit,
+ &coeff_a1.xor_jit,
+ &coeff_b1.xor_jit,
+ &coeff_c1.xor_jit,
+ &coeff_d1.xor_jit,
+ &coeff_a2.xor_jit,
+ &coeff_b2.xor_jit,
+ &coeff_c2.xor_jit,
+ &coeff_d2.xor_jit,
+ &coeff_a3.xor_jit,
+ &coeff_b3.xor_jit,
+ &coeff_c3.xor_jit,
+ &coeff_d3.xor_jit,
+ ) {
+ (
+ Some(prepared_a0),
+ Some(prepared_b0),
+ Some(prepared_c0),
+ Some(prepared_d0),
+ Some(prepared_a1),
+ Some(prepared_b1),
+ Some(prepared_c1),
+ Some(prepared_d1),
+ Some(prepared_a2),
+ Some(prepared_b2),
+ Some(prepared_c2),
+ Some(prepared_d2),
+ Some(prepared_a3),
+ Some(prepared_b3),
+ Some(prepared_c3),
+ Some(prepared_d3),
+ ) => unsafe {
+ process_slices_multiply_add_xor_jit_x4_inputs_x4_outputs(
+ input_a,
+ input_b,
+ input_c,
+ input_d,
+ prepared_a0,
+ prepared_b0,
+ prepared_c0,
+ prepared_d0,
+ output_a,
+ prepared_a1,
+ prepared_b1,
+ prepared_c1,
+ prepared_d1,
+ output_b,
+ prepared_a2,
+ prepared_b2,
+ prepared_c2,
+ prepared_d2,
+ output_c,
+ prepared_a3,
+ prepared_b3,
+ prepared_c3,
+ prepared_d3,
+ output_d,
+ flavor,
+ );
+ },
+ _ => panic!("XOR-JIT create backend missing prepared coefficient"),
+ }
+ batch_idx += 4;
+ }
+
+ if batch_idx < job.batch_len {
+ process_batch_add_avx2_xor_jit_x2_outputs(
+ ComputeJob {
+ batch_len: job.batch_len - batch_idx,
+ input_base: unsafe {
+ (job.input_base as *const u8).add(batch_idx * job.aligned_chunk_len) as usize
+ },
+ source_indices: unsafe {
+ (job.source_indices as *const usize).add(batch_idx) as usize
+ },
+ ..job
+ },
+ recovery_a,
+ output_a,
+ recovery_b,
+ output_b,
+ flavor,
+ );
+ process_batch_add_avx2_xor_jit_x2_outputs(
+ ComputeJob {
+ batch_len: job.batch_len - batch_idx,
+ input_base: unsafe {
+ (job.input_base as *const u8).add(batch_idx * job.aligned_chunk_len) as usize
+ },
+ source_indices: unsafe {
+ (job.source_indices as *const usize).add(batch_idx) as usize
+ },
+ ..job
+ },
+ recovery_c,
+ output_c,
+ recovery_d,
+ output_d,
+ flavor,
+ );
+ }
+}
+
+#[cfg(target_arch = "x86_64")]
+fn process_batch_add_avx2_xor_jit_x2_outputs(
+ job: ComputeJob,
+ recovery_a: usize,
+ output_a: &mut [u8],
+ recovery_b: usize,
+ output_b: &mut [u8],
+ flavor: XorJitFlavor,
+) {
+ let mut batch_idx = 0;
+ while batch_idx + 3 < job.batch_len {
+ let source_a = unsafe { *(job.source_indices as *const usize).add(batch_idx) };
+ let source_b = unsafe { *(job.source_indices as *const usize).add(batch_idx + 1) };
+ let source_c = unsafe { *(job.source_indices as *const usize).add(batch_idx + 2) };
+ let source_d = unsafe { *(job.source_indices as *const usize).add(batch_idx + 3) };
+ let coeff_a0 = coeff_for(job, recovery_a, source_a);
+ let coeff_b0 = coeff_for(job, recovery_a, source_b);
+ let coeff_c0 = coeff_for(job, recovery_a, source_c);
+ let coeff_d0 = coeff_for(job, recovery_a, source_d);
+ let coeff_a1 = coeff_for(job, recovery_b, source_a);
+ let coeff_b1 = coeff_for(job, recovery_b, source_b);
+ let coeff_c1 = coeff_for(job, recovery_b, source_c);
+ let coeff_d1 = coeff_for(job, recovery_b, source_d);
+ let input_a = input_segment(job, batch_idx);
+ let input_b = input_segment(job, batch_idx + 1);
+ let input_c = input_segment(job, batch_idx + 2);
+ let input_d = input_segment(job, batch_idx + 3);
+
+ match (
+ &coeff_a0.xor_jit,
+ &coeff_b0.xor_jit,
+ &coeff_c0.xor_jit,
+ &coeff_d0.xor_jit,
+ &coeff_a1.xor_jit,
+ &coeff_b1.xor_jit,
+ &coeff_c1.xor_jit,
+ &coeff_d1.xor_jit,
+ ) {
+ (
+ Some(prepared_a0),
+ Some(prepared_b0),
+ Some(prepared_c0),
+ Some(prepared_d0),
+ Some(prepared_a1),
+ Some(prepared_b1),
+ Some(prepared_c1),
+ Some(prepared_d1),
+ ) => unsafe {
+ process_slices_multiply_add_xor_jit_x4_inputs_x2_outputs(
+ input_a,
+ input_b,
+ input_c,
+ input_d,
+ prepared_a0,
+ prepared_b0,
+ prepared_c0,
+ prepared_d0,
+ output_a,
+ prepared_a1,
+ prepared_b1,
+ prepared_c1,
+ prepared_d1,
+ output_b,
+ flavor,
+ );
+ },
+ _ => panic!("XOR-JIT create backend missing prepared coefficient"),
+ }
+ batch_idx += 4;
+ }
+
+ while batch_idx + 1 < job.batch_len {
+ let source_a = unsafe { *(job.source_indices as *const usize).add(batch_idx) };
+ let source_b = unsafe { *(job.source_indices as *const usize).add(batch_idx + 1) };
+ let coeff_a0 = coeff_for(job, recovery_a, source_a);
+ let coeff_b0 = coeff_for(job, recovery_a, source_b);
+ let coeff_a1 = coeff_for(job, recovery_b, source_a);
+ let coeff_b1 = coeff_for(job, recovery_b, source_b);
+ let input_a = input_segment(job, batch_idx);
+ let input_b = input_segment(job, batch_idx + 1);
+
+ match (
+ &coeff_a0.xor_jit,
+ &coeff_b0.xor_jit,
+ &coeff_a1.xor_jit,
+ &coeff_b1.xor_jit,
+ ) {
+ (Some(prepared_a0), Some(prepared_b0), Some(prepared_a1), Some(prepared_b1)) => unsafe {
+ process_slices_multiply_add_xor_jit_x2(
+ input_a,
+ prepared_a0,
+ input_b,
+ prepared_b0,
+ output_a,
+ flavor,
+ );
+ process_slices_multiply_add_xor_jit_x2(
+ input_a,
+ prepared_a1,
+ input_b,
+ prepared_b1,
+ output_b,
+ flavor,
+ );
+ },
+ _ => panic!("XOR-JIT create backend missing prepared coefficient"),
+ }
+ batch_idx += 2;
+ }
+
+ while batch_idx < job.batch_len {
+ let source_idx = unsafe { *(job.source_indices as *const usize).add(batch_idx) };
+ let coeff_a = coeff_for(job, recovery_a, source_idx);
+ let coeff_b = coeff_for(job, recovery_b, source_idx);
+ let input = input_segment(job, batch_idx);
+ match (&coeff_a.xor_jit, &coeff_b.xor_jit) {
+ (Some(prepared_a), Some(prepared_b)) => unsafe {
+ process_slice_multiply_add_xor_jit(input, output_a, prepared_a, flavor);
+ process_slice_multiply_add_xor_jit(input, output_b, prepared_b, flavor);
+ },
+ _ => panic!("XOR-JIT create backend missing prepared coefficient"),
+ }
+ batch_idx += 1;
+ }
+}
+
+#[cfg(target_arch = "x86_64")]
+fn process_batch_add_avx2_xor_jit(
+ job: ComputeJob,
+ recovery_idx: usize,
+ output: &mut [u8],
+ flavor: XorJitFlavor,
+) {
+ let mut batch_idx = 0;
+ while batch_idx + 3 < job.batch_len {
+ let source_a = unsafe { *(job.source_indices as *const usize).add(batch_idx) };
+ let source_b = unsafe { *(job.source_indices as *const usize).add(batch_idx + 1) };
+ let source_c = unsafe { *(job.source_indices as *const usize).add(batch_idx + 2) };
+ let source_d = unsafe { *(job.source_indices as *const usize).add(batch_idx + 3) };
+ let coeff_a = coeff_for(job, recovery_idx, source_a);
+ let coeff_b = coeff_for(job, recovery_idx, source_b);
+ let coeff_c = coeff_for(job, recovery_idx, source_c);
+ let coeff_d = coeff_for(job, recovery_idx, source_d);
+ let input_a = input_segment(job, batch_idx);
+ let input_b = input_segment(job, batch_idx + 1);
+ let input_c = input_segment(job, batch_idx + 2);
+ let input_d = input_segment(job, batch_idx + 3);
+
+ match (
+ &coeff_a.xor_jit,
+ &coeff_b.xor_jit,
+ &coeff_c.xor_jit,
+ &coeff_d.xor_jit,
+ ) {
+ (Some(prepared_a), Some(prepared_b), Some(prepared_c), Some(prepared_d)) => unsafe {
+ process_slices_multiply_add_xor_jit_x4(
+ input_a, prepared_a, input_b, prepared_b, input_c, prepared_c, input_d,
+ prepared_d, output, flavor,
+ );
+ },
+ _ => panic!("XOR-JIT create backend missing prepared coefficient"),
+ }
+ batch_idx += 4;
+ }
+
+ while batch_idx + 1 < job.batch_len {
+ let source_a = unsafe { *(job.source_indices as *const usize).add(batch_idx) };
+ let source_b = unsafe { *(job.source_indices as *const usize).add(batch_idx + 1) };
+ let coeff_a = coeff_for(job, recovery_idx, source_a);
+ let coeff_b = coeff_for(job, recovery_idx, source_b);
+ let input_a = input_segment(job, batch_idx);
+ let input_b = input_segment(job, batch_idx + 1);
+
+ match (&coeff_a.xor_jit, &coeff_b.xor_jit) {
+ (Some(prepared_a), Some(prepared_b)) => unsafe {
+ process_slices_multiply_add_xor_jit_x2(
+ input_a, prepared_a, input_b, prepared_b, output, flavor,
+ );
+ },
+ _ => panic!("XOR-JIT create backend missing prepared coefficient"),
+ }
+ batch_idx += 2;
+ }
+
+ while batch_idx < job.batch_len {
+ let source_idx = unsafe { *(job.source_indices as *const usize).add(batch_idx) };
+ let coeff = coeff_for(job, recovery_idx, source_idx);
+ let input = input_segment(job, batch_idx);
+ match &coeff.xor_jit {
+ Some(prepared) => unsafe {
+ process_slice_multiply_add_xor_jit(input, output, prepared, flavor);
+ },
+ None => panic!("XOR-JIT create backend missing prepared coefficient"),
+ }
+ batch_idx += 1;
+ }
+}
+
+#[cfg(target_arch = "x86_64")]
+fn process_batch_add_avx2_pshufb(job: ComputeJob, recovery_idx: usize, output: &mut [u8]) {
+ let mut batch_idx = 0;
+ while PSHUFB_PACKED_INPUTS >= 4 && batch_idx + 3 < job.batch_len {
+ let source_a = unsafe { *(job.source_indices as *const usize).add(batch_idx) };
+ let source_b = unsafe { *(job.source_indices as *const usize).add(batch_idx + 1) };
+ let source_c = unsafe { *(job.source_indices as *const usize).add(batch_idx + 2) };
+ let source_d = unsafe { *(job.source_indices as *const usize).add(batch_idx + 3) };
+ let coeff_a = unsafe {
+ &*(job.coeffs as *const CreateCoeff).add(gf_coeff_index(
+ recovery_idx,
+ source_a,
+ job.source_count,
+ ))
+ };
+ let coeff_b = unsafe {
+ &*(job.coeffs as *const CreateCoeff).add(gf_coeff_index(
+ recovery_idx,
+ source_b,
+ job.source_count,
+ ))
+ };
+ let coeff_c = unsafe {
+ &*(job.coeffs as *const CreateCoeff).add(gf_coeff_index(
+ recovery_idx,
+ source_c,
+ job.source_count,
+ ))
+ };
+ let coeff_d = unsafe {
+ &*(job.coeffs as *const CreateCoeff).add(gf_coeff_index(
+ recovery_idx,
+ source_d,
+ job.source_count,
+ ))
+ };
+ let input_a = input_segment(job, batch_idx);
+ let input_b = input_segment(job, batch_idx + 1);
+ let input_c = input_segment(job, batch_idx + 2);
+ let input_d = input_segment(job, batch_idx + 3);
+
+ match (&coeff_a.avx2, &coeff_b.avx2, &coeff_c.avx2, &coeff_d.avx2) {
+ (Some(prepared_a), Some(prepared_b), Some(prepared_c), Some(prepared_d)) => unsafe {
+ process_slices_multiply_add_prepared_avx2_x4(
+ input_a,
+ prepared_a,
+ &coeff_a.split,
+ input_b,
+ prepared_b,
+ &coeff_b.split,
+ input_c,
+ prepared_c,
+ &coeff_c.split,
+ input_d,
+ prepared_d,
+ &coeff_d.split,
+ output,
+ );
+ },
+ _ => {
+ process_slice_multiply_add(input_a, output, &coeff_a.split);
+ process_slice_multiply_add(input_b, output, &coeff_b.split);
+ process_slice_multiply_add(input_c, output, &coeff_c.split);
+ process_slice_multiply_add(input_d, output, &coeff_d.split);
+ }
+ }
+ batch_idx += 4;
+ }
+
+ while PSHUFB_PACKED_INPUTS >= 2 && batch_idx + 1 < job.batch_len {
+ let source_a = unsafe { *(job.source_indices as *const usize).add(batch_idx) };
+ let source_b = unsafe { *(job.source_indices as *const usize).add(batch_idx + 1) };
+ let coeff_a = unsafe {
+ &*(job.coeffs as *const CreateCoeff).add(gf_coeff_index(
+ recovery_idx,
+ source_a,
+ job.source_count,
+ ))
+ };
+ let coeff_b = unsafe {
+ &*(job.coeffs as *const CreateCoeff).add(gf_coeff_index(
+ recovery_idx,
+ source_b,
+ job.source_count,
+ ))
+ };
+ let input_a = input_segment(job, batch_idx);
+ let input_b = input_segment(job, batch_idx + 1);
+
+ match (&coeff_a.avx2, &coeff_b.avx2) {
+ (Some(prepared_a), Some(prepared_b)) => unsafe {
+ process_slices_multiply_add_prepared_avx2_x2(
+ input_a,
+ prepared_a,
+ &coeff_a.split,
+ input_b,
+ prepared_b,
+ &coeff_b.split,
+ output,
+ );
+ },
+ _ => {
+ process_slice_multiply_add(input_a, output, &coeff_a.split);
+ process_slice_multiply_add(input_b, output, &coeff_b.split);
+ }
+ }
+ batch_idx += 2;
+ }
+
+ while batch_idx < job.batch_len {
+ let source_idx = unsafe { *(job.source_indices as *const usize).add(batch_idx) };
+ let coeff = unsafe {
+ &*(job.coeffs as *const CreateCoeff).add(gf_coeff_index(
+ recovery_idx,
+ source_idx,
+ job.source_count,
+ ))
+ };
+ let input = input_segment(job, batch_idx);
+ match &coeff.avx2 {
+ Some(prepared) => unsafe {
+ process_slice_multiply_add_prepared_avx2(input, output, prepared, &coeff.split);
+ },
+ None => process_slice_multiply_add(input, output, &coeff.split),
+ }
+ batch_idx += 1;
+ }
+}
+
+#[cfg(target_arch = "x86_64")]
+#[inline]
+fn input_segment(job: ComputeJob, batch_idx: usize) -> &'static [u8] {
+ let start = batch_idx * job.aligned_chunk_len + job.segment_start;
+ unsafe { std::slice::from_raw_parts((job.input_base as *const u8).add(start), job.segment_len) }
+}
+
+#[cfg(target_arch = "x86_64")]
+#[inline]
+fn coeff_for(job: ComputeJob, recovery_idx: usize, source_idx: usize) -> &'static CreateCoeff {
+ unsafe {
+ &*(job.coeffs as *const CreateCoeff).add(gf_coeff_index(
+ recovery_idx,
+ source_idx,
+ job.source_count,
+ ))
}
}
@@ -338,32 +2367,135 @@ pub fn gf_coeff_index(recovery_idx: usize, source_idx: usize, source_count: usiz
recovery_idx * source_count + source_idx
}
+#[inline]
+fn input_grouping(source_count: usize, method: CreateGf16Method) -> usize {
+ let default = DEFAULT_INPUT_GROUPING;
+ let small = source_count.div_ceil(2).max(1);
+ let requested = if small < default { small } else { default };
+ let multiple = method.ideal_input_multiple();
+ align_up(requested, multiple)
+ .max(multiple)
+ .min(source_count.max(1))
+}
+
+#[cfg(target_arch = "x86_64")]
+#[inline]
+fn xor_jit_request_capacity(worker_count: usize) -> usize {
+ // Turbo dispatches at most two requests per worker for one slice:
+ // one leftover-split request in the first phase and one full-chunk
+ // request in the second phase.
+ worker_count.max(1) * 2
+}
+
+#[inline]
+fn max_compute_jobs(
+ max_chunk_len: usize,
+ recovery_count: usize,
+ worker_count: usize,
+ method: CreateGf16Method,
+) -> usize {
+ #[cfg(target_arch = "x86_64")]
+ if method.xor_jit_flavor().is_some() {
+ return xor_jit_request_capacity(worker_count);
+ }
+
+ let segment_len = align_down(
+ method
+ .ideal_segment_len()
+ .min(max_chunk_len.max(AVX2_ALIGNMENT))
+ .max(AVX2_ALIGNMENT),
+ AVX2_ALIGNMENT,
+ )
+ .max(AVX2_ALIGNMENT);
+ let segment_count = max_chunk_len.max(1).div_ceil(segment_len);
+ let output_groups = worker_count.min(recovery_count).max(1);
+ (segment_count * output_groups).max(1)
+}
+
+#[inline]
+fn align_up(value: usize, alignment: usize) -> usize {
+ if value == 0 {
+ 0
+ } else {
+ value.div_ceil(alignment) * alignment
+ }
+}
+
+#[inline]
+fn align_down(value: usize, alignment: usize) -> usize {
+ value & !(alignment - 1)
+}
+
#[cfg(test)]
mod tests {
use super::*;
use crate::reed_solomon::RecoveryBlockEncoder;
+ use std::sync::{Mutex, OnceLock};
+
+ fn env_lock() -> &'static Mutex<()> {
+ static LOCK: OnceLock> = OnceLock::new();
+ LOCK.get_or_init(|| Mutex::new(()))
+ }
+
+ fn deterministic_inputs(source_count: usize, block_size: usize) -> Vec> {
+ (0..source_count)
+ .map(|src| {
+ (0..block_size)
+ .map(|byte| (src * 31 + byte * 17 + (byte / 7)) as u8)
+ .collect::>()
+ })
+ .collect::>()
+ }
+
+ fn assert_forced_backend_matches_encoder(method: &str, expected_method: CreateGf16Method) {
+ let _guard = env_lock().lock().unwrap();
+ std::env::set_var("PAR2RS_CREATE_GF16", method);
+
+ let block_size = 1024;
+ let source_count = 12;
+ let recovery_count = 5;
+ let encoder = RecoveryBlockEncoder::new(block_size, source_count);
+ let inputs = deterministic_inputs(source_count, block_size);
+ let mut backend =
+ CreateRecoveryBackend::new(encoder.base_values(), 3, recovery_count, block_size, 4);
+ std::env::remove_var("PAR2RS_CREATE_GF16");
+
+ if backend.selected_method() != expected_method {
+ return;
+ }
+
+ let mut recovery_blocks = backend.recovery_blocks(block_size);
+ backend.begin_chunk(block_size);
+ inputs
+ .iter()
+ .enumerate()
+ .for_each(|(idx, input)| backend.add_input(idx, input));
+ assert!(backend.finish_chunk(&mut recovery_blocks, block_size));
+
+ recovery_blocks
+ .iter()
+ .for_each(|(exponent, recovery_data)| {
+ let refs = inputs.iter().map(Vec::as_slice).collect::>();
+ let expected = encoder.encode_recovery_block(*exponent, &refs).unwrap();
+ assert_eq!(recovery_data, &expected);
+ });
+ }
#[test]
fn backend_output_matches_encoder_for_partial_batch() {
let block_size = 64;
let source_count = 5;
let encoder = RecoveryBlockEncoder::new(block_size, source_count);
- let inputs = (0..source_count)
- .map(|src| {
- (0..block_size)
- .map(|byte| (src * 17 + byte) as u8)
- .collect::>()
- })
- .collect::>();
+ let inputs = deterministic_inputs(source_count, block_size);
- let mut backend = CreateRecoveryBackend::new(encoder.base_values(), 0, 3, block_size);
+ let mut backend = CreateRecoveryBackend::new(encoder.base_values(), 0, 3, block_size, 2);
let mut recovery_blocks = backend.recovery_blocks(block_size);
backend.begin_chunk(block_size);
inputs
.iter()
.enumerate()
.for_each(|(idx, input)| backend.add_input(idx, input));
- backend.finish_chunk(&mut recovery_blocks, block_size);
+ assert!(backend.finish_chunk(&mut recovery_blocks, block_size));
recovery_blocks
.iter()
@@ -374,10 +2506,42 @@ mod tests {
});
}
+ #[test]
+ fn forced_pshufb_backend_matches_encoder_for_full_batch() {
+ assert_forced_backend_matches_encoder("pshufb", CreateGf16Method::Avx2PshufbPrepared);
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn forced_xor_jit_backend_matches_encoder_for_full_batch() {
+ assert_forced_backend_matches_encoder("xor-jit", CreateGf16Method::Avx2XorJit);
+ }
+
+ #[test]
+ #[cfg(target_arch = "x86_64")]
+ fn forced_xor_jit_backend_uses_bitplane_without_legacy_fallback() {
+ let _guard = env_lock().lock().unwrap();
+ std::env::set_var("PAR2RS_CREATE_GF16", "xor-jit");
+
+ let encoder = RecoveryBlockEncoder::new(1024, 12);
+ let backend = CreateRecoveryBackend::new(encoder.base_values(), 0, 5, 1024, 4);
+ std::env::remove_var("PAR2RS_CREATE_GF16");
+
+ if backend.selected_method() != CreateGf16Method::Avx2XorJit {
+ return;
+ }
+
+ assert!(backend.coeffs.iter().all(|coeff| coeff.xor_jit.is_some()));
+ assert!(backend
+ .coeffs
+ .iter()
+ .all(|coeff| coeff.xor_jit_bitplane.is_some()));
+ }
+
#[test]
fn backend_reuses_fixed_transfer_buffers() {
let encoder = RecoveryBlockEncoder::new(64, 2);
- let mut backend = CreateRecoveryBackend::new(encoder.base_values(), 7, 1, 64);
+ let mut backend = CreateRecoveryBackend::new(encoder.base_values(), 7, 1, 64, 1);
backend.begin_chunk(32);
let first = backend.prepare_transfer_buffer(0).as_ptr();
let second = backend.prepare_transfer_buffer(1).as_ptr();
@@ -388,4 +2552,697 @@ mod tests {
assert_eq!(first as usize % 32, 0);
assert_eq!(second as usize % 32, 0);
}
+
+ #[test]
+ fn env_method_override_selects_scalar() {
+ let _guard = env_lock().lock().unwrap();
+ std::env::set_var("PAR2RS_CREATE_GF16", "scalar");
+ let encoder = RecoveryBlockEncoder::new(64, 2);
+ let backend = CreateRecoveryBackend::new(encoder.base_values(), 0, 1, 64, 1);
+ std::env::remove_var("PAR2RS_CREATE_GF16");
+ assert_eq!(backend.selected_method(), CreateGf16Method::Scalar);
+ }
+
+ #[test]
+ fn recovery_vectors_keep_exact_reserved_capacity() {
+ let encoder = RecoveryBlockEncoder::new(96, 4);
+ let backend = CreateRecoveryBackend::new(encoder.base_values(), 0, 2, 32, 1);
+ let blocks = backend.recovery_blocks(96);
+ assert!(blocks.iter().all(|(_, bytes)| bytes.capacity() == 96));
+ }
+
+ #[test]
+ fn input_grouping_handles_non_power_of_two_multiple() {
+ assert_eq!(align_up(12, 12), 12);
+ assert_eq!(align_up(13, 12), 24);
+ #[cfg(target_arch = "x86_64")]
+ assert_eq!(input_grouping(256, CreateGf16Method::Avx2XorJit), 12);
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn xor_jit_runtime_chunk_len_matches_turbo_calc_chunksize() {
+ let _guard = env_lock().lock().unwrap();
+ std::env::remove_var(XOR_JIT_SEGMENT_LEN_ENV);
+ assert_eq!(
+ xor_jit_runtime_chunk_len(1024 * 1024, 1, CreateGf16Method::Avx2XorJit),
+ 131_584
+ );
+ assert_eq!(
+ xor_jit_runtime_chunk_len(1024 * 1024, 16, CreateGf16Method::Avx2XorJit),
+ 66_048
+ );
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn xor_jit_bitplane_layout_offsets_are_segment_major() {
+ let layout = XorJitBitplaneLayout::new(1024, 512, 3, 4, xor_jit_create_avx2_method_info());
+ let aligned_current_slice_size = layout.aligned_slice_len;
+ let first_slice_offset = layout.slice_offset(0);
+ let second_slice_offset = layout.slice_offset(1);
+ let first_proc_size =
+ layout.segment_len_for(first_slice_offset, aligned_current_slice_size);
+ let second_proc_size =
+ layout.segment_len_for(second_slice_offset, aligned_current_slice_size);
+
+ assert_eq!(
+ layout.input_offset(first_slice_offset, 0, first_proc_size),
+ 0
+ );
+ assert_eq!(
+ layout.input_offset(first_slice_offset, 1, first_proc_size),
+ 512
+ );
+ assert_eq!(
+ layout.input_offset(first_slice_offset, 2, first_proc_size),
+ 1024
+ );
+ assert_eq!(
+ layout.input_offset(second_slice_offset, 0, second_proc_size),
+ 1536
+ );
+ assert_eq!(
+ layout.input_offset(second_slice_offset, 1, second_proc_size),
+ 2048
+ );
+
+ assert_eq!(
+ layout.output_offset(first_slice_offset, 0, first_proc_size),
+ 0
+ );
+ assert_eq!(
+ layout.output_offset(first_slice_offset, 1, first_proc_size),
+ 512
+ );
+ assert_eq!(
+ layout.output_offset(first_slice_offset, 3, first_proc_size),
+ 1536
+ );
+ assert_eq!(
+ layout.output_offset(second_slice_offset, 0, second_proc_size),
+ 2048
+ );
+ assert_eq!(
+ layout.output_offset(second_slice_offset, 1, second_proc_size),
+ 2560
+ );
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn xor_jit_bitplane_layout_offsets_do_not_overlap() {
+ use std::collections::HashSet;
+
+ let layout = XorJitBitplaneLayout::new(2048, 512, 12, 5, xor_jit_create_avx2_method_info());
+ let aligned_current_slice_size = layout.aligned_slice_len;
+ let segment_count = layout.segment_count_for(aligned_current_slice_size);
+ let mut input_offsets = HashSet::new();
+ for segment_idx in 0..segment_count {
+ let slice_offset = layout.slice_offset(segment_idx);
+ let proc_size = layout.segment_len_for(slice_offset, aligned_current_slice_size);
+ for batch_idx in 0..layout.input_batch_size {
+ assert!(input_offsets.insert(layout.input_offset(
+ slice_offset,
+ batch_idx,
+ proc_size
+ )));
+ }
+ }
+ assert_eq!(input_offsets.len(), segment_count * layout.input_batch_size);
+
+ let mut output_offsets = HashSet::new();
+ for segment_idx in 0..segment_count {
+ let slice_offset = layout.slice_offset(segment_idx);
+ let proc_size = layout.segment_len_for(slice_offset, aligned_current_slice_size);
+ for recovery_idx in 0..layout.recovery_count {
+ assert!(output_offsets.insert(layout.output_offset(
+ slice_offset,
+ recovery_idx,
+ proc_size
+ )));
+ }
+ }
+ assert_eq!(output_offsets.len(), segment_count * layout.recovery_count);
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn xor_jit_bitplane_layout_handles_partial_final_segment() {
+ let layout = XorJitBitplaneLayout::new(1024, 1024, 3, 2, xor_jit_create_avx2_method_info());
+
+ assert_eq!(layout.segment_count_for(layout.aligned_slice_len), 2);
+ assert_eq!(layout.slice_offset(0), 0);
+ assert_eq!(layout.slice_offset(1), 1024);
+ assert_eq!(layout.segment_len_for(0, layout.aligned_slice_len), 1024);
+ assert_eq!(layout.segment_len_for(1024, layout.aligned_slice_len), 512);
+ assert_eq!(layout.input_storage_len(), 3 * 1024 + 3 * 512);
+ assert_eq!(layout.output_storage_len(), 2 * 1024 + 2 * 512);
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn xor_jit_localized_request_build_matches_turbo_single_thread() {
+ let _guard = env_lock().lock().unwrap();
+ std::env::set_var("PAR2RS_CREATE_GF16", "xor-jit");
+
+ let block_size = 1024 * 1024;
+ let encoder = RecoveryBlockEncoder::new(block_size, 24);
+ let mut backend = CreateRecoveryBackend::new(encoder.base_values(), 0, 8, block_size, 1);
+ std::env::remove_var("PAR2RS_CREATE_GF16");
+
+ if backend.selected_method() != CreateGf16Method::Avx2XorJit {
+ return;
+ }
+
+ let input = vec![0u8; block_size];
+ backend.begin_chunk(block_size);
+ for source_idx in 0..3 {
+ backend.add_input(source_idx, &input);
+ }
+
+ let request_count = backend.build_xor_jit_requests(0, false);
+ let req = backend.xor_jit_req_storage[0];
+ let out_non_zero = unsafe { std::slice::from_raw_parts(req.out_non_zero as *const u8, 8) };
+ let staging = &backend.staging[0];
+
+ assert_eq!(request_count, 1);
+ assert_eq!(req.method, CreateGf16Method::Avx2XorJit);
+ assert_eq!(req.input, staging.inputs.as_ptr() as usize);
+ assert_eq!(req.output, backend.output_chunks.as_ptr() as usize);
+ assert_eq!(req.coeffs, staging.xor_jit_coeffs.as_ptr() as usize);
+ assert_eq!(
+ req.out_non_zero,
+ staging.xor_jit_out_non_zero.as_ptr() as usize
+ );
+ assert_eq!(req.num_inputs, 3);
+ assert_eq!(req.input_grouping, 12);
+ assert_eq!(req.chunk_size, 131_584);
+ assert_eq!(req.num_chunks, 8);
+ assert_eq!(req.len, 1_049_088);
+ assert_eq!(req.num_outputs, 8);
+ assert!(!req.add);
+ assert_eq!(out_non_zero[0], 0);
+ assert!(out_non_zero[1..].iter().all(|&value| value == 1));
+
+ backend.build_xor_jit_requests(0, true);
+ assert!(backend.xor_jit_req_storage[0].add);
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn xor_jit_localized_request_build_matches_turbo_leftover_split() {
+ let _guard = env_lock().lock().unwrap();
+ std::env::set_var("PAR2RS_CREATE_GF16", "xor-jit");
+
+ let block_size = 1024;
+ let encoder = RecoveryBlockEncoder::new(block_size, 24);
+ let mut backend = CreateRecoveryBackend::new(encoder.base_values(), 0, 7, block_size, 4);
+ std::env::remove_var("PAR2RS_CREATE_GF16");
+
+ if backend.selected_method() != CreateGf16Method::Avx2XorJit {
+ return;
+ }
+
+ let input = vec![0u8; block_size];
+ backend.begin_chunk(block_size);
+ for source_idx in 0..3 {
+ backend.add_input(source_idx, &input);
+ }
+
+ let request_count = backend.build_xor_jit_requests(0, false);
+ let requests = &backend.xor_jit_req_storage[..request_count];
+ let staging = &backend.staging[0];
+ let layout = backend
+ .xor_jit_layout
+ .expect("XOR-JIT bitplane layout initialized");
+ let req_len = layout.aligned_current_slice_size(block_size);
+ let expected_output_starts = [0usize, 2, 4, 5];
+ let expected_output_counts = [2usize, 2, 1, 2];
+
+ assert_eq!(request_count, 4);
+ assert_eq!(requests.iter().map(|req| req.num_outputs).sum::(), 7);
+
+ for (idx, req) in requests.iter().enumerate() {
+ let output_start = expected_output_starts[idx];
+ assert_eq!(req.method, CreateGf16Method::Avx2XorJit);
+ assert_eq!(req.input, staging.inputs.as_ptr() as usize);
+ assert_eq!(
+ req.output,
+ backend.output_chunks.as_ptr() as usize + output_start * req_len
+ );
+ assert_eq!(
+ req.coeffs,
+ staging.xor_jit_coeffs.as_ptr() as usize
+ + output_start * backend.input_grouping * std::mem::size_of::()
+ );
+ assert_eq!(
+ req.out_non_zero,
+ staging.xor_jit_out_non_zero.as_ptr() as usize + output_start
+ );
+ assert_eq!(req.num_inputs, 3);
+ assert_eq!(req.input_grouping, 12);
+ assert_eq!(req.chunk_size, req_len);
+ assert_eq!(req.num_chunks, 1);
+ assert_eq!(req.len, req_len);
+ assert_eq!(req.num_outputs, expected_output_counts[idx]);
+ assert!(!req.add);
+ }
+
+ assert_eq!(
+ requests[0].output + requests[0].num_outputs * requests[0].len,
+ requests[1].output
+ );
+ assert_eq!(
+ requests[1].output + requests[1].num_outputs * requests[1].len,
+ requests[2].output
+ );
+ assert_eq!(
+ requests[2].output + requests[2].num_outputs * requests[2].len,
+ requests[3].output
+ );
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn xor_jit_local_output_prefetch_stops_at_worker_partition_boundary() {
+ let _guard = env_lock().lock().unwrap();
+ std::env::set_var("PAR2RS_CREATE_GF16", "xor-jit");
+
+ let block_size = 1024;
+ let encoder = RecoveryBlockEncoder::new(block_size, 24);
+ let mut backend = CreateRecoveryBackend::new(encoder.base_values(), 0, 7, block_size, 4);
+ std::env::remove_var("PAR2RS_CREATE_GF16");
+
+ if backend.selected_method() != CreateGf16Method::Avx2XorJit {
+ return;
+ }
+
+ let input = vec![0u8; block_size];
+ backend.begin_chunk(block_size);
+ for source_idx in 0..3 {
+ backend.add_input(source_idx, &input);
+ }
+
+ let request_count = backend.build_xor_jit_requests(0, false);
+ let requests = &backend.xor_jit_req_storage[..request_count];
+ let first = requests[0];
+ let second = requests[1];
+
+ assert!(first.num_outputs < backend.recovery_exponents.len());
+ assert_eq!(
+ xor_jit_output_prefetch_ptr(first, 0, 0).map(|ptr| ptr as usize),
+ Some(first.output + first.len)
+ );
+ assert_eq!(first.output + first.num_outputs * first.len, second.output);
+ assert_eq!(
+ xor_jit_output_prefetch_ptr(first, 0, first.num_outputs - 1).map(|ptr| ptr as usize),
+ None
+ );
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn xor_jit_localized_requests_keep_turbo_worker_ownership_order() {
+ let _guard = env_lock().lock().unwrap();
+ std::env::set_var("PAR2RS_CREATE_GF16", "xor-jit");
+
+ let block_size = 1024;
+ let encoder = RecoveryBlockEncoder::new(block_size, 24);
+ let mut backend = CreateRecoveryBackend::new(encoder.base_values(), 0, 7, block_size, 8);
+ std::env::remove_var("PAR2RS_CREATE_GF16");
+
+ if backend.selected_method() != CreateGf16Method::Avx2XorJit {
+ return;
+ }
+
+ let input = vec![0u8; block_size];
+ backend.begin_chunk(block_size);
+ for source_idx in 0..3 {
+ backend.add_input(source_idx, &input);
+ }
+
+ let request_count = backend.build_xor_jit_requests(0, false);
+
+ assert_eq!(request_count, 7);
+ assert_eq!(
+ backend.xor_jit_req_thread_starts,
+ vec![0, 1, 2, 3, 4, 5, 6, 7]
+ );
+ assert_eq!(
+ backend.xor_jit_req_thread_counts,
+ vec![1, 1, 1, 1, 1, 1, 1, 0]
+ );
+ assert!(backend.xor_jit_req_storage[..7]
+ .iter()
+ .all(|req| req.num_chunks == 1));
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn xor_jit_input_prefetch_uses_turbo_stream_offsets() {
+ let segment_len = 128 * 1024;
+ let method_info = xor_jit_create_avx2_method_info();
+ let layout = XorJitBitplaneLayout::new(segment_len * 2, segment_len, 12, 512, method_info);
+ let req = XorJitComputeReq {
+ method: CreateGf16Method::Avx2XorJit,
+ input: 4096,
+ num_inputs: 12,
+ input_grouping: layout.input_batch_size,
+ chunk_size: layout.chunk_len,
+ num_chunks: layout.segment_count_for(layout.aligned_slice_len),
+ len: layout.aligned_slice_len,
+ num_outputs: layout.recovery_count,
+ ..XorJitComputeReq::default()
+ };
+ assert_eq!(
+ xor_jit_input_prefetch_ptr(req, 0, 508).map(|ptr| ptr as usize),
+ None
+ );
+ assert_eq!(
+ xor_jit_input_prefetch_ptr(req, 0, 509).map(|ptr| ptr as usize),
+ Some(4096 + layout.input_offset(segment_len, 0, segment_len))
+ );
+ assert_eq!(
+ xor_jit_input_prefetch_ptr(req, 0, 509).map(|ptr| ptr as usize),
+ Some(4096 + layout.input_offset(segment_len, 0, segment_len))
+ );
+ assert_eq!(
+ xor_jit_input_prefetch_ptr(req, 0, 510).map(|ptr| ptr as usize),
+ Some(
+ 4096 + layout.input_offset(segment_len, 0, segment_len)
+ + 10 * (segment_len >> method_info.prefetch_downscale)
+ )
+ );
+ assert_eq!(
+ xor_jit_input_prefetch_ptr(req, 0, 511).map(|ptr| ptr as usize),
+ Some(
+ 4096 + layout.input_offset(segment_len, 0, segment_len)
+ + 20 * (segment_len >> method_info.prefetch_downscale)
+ )
+ );
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn xor_jit_worker_prefetch_schedule_matches_turbo_avx2() {
+ let req = XorJitComputeReq {
+ method: CreateGf16Method::Avx2XorJit,
+ num_inputs: 12,
+ input_grouping: 12,
+ num_outputs: 8,
+ ..XorJitComputeReq::default()
+ };
+
+ let (inputs_prefetched_per_invok, input_prefetch_out_offset) =
+ xor_jit_input_prefetch_schedule(req);
+
+ assert_eq!(inputs_prefetched_per_invok, 40);
+ assert_eq!(input_prefetch_out_offset, 5);
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn xor_jit_create_method_info_matches_turbo_avx2() {
+ let info = CreateGf16Method::Avx2XorJit
+ .xor_jit_create_method_info()
+ .expect("xor-jit method info");
+ assert_eq!(info.ideal_input_multiple, 1);
+ assert_eq!(info.prefetch_downscale, 1);
+ assert_eq!(info.alignment, 32);
+ assert_eq!(info.stride, 512);
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn xor_jit_worker_branch_shape_matches_turbo() {
+ assert_eq!(
+ xor_jit_worker_branch(false, false, true),
+ XorJitWorkerBranch::MulAddMultiPackpf
+ );
+ assert_eq!(
+ xor_jit_worker_branch(false, false, false),
+ XorJitWorkerBranch::AddMultiPackpf
+ );
+ assert_eq!(
+ xor_jit_worker_branch(true, false, true),
+ XorJitWorkerBranch::MulAddMultiPackpf
+ );
+ assert_eq!(
+ xor_jit_worker_branch(true, false, false),
+ XorJitWorkerBranch::AddMultiPackpf
+ );
+ assert_eq!(
+ xor_jit_worker_branch(true, true, true),
+ XorJitWorkerBranch::MulAddMultiPacked
+ );
+ assert_eq!(
+ xor_jit_worker_branch(true, true, false),
+ XorJitWorkerBranch::MulAddMultiPacked
+ );
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn xor_jit_output_prefetch_rounds_are_method_info_driven() {
+ let info = xor_jit_create_avx2_method_info();
+ let prefetch_plan = xor_jit_create_prefetch_plan(info, 128 * 1024);
+ assert_eq!(
+ prefetch_plan.output_prefetch_rounds,
+ crate::reed_solomon::simd::xor_jit_create_output_prefetch_rounds(info)
+ );
+ assert_eq!(
+ prefetch_plan.output_prefetch_rounds,
+ 1 << info.prefetch_downscale
+ );
+ assert_eq!(prefetch_plan.pf_len, 128 * 1024 >> info.prefetch_downscale);
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn xor_jit_add_only_local_request_stays_within_partition_boundaries() {
+ let _guard = env_lock().lock().unwrap();
+ std::env::set_var("PAR2RS_CREATE_GF16", "xor-jit");
+
+ let block_size = 1024 * 1024;
+ let encoder = RecoveryBlockEncoder::new(block_size, 24);
+ let mut backend = CreateRecoveryBackend::new(encoder.base_values(), 0, 8, block_size, 1);
+ std::env::remove_var("PAR2RS_CREATE_GF16");
+
+ if backend.selected_method() != CreateGf16Method::Avx2XorJit {
+ return;
+ }
+
+ let input = vec![0u8; block_size];
+ backend.begin_chunk(block_size);
+ for source_idx in 0..3 {
+ backend.add_input(source_idx, &input);
+ }
+
+ backend.build_xor_jit_requests(0, false);
+ let req = backend.xor_jit_req_storage[0];
+ let output_prefetch = xor_jit_output_prefetch_ptr(req, 0, 0).map(|ptr| ptr as usize);
+ let last_prefetch = xor_jit_input_prefetch_ptr(req, 0, req.num_outputs - 1)
+ .map(|ptr| ptr as usize)
+ .expect("final output should schedule input prefetch on non-final round");
+ let request_input_end = req.input + req.num_chunks * req.chunk_size * req.input_grouping;
+ let request_output_end = req.output + req.num_outputs * req.len;
+
+ assert_eq!(output_prefetch, Some(req.output + req.chunk_size));
+ assert!(output_prefetch.unwrap() < request_output_end);
+ assert!(last_prefetch >= req.input + req.chunk_size * req.input_grouping);
+ assert!(last_prefetch < request_input_end);
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn forced_xor_jit_backend_uses_packed_bitplane_layout() {
+ let _guard = env_lock().lock().unwrap();
+ std::env::set_var("PAR2RS_CREATE_GF16", "xor-jit");
+
+ let encoder = RecoveryBlockEncoder::new(1024 * 1024, 24);
+ let backend = CreateRecoveryBackend::new(encoder.base_values(), 0, 7, 1024 * 1024, 1);
+ std::env::remove_var("PAR2RS_CREATE_GF16");
+
+ if backend.selected_method() != CreateGf16Method::Avx2XorJit {
+ return;
+ }
+
+ let layout = backend
+ .xor_jit_layout
+ .expect("forced XOR-JIT should initialize packed layout");
+ assert_eq!(layout.chunk_len, 131_584);
+ assert_eq!(layout.segment_count_for(layout.aligned_slice_len), 8);
+ assert_eq!(
+ layout.input_offset(layout.chunk_len, 0, layout.chunk_len),
+ backend.input_grouping * layout.chunk_len
+ );
+ assert_eq!(
+ layout.output_offset(layout.chunk_len, 0, layout.chunk_len),
+ 7 * layout.chunk_len
+ );
+ assert_eq!(backend.staging[0].inputs.len(), layout.input_storage_len());
+ assert_eq!(backend.output_chunks.len(), layout.output_storage_len());
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn forced_xor_jit_backend_honors_segment_len_override() {
+ let _guard = env_lock().lock().unwrap();
+ std::env::set_var("PAR2RS_CREATE_GF16", "xor-jit");
+ std::env::set_var(XOR_JIT_SEGMENT_LEN_ENV, "65536");
+
+ let encoder = RecoveryBlockEncoder::new(1024 * 1024, 24);
+ let backend = CreateRecoveryBackend::new(encoder.base_values(), 0, 7, 1024 * 1024, 1);
+ std::env::remove_var(XOR_JIT_SEGMENT_LEN_ENV);
+ std::env::remove_var("PAR2RS_CREATE_GF16");
+
+ if backend.selected_method() != CreateGf16Method::Avx2XorJit {
+ return;
+ }
+
+ let layout = backend
+ .xor_jit_layout
+ .expect("forced XOR-JIT should initialize packed layout");
+ assert_eq!(layout.chunk_len, 66_048);
+ assert_eq!(layout.segment_count_for(layout.aligned_slice_len), 16);
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn forced_xor_jit_backend_reserves_turbo_two_phase_request_capacity() {
+ let _guard = env_lock().lock().unwrap();
+ std::env::set_var("PAR2RS_CREATE_GF16", "xor-jit");
+ std::env::set_var(XOR_JIT_SEGMENT_LEN_ENV, "8192");
+
+ let block_size = 256 * 1024;
+ let encoder = RecoveryBlockEncoder::new(block_size, 24);
+ let mut backend = CreateRecoveryBackend::new(encoder.base_values(), 0, 7, block_size, 2);
+
+ if backend.selected_method() != CreateGf16Method::Avx2XorJit {
+ std::env::remove_var(XOR_JIT_SEGMENT_LEN_ENV);
+ std::env::remove_var("PAR2RS_CREATE_GF16");
+ return;
+ }
+
+ let input = vec![0u8; block_size];
+ backend.begin_chunk(block_size);
+ std::env::remove_var(XOR_JIT_SEGMENT_LEN_ENV);
+ std::env::remove_var("PAR2RS_CREATE_GF16");
+ for source_idx in 0..3 {
+ backend.add_input(source_idx, &input);
+ }
+
+ let request_count = backend.build_xor_jit_requests(0, false);
+ assert_eq!(request_count, 4);
+ assert_eq!(
+ backend.xor_jit_req_storage.len(),
+ xor_jit_request_capacity(backend.workers.worker_count())
+ );
+ assert!(request_count <= backend.xor_jit_req_storage.len());
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ fn build_xor_jit_test_coeffs(base_values: &[u16], recovery_count: usize) -> Vec {
+ let mut cache = XorJitPreparedCoeffCache::new();
+ (0..recovery_count)
+ .flat_map(|exponent| {
+ base_values
+ .iter()
+ .map(move |&base| Galois16::new(base).pow(exponent as u16).value())
+ })
+ .map(|value| CreateCoeff::new(value, false, true, &mut cache))
+ .collect()
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn direct_xor_jit_compute_job_matches_encoder_across_x4_x2_and_x1_paths() {
+ let source_count = 7;
+ let recovery_count = 7;
+ let block_size = 241;
+ let encoder = RecoveryBlockEncoder::new(block_size, source_count);
+ let coeffs = build_xor_jit_test_coeffs(encoder.base_values(), recovery_count);
+ let inputs = deterministic_inputs(source_count, block_size);
+ let refs = inputs.iter().map(Vec::as_slice).collect::>();
+ let source_indices = (0..source_count).collect::>();
+ let mut packed_inputs = vec![0u8; source_count * 256];
+ let mut outputs = vec![0u8; recovery_count * 256];
+
+ for (idx, input) in inputs.iter().enumerate() {
+ let start = idx * 256;
+ packed_inputs[start..start + input.len()].copy_from_slice(input);
+ }
+
+ let job = ComputeJob {
+ method: CreateGf16Method::Avx2XorJit,
+ input_base: packed_inputs.as_ptr() as usize,
+ output_base: outputs.as_mut_ptr() as usize,
+ coeffs: coeffs.as_ptr() as usize,
+ source_indices: source_indices.as_ptr() as usize,
+ source_count,
+ batch_len: source_count,
+ aligned_chunk_len: 256,
+ segment_start: 0,
+ segment_len: block_size,
+ output_start: 0,
+ output_end: recovery_count,
+ };
+
+ process_compute_job(job, &mut WorkerContext::default());
+
+ for recovery_idx in 0..recovery_count {
+ let expected = encoder
+ .encode_recovery_block(recovery_idx as u16, &refs)
+ .expect("encode recovery block");
+ let start = recovery_idx * 256;
+ assert_eq!(&outputs[start..start + block_size], expected.as_slice());
+ assert!(outputs[start + block_size..start + 256]
+ .iter()
+ .all(|&byte| byte == 0));
+ }
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ #[ignore]
+ fn dump_xor_jit_prepared_staging_for_compare() {
+ let output_path = std::env::var("PAR2RS_XOR_JIT_PREPARED_DUMP_PATH")
+ .expect("PAR2RS_XOR_JIT_PREPARED_DUMP_PATH");
+ let slice_len = std::env::var("PAR2RS_XOR_JIT_PREPARED_SLICE_LEN")
+ .ok()
+ .and_then(|value| value.parse::().ok())
+ .unwrap_or(1024 * 1024);
+ let chunk_len = std::env::var("PAR2RS_XOR_JIT_PREPARED_CHUNK_LEN")
+ .ok()
+ .and_then(|value| value.parse::().ok())
+ .unwrap_or(128 * 1024);
+ let input_grouping = std::env::var("PAR2RS_XOR_JIT_PREPARED_INPUT_GROUPING")
+ .ok()
+ .and_then(|value| value.parse::().ok())
+ .unwrap_or(12);
+ let slot = std::env::var("PAR2RS_XOR_JIT_PREPARED_SLOT")
+ .ok()
+ .and_then(|value| value.parse::().ok())
+ .unwrap_or(5);
+ let src_len = std::env::var("PAR2RS_XOR_JIT_PREPARED_SRC_LEN")
+ .ok()
+ .and_then(|value| value.parse::().ok())
+ .unwrap_or(slice_len);
+
+ let layout = XorJitBitplaneLayout::new(
+ slice_len,
+ chunk_len,
+ input_grouping,
+ 1,
+ xor_jit_create_avx2_method_info(),
+ );
+ let mut staging = StagingArea::new(input_grouping, layout.input_storage_len(), 1);
+ let input = (0..src_len)
+ .map(|idx| ((idx * 37 + 11) & 0xff) as u8)
+ .collect::>();
+
+ prepare_xor_jit_bitplane_staging(layout, &mut staging, slot, slice_len, &input);
+ std::fs::write(output_path, &staging.inputs[..]).expect("write prepared staging dump");
+ }
}
diff --git a/src/create/context.rs b/src/create/context.rs
index fb8550cd..936893a8 100644
--- a/src/create/context.rs
+++ b/src/create/context.rs
@@ -7,6 +7,7 @@ use super::error_helpers::{
create_file, create_new_output_file, get_metadata, open_for_reading, packet_write_error,
};
use super::packet_generator::generate_recovery_set_id;
+use super::profile::{CreateProfile, CreateProfileCounters, CreateProfilePhase};
use super::progress::CreateReporter;
use super::source_file::{normalize_packet_path, packet_name_from_path, SourceFileInfo};
use super::types::CreateConfig;
@@ -37,6 +38,16 @@ fn write_recovery_slice_packet(
recovery_data: &[u8],
recovery_set_id: RecoverySetId,
) -> std::io::Result<()> {
+ let header = build_recovery_slice_header(exponent, recovery_data, recovery_set_id);
+ writer.write_all(&header)?;
+ writer.write_all(recovery_data)
+}
+
+fn build_recovery_slice_header(
+ exponent: u32,
+ recovery_data: &[u8],
+ recovery_set_id: RecoverySetId,
+) -> [u8; 68] {
use md5::{Digest, Md5};
let packet_length = 8 + 8 + 16 + 16 + 16 + 4 + recovery_data.len() as u64;
@@ -49,13 +60,14 @@ fn write_recovery_slice_packet(
hasher.update(recovery_data);
let computed_md5 = hasher.finalize();
- writer.write_all(crate::packets::MAGIC_BYTES)?;
- writer.write_all(&packet_length.to_le_bytes())?;
- writer.write_all(&computed_md5)?;
- writer.write_all(recovery_set_id.as_bytes())?;
- writer.write_all(RECOVERY_PACKET_TYPE)?;
- writer.write_all(&exponent.to_le_bytes())?;
- writer.write_all(recovery_data)
+ let mut header = [0u8; 68];
+ header[0..8].copy_from_slice(crate::packets::MAGIC_BYTES);
+ header[8..16].copy_from_slice(&packet_length.to_le_bytes());
+ header[16..32].copy_from_slice(&computed_md5);
+ header[32..48].copy_from_slice(recovery_set_id.as_bytes());
+ header[48..64].copy_from_slice(RECOVERY_PACKET_TYPE);
+ header[64..68].copy_from_slice(&exponent.to_le_bytes());
+ header
}
/// Compute the chunk size for chunked processing.
@@ -81,6 +93,33 @@ fn calculate_chunk_size_impl(
aligned.clamp(4, block_size.min(MAX_CREATE_CHUNK_SIZE))
}
+#[derive(Debug, Clone, Copy)]
+struct CreateProcessConfig {
+ chunk_size: ChunkSize,
+ defer_hash_computation: bool,
+}
+
+impl CreateProcessConfig {
+ fn new(
+ block_size: BlockSize,
+ source_block_count: u32,
+ recovery_block_count: u32,
+ memory_limit: usize,
+ ) -> Self {
+ let chunk_size = ChunkSize::new(calculate_chunk_size_impl(
+ block_size.as_usize(),
+ source_block_count as usize,
+ recovery_block_count as usize,
+ memory_limit,
+ ));
+ let defer_hash_computation = chunk_size.as_usize() >= block_size.as_usize();
+ Self {
+ chunk_size,
+ defer_hash_computation,
+ }
+ }
+}
+
/// Per-file hash data computed during `encode_and_hash_files`.
struct FileHashState {
hash_16k: crate::domain::Md5Hash,
@@ -94,6 +133,12 @@ struct FileHashState {
/// Recovery blocks paired with their exponents, as returned by `encode_and_hash_files`.
type RecoveryBlockVec = Vec<(u16, Vec)>;
+struct EncodeProfile {
+ counters: CreateProfileCounters,
+ source_open_hash_prepass: std::time::Duration,
+ recovery_chunk_processing: std::time::Duration,
+}
+
/// Encode all source files into recovery blocks while simultaneously computing
/// file/block hashes in a single pass.
///
@@ -111,10 +156,12 @@ fn encode_and_hash_files(
recovery_count: usize,
thread_count: usize,
reporter: &dyn CreateReporter,
-) -> CreateResult<(RecoveryBlockVec, Vec)> {
+ profile_enabled: bool,
+) -> CreateResult<(RecoveryBlockVec, Vec, Option)> {
use crate::checksum::compute_file_id;
use crate::create::source_file::BlockChecksum;
- use crc32fast::Hasher as Crc32Hasher;
+ use crate::parpar_hasher::hasher_input_dyn::HasherInputDyn;
+ use crate::parpar_hasher::BlockHash;
use md5::{Digest, Md5};
use std::fs::File;
use std::io::{Read, Seek};
@@ -124,50 +171,212 @@ fn encode_and_hash_files(
.build()
.map_err(|err| CreateError::Other(format!("failed to create thread pool: {err}")))?;
+ // True when the recovery loop traverses each file in strict
+ // sequential byte order (chunk_len == block_size for every block),
+ // letting us run HasherInputDyn inline with recovery generation —
+ // mirroring par2cmdline-turbo's `deferhashcomputation = true` path.
+ // Otherwise (slim mode), the recovery loop is chunk-offset-major
+ // across blocks, so per-file bytes are non-sequential there and
+ // we must hash each file in a separate sequential pass — that's
+ // turbo's `deferhashcomputation = false` path
+ // (`Par2CreatorSourceFile::Open` non-defer branch).
+ let defer_hash_to_recovery_loop = chunk_size >= block_size as usize;
+
+ let source_open_hash_start = profile_enabled.then(std::time::Instant::now);
let mut file_handles: Vec = Vec::with_capacity(source_files.len());
- let mut file_md5_states: Vec = Vec::with_capacity(source_files.len());
let mut file_16k_buffers: Vec> = Vec::with_capacity(source_files.len());
- let mut block_md5_states: Vec = Vec::with_capacity(source_block_count as usize);
- let mut block_crc32_states: Vec = Vec::with_capacity(source_block_count as usize);
+ // One HasherInputDyn per source file when we're going to drive it
+ // from the recovery loop. In slim mode we still allocate the slot
+ // but don't use it — kept symmetric for indexing.
+ let mut file_hashers: Vec> = Vec::with_capacity(source_files.len());
+
+ // Per-block outputs, populated as `get_block` calls retire each block.
+ let mut block_hashes: Vec> = vec![None; source_block_count as usize];
+
+ // Per-file file-MD5, populated either by the slim-mode pre-pass
+ // (turbo's non-defer `Open()` branch) or by `hasher.end()` after
+ // the recovery loop (defer branch).
+ let mut file_full_md5: Vec> = vec![None; source_files.len()];
// Per-file metadata: (block_count, global_block_offset)
let mut file_block_meta: Vec<(u32, u32)> = Vec::with_capacity(source_files.len());
let mut global_block_offset = 0u32;
for file in source_files {
- file_handles.push(open_for_reading(&file.path)?);
- file_md5_states.push(Md5::new());
+ if defer_hash_to_recovery_loop {
+ file_handles.push(open_for_reading(&file.path)?);
+ }
file_16k_buffers.push(vec![0u8; (file.size as usize).min(16 * 1024)]);
+ file_hashers.push(if defer_hash_to_recovery_loop {
+ Some(HasherInputDyn::new())
+ } else {
+ None
+ });
let block_count = file.calculate_block_count(block_size);
file_block_meta.push((block_count, global_block_offset));
- for _ in 0..block_count {
- block_md5_states.push(Md5::new());
- block_crc32_states.push(Crc32Hasher::new());
- }
global_block_offset += block_count;
}
- let recovery_blocks = pool.install(|| {
+ // Slim mode (turbo's `deferhashcomputation = false`): turbo computes
+ // file MD5, hash16k, and per-block (MD5, CRC32) in
+ // `Par2CreatorSourceFile::Open()` itself, *before* the recovery
+ // loop runs — because in slim mode the recovery loop visits each
+ // file in chunk-offset-major order across blocks, which is not
+ // sequential per-file. We mirror that here: a parallel-across-files
+ // pre-pass that streams each file end-to-end through HasherInputDyn,
+ // then reopen the recovery-loop handles afterwards. That avoids
+ // keeping two live FDs per source file at once on large input sets.
+ if !defer_hash_to_recovery_loop {
+ use rayon::prelude::*;
+ use std::io::SeekFrom;
+ // Each task gets its own File handle (we can't share &mut File
+ // across rayon tasks). Recovery-loop handles are opened only
+ // after this pre-pass completes.
+ let per_file_results: CreateResult> = pool.install(|| {
+ source_files
+ .par_iter()
+ .enumerate()
+ .map(|(file_idx, file)| -> CreateResult<_> {
+ let (block_count, g_offset) = file_block_meta[file_idx];
+ let mut handle = open_for_reading(&file.path)?;
+ handle
+ .seek(SeekFrom::Start(0))
+ .map_err(|e| CreateError::FileReadError {
+ file: file.path.to_string_lossy().to_string(),
+ source: e,
+ })?;
+
+ let mut hasher = HasherInputDyn::new();
+ let mut hash16k_buf = vec![0u8; (file.size as usize).min(16 * 1024)];
+ let mut block_hashes_local: Vec =
+ Vec::with_capacity(block_count as usize);
+
+ // 1 MiB I/O buffer, capped at min(blocksize, filesize),
+ // matching turbo's non-defer `Open()`.
+ let buffersize = (1024 * 1024)
+ .min(block_size as usize)
+ .min(file.size as usize)
+ .max(1);
+ let mut buf = vec![0u8; buffersize];
+
+ let mut offset: u64 = 0;
+ let mut blocknumber: u32 = 0;
+ let mut need: u64 = block_size;
+
+ while offset < file.size {
+ let want = (file.size - offset).min(buffersize as u64) as usize;
+ handle.read_exact(&mut buf[..want]).map_err(|e| {
+ CreateError::FileReadError {
+ file: file.path.to_string_lossy().to_string(),
+ source: e,
+ }
+ })?;
+
+ // Capture first 16 KiB for hash16k.
+ if offset < 16 * 1024 {
+ let cap_start = offset as usize;
+ let cap_end = (cap_start + want).min(16 * 1024);
+ hash16k_buf[cap_start..cap_end]
+ .copy_from_slice(&buf[..(cap_end - cap_start)]);
+ }
+
+ let mut used = 0usize;
+ while used < want {
+ let use_n = need.min((want - used) as u64) as usize;
+ hasher.update(&buf[used..used + use_n]);
+ used += use_n;
+ need -= use_n as u64;
+
+ if need == 0 {
+ let bh = hasher.get_block(0);
+ block_hashes_local.push(bh);
+ blocknumber += 1;
+ if blocknumber < block_count {
+ need = block_size;
+ }
+ }
+ }
+
+ offset += want as u64;
+ }
+
+ // Final short block: feed zero pad to BLOCK lane only.
+ if need > 0 && block_count > 0 {
+ let bh = hasher.get_block(need);
+ block_hashes_local.push(bh);
+ }
+
+ let file_md5 = hasher.end();
+
+ Ok((
+ file_idx,
+ g_offset,
+ hash16k_buf,
+ file_md5,
+ block_hashes_local,
+ ))
+ })
+ .collect()
+ });
+
+ for (file_idx, g_offset, hash16k_buf, file_md5, blocks) in per_file_results? {
+ file_16k_buffers[file_idx] = hash16k_buf;
+ file_full_md5[file_idx] = Some(crate::domain::Md5Hash::new(file_md5));
+ for (i, bh) in blocks.into_iter().enumerate() {
+ block_hashes[(g_offset + i as u32) as usize] = Some(bh);
+ }
+ }
+
+ file_handles = source_files
+ .iter()
+ .map(|file| open_for_reading(&file.path))
+ .collect::>>()?;
+ }
+ let source_hash_bytes_read = if !profile_enabled || defer_hash_to_recovery_loop {
+ 0
+ } else {
+ source_files.iter().map(|file| file.size).sum()
+ };
+ let source_hash_seek_count = if !profile_enabled || defer_hash_to_recovery_loop {
+ 0
+ } else {
+ source_files.len() as u64
+ };
+ let source_open_hash_prepass = source_open_hash_start
+ .map(|start| start.elapsed())
+ .unwrap_or_default();
+
+ let recovery_processing_start = profile_enabled.then(std::time::Instant::now);
+ let (recovery_blocks, recovery_counters) = pool.install(|| {
let mut backend = CreateRecoveryBackend::new(
base_values,
first_recovery_block,
recovery_count,
chunk_size,
+ thread_count,
);
+ let selected_backend = profile_enabled.then(|| format!("{:?}", backend.selected_method()));
let mut recovery_blocks = backend.recovery_blocks(block_size as usize);
+ let mut source_recovery_bytes_read = 0u64;
+ let mut source_seek_count = 0u64;
+ let mut recovery_chunk_count = 0u64;
+ let mut file_positions = vec![0u64; source_files.len()];
// Main chunk loop
let mut block_offset = 0u64;
while block_offset < block_size {
let chunk_len = ((block_size - block_offset) as usize).min(chunk_size);
+ if profile_enabled {
+ recovery_chunk_count += 1;
+ }
backend.begin_chunk(chunk_len);
let mut file_block_idx = 0usize;
for (file_idx, file) in source_files.iter().enumerate() {
- let (block_count, _) = file_block_meta[file_idx];
+ let (block_count, g_offset) = file_block_meta[file_idx];
for block_idx in 0..block_count {
let is_last = block_idx == block_count - 1;
let block_actual = if is_last && file.size % block_size != 0 {
@@ -185,18 +394,28 @@ fn encode_and_hash_files(
let chunk = backend.prepare_transfer_buffer(file_block_idx);
if bytes_to_read > 0 {
let file_pos = block_idx as u64 * block_size + block_offset;
- file_handles[file_idx]
- .seek(std::io::SeekFrom::Start(file_pos))
- .map_err(|e| CreateError::FileReadError {
- file: file.path.to_string_lossy().to_string(),
- source: e,
- })?;
+ if file_positions[file_idx] != file_pos {
+ file_handles[file_idx]
+ .seek(std::io::SeekFrom::Start(file_pos))
+ .map_err(|e| CreateError::FileReadError {
+ file: file.path.to_string_lossy().to_string(),
+ source: e,
+ })?;
+ file_positions[file_idx] = file_pos;
+ if profile_enabled {
+ source_seek_count += 1;
+ }
+ }
file_handles[file_idx]
.read_exact(&mut chunk[..bytes_to_read])
.map_err(|e| CreateError::FileReadError {
file: file.path.to_string_lossy().to_string(),
source: e,
})?;
+ if profile_enabled {
+ source_recovery_bytes_read += bytes_to_read as u64;
+ }
+ file_positions[file_idx] += bytes_to_read as u64;
if file_pos < 16 * 1024 {
let capture_start = file_pos as usize;
let capture_end = (capture_start + bytes_to_read).min(16 * 1024);
@@ -204,17 +423,42 @@ fn encode_and_hash_files(
file_16k_buffers[file_idx][capture_start..capture_end]
.copy_from_slice(&chunk[..capture_len]);
}
- file_md5_states[file_idx].update(&chunk[..bytes_to_read]);
}
- block_md5_states[file_block_idx].update(&chunk[..chunk_len]);
- block_crc32_states[file_block_idx].update(&chunk[..chunk_len]);
+
+ // Defer mode: hash inline with recovery, mirroring
+ // par2cmdline-turbo's `Par2CreatorSourceFile::UpdateHashes`
+ // path. Each block is read in full this iteration
+ // (chunk_len == block_size), and we visit all blocks
+ // of file N before any block of file N+1, so per-file
+ // bytes arrive in strict sequential order — exactly
+ // what HasherInputDyn requires.
+ //
+ // Slim mode skips this; a separate per-file pass
+ // below drives HasherInputDyn for those files.
+ if defer_hash_to_recovery_loop {
+ let hasher = file_hashers[file_idx]
+ .as_mut()
+ .expect("hasher allocated in defer mode");
+ if bytes_to_read > 0 {
+ hasher.update(&chunk[..bytes_to_read]);
+ }
+ // PAR2 spec: BLOCK hashes include trailing zero
+ // pad up to block_size; FILE hash does not. The
+ // hasher's `get_block(zero_pad)` enforces both
+ // halves of that.
+ let zero_pad = (chunk_len - bytes_to_read) as u64;
+ let bh = hasher.get_block(zero_pad);
+ block_hashes[(g_offset + block_idx) as usize] = Some(bh);
+ }
}
backend.add_transfer_input(file_block_idx, file_block_idx);
file_block_idx += 1;
}
}
- backend.finish_chunk(&mut recovery_blocks, block_size as usize);
+ if !backend.finish_chunk(&mut recovery_blocks, block_size as usize) {
+ return Err(CreateError::XorJitChecksumValidationFailed);
+ }
block_offset += chunk_len as u64;
let progress =
@@ -225,18 +469,44 @@ fn encode_and_hash_files(
);
}
- Ok::<_, CreateError>(recovery_blocks)
+ let counters = CreateProfileCounters {
+ source_recovery_bytes_read,
+ source_seek_count,
+ recovery_chunk_count,
+ selected_backend,
+ ..CreateProfileCounters::default()
+ };
+ Ok::<_, CreateError>((recovery_blocks, counters))
})?;
+ let recovery_chunk_processing = recovery_processing_start
+ .map(|start| start.elapsed())
+ .unwrap_or_default();
+ let encode_profile = profile_enabled.then(|| {
+ let mut counters = recovery_counters;
+ counters.source_hash_bytes_read = source_hash_bytes_read;
+ counters.source_seek_count += source_hash_seek_count;
+ EncodeProfile {
+ counters,
+ source_open_hash_prepass,
+ recovery_chunk_processing,
+ }
+ });
+
+ // Finalize file MD5s. In defer mode each per-file hasher has been
+ // streamed in block order through the recovery loop and we now
+ // call `end()` to retrieve the file MD5 — turbo's
+ // `Par2Creator::FinishFileHashComputation` path. In slim mode
+ // `file_full_md5[i]` was already populated by the pre-pass above.
+ if defer_hash_to_recovery_loop {
+ for (i, slot) in file_hashers.into_iter().enumerate() {
+ let h = slot.expect("hasher allocated in defer mode");
+ file_full_md5[i] = Some(crate::domain::Md5Hash::new(h.end()));
+ }
+ }
- // Finalize file MD5s and block checksums
- let finalized_file_md5s: Vec<[u8; 16]> = file_md5_states
+ let full_file_hashes: Vec = file_full_md5
.into_iter()
- .map(|s| {
- let h = s.finalize();
- let mut b = [0u8; 16];
- b.copy_from_slice(&h);
- b
- })
+ .map(|o| o.expect("file MD5 populated by either defer or slim path"))
.collect();
let file_16k_hashes = file_16k_buffers
@@ -249,29 +519,26 @@ fn encode_and_hash_files(
for (file_idx, file) in source_files.iter().enumerate() {
let (block_count, g_offset) = file_block_meta[file_idx];
- let full_md5 = crate::domain::Md5Hash::new(finalized_file_md5s[file_idx]);
+ let full_md5 = full_file_hashes[file_idx];
let hash_16k = file_16k_hashes[file_idx];
let filename = file.packet_name().as_bytes();
let file_id = compute_file_id(&hash_16k, file.size, filename);
let mut checksums = Vec::with_capacity(block_count as usize);
for block_idx in 0..block_count {
- let md5_raw = block_md5_states[global_block_idx].clone().finalize();
- let mut md5_bytes = [0u8; 16];
- md5_bytes.copy_from_slice(&md5_raw);
- let crc32 = block_crc32_states[global_block_idx].clone().finalize();
-
+ let bh =
+ block_hashes[global_block_idx].expect("block hash populated by defer or slim path");
log::debug!(
"Block {}: MD5={:02x}{:02x}..., CRC32={:08x}",
global_block_idx,
- md5_bytes[0],
- md5_bytes[1],
- crc32
+ bh.md5[0],
+ bh.md5[1],
+ bh.crc32
);
checksums.push(BlockChecksum {
- crc32,
- hash: crate::domain::Md5Hash::new(md5_bytes),
+ crc32: bh.crc32,
+ hash: crate::domain::Md5Hash::new(bh.md5),
global_index: g_offset + block_idx,
});
global_block_idx += 1;
@@ -289,7 +556,7 @@ fn encode_and_hash_files(
});
}
- Ok((recovery_blocks, hash_states))
+ Ok((recovery_blocks, hash_states, encode_profile))
}
/// Populate `source_files` from the hash data computed during `encode_and_hash_files`.
@@ -345,6 +612,9 @@ pub struct CreateContext {
/// Output PAR2 files created
output_files: Vec,
+
+ /// Opt-in create phase/counter profiler.
+ profile: Option,
}
impl CreateContext {
@@ -356,6 +626,7 @@ impl CreateContext {
config: CreateConfig,
reporter: Box,
) -> CreateResult {
+ let profile = CreateProfile::from_env();
let mut context = CreateContext {
config,
reporter,
@@ -366,12 +637,21 @@ impl CreateContext {
recovery_block_count: 0,
recovery_blocks: Vec::new(),
output_files: Vec::new(),
+ profile,
};
// Perform initial setup
+ let scan_start = std::time::Instant::now();
context.scan_source_files()?;
+ if let Some(profile) = &mut context.profile {
+ profile.add_duration(CreateProfilePhase::SourceScanMetadata, scan_start.elapsed());
+ let counters = profile.counters_mut();
+ counters.source_file_count = context.source_files.len();
+ counters.source_bytes = context.source_files.iter().map(|file| file.size).sum();
+ }
context.calculate_block_size()?;
context.calculate_recovery_blocks()?;
+ context.record_static_profile_counters();
Ok(context)
}
@@ -396,6 +676,9 @@ impl CreateContext {
// Report completion
self.reporter.report_complete(&self.output_files);
+ if let Some(profile) = &self.profile {
+ profile.emit();
+ }
Ok(())
}
@@ -605,6 +888,27 @@ impl CreateContext {
Ok(())
}
+ fn process_config(&self) -> CreateProcessConfig {
+ CreateProcessConfig::new(
+ self.block_size,
+ self.source_block_count,
+ self.recovery_block_count,
+ self.config.memory_limit.unwrap_or(DEFAULT_MEMORY_LIMIT),
+ )
+ }
+
+ fn record_static_profile_counters(&mut self) {
+ let process_config = self.process_config();
+ if let Some(profile) = &mut self.profile {
+ let counters = profile.counters_mut();
+ counters.block_size = self.block_size.as_u64();
+ counters.source_block_count = self.source_block_count;
+ counters.recovery_block_count = self.recovery_block_count;
+ counters.chunk_size = process_config.chunk_size.as_usize();
+ counters.defer_hash_computation = process_config.defer_hash_computation;
+ }
+ }
+
fn checked_recovery_block_count(&self, recovery_blocks: u64) -> CreateResult {
if recovery_blocks > 65536 {
return Err(CreateError::Other(
@@ -691,7 +995,8 @@ impl CreateContext {
let encoder =
RecoveryBlockEncoder::new(self.block_size.as_usize(), self.source_block_count as usize);
- let chunk_size = self.calculate_chunk_size();
+ let process_config = self.process_config();
+ let chunk_size = process_config.chunk_size;
self.reporter.report_scanning_files(
0,
@@ -702,7 +1007,7 @@ impl CreateContext {
),
);
- let (recovery_blocks, hash_states) = encode_and_hash_files(
+ let (recovery_blocks, hash_states, encode_profile) = encode_and_hash_files(
&self.source_files,
self.block_size.as_u64(),
chunk_size.as_usize(),
@@ -712,7 +1017,25 @@ impl CreateContext {
self.recovery_block_count as usize,
self.config.effective_threads(),
self.reporter.as_ref(),
+ self.profile.is_some(),
)?;
+ if let (Some(profile), Some(encode_profile)) = (&mut self.profile, encode_profile) {
+ profile.add_duration(
+ CreateProfilePhase::SourceOpenHashPrepass,
+ encode_profile.source_open_hash_prepass,
+ );
+ profile.add_duration(
+ CreateProfilePhase::RecoveryChunkProcessing,
+ encode_profile.recovery_chunk_processing,
+ );
+ let counters = profile.counters_mut();
+ counters.source_hash_bytes_read = encode_profile.counters.source_hash_bytes_read;
+ counters.source_recovery_bytes_read =
+ encode_profile.counters.source_recovery_bytes_read;
+ counters.source_seek_count = encode_profile.counters.source_seek_count;
+ counters.recovery_chunk_count = encode_profile.counters.recovery_chunk_count;
+ counters.selected_backend = encode_profile.counters.selected_backend;
+ }
finalize_file_hashes(hash_states, &mut self.source_files)?;
@@ -726,17 +1049,6 @@ impl CreateContext {
Ok(())
}
- /// Calculate optimal chunk size for processing.
- /// Reference: par2cmdline-turbo/src/par2creator.cpp:329-360 CalculateProcessBlockSize()
- fn calculate_chunk_size(&self) -> ChunkSize {
- ChunkSize::new(calculate_chunk_size_impl(
- self.block_size.as_usize(),
- self.source_block_count as usize,
- self.recovery_block_count as usize,
- self.config.memory_limit.unwrap_or(DEFAULT_MEMORY_LIMIT),
- ))
- }
-
/// Write PAR2 files: index file (critical packets only) + volume files (critical + recovery)
///
/// Reference: par2cmdline-turbo/src/par2creator.cpp WriteCriticalPackets() and
@@ -756,6 +1068,7 @@ impl CreateContext {
// Generate all critical packets
// Reference: par2cmdline-turbo/src/par2creator.cpp CreateMainPacket(), CreateCreatorPacket()
+ let packet_serialization_start = std::time::Instant::now();
let main_packet = generate_main_packet(
recovery_set_id,
self.block_size.as_u64(),
@@ -790,6 +1103,12 @@ impl CreateContext {
write_file_verification_packet(&mut critical_bytes, packet)
.map_err(|e| packet_write_error("file verification packet", e))?;
}
+ if let Some(profile) = &mut self.profile {
+ profile.add_duration(
+ CreateProfilePhase::CriticalPacketSerialization,
+ packet_serialization_start.elapsed(),
+ );
+ }
// Determine output directory and base name
let output_path = Path::new(&self.config.output_name);
@@ -857,6 +1176,7 @@ impl CreateContext {
// Write index file: critical packets only, no recovery data
// Reference: par2cmdline-turbo creates base.par2 with no recovery slices
+ let output_write_start = std::time::Instant::now();
let mut index_file = open_output(&index_path, self.config.overwrite_existing)?;
index_file
.write_all(&critical_bytes)
@@ -872,10 +1192,17 @@ impl CreateContext {
})?;
self.output_files
.push(index_path.to_string_lossy().to_string());
+ if let Some(profile) = &mut self.profile {
+ profile.add_duration(
+ CreateProfilePhase::OutputFileWrites,
+ output_write_start.elapsed(),
+ );
+ }
// Write each volume file: critical packets + its slice of recovery blocks
// Reference: par2cmdline-turbo/src/par2creator.cpp WriteRecoveryPackets()
for (entry, vol_path) in plan.iter().zip(volume_paths) {
+ let output_write_start = std::time::Instant::now();
let mut vol_file = open_output(&vol_path, self.config.overwrite_existing)?;
vol_file
@@ -885,25 +1212,66 @@ impl CreateContext {
source: e,
})?;
+ if let Some(profile) = &mut self.profile {
+ profile.add_duration(
+ CreateProfilePhase::OutputFileWrites,
+ output_write_start.elapsed(),
+ );
+ }
+
// Write recovery slice packets for this volume
for i in 0..entry.block_count {
let packet_exponent = entry.first_exponent + i;
let local_idx = (packet_exponent - self.config.first_recovery_block) as usize;
let (recovery_exponent, recovery_data) = &self.recovery_blocks[local_idx];
- write_recovery_slice_packet(
- &mut vol_file,
- *recovery_exponent as u32,
- recovery_data,
- recovery_set_id,
- )
- .map_err(|e| packet_write_error("recovery packet", e))?;
+ if self.profile.is_some() {
+ let packet_serialization_start = std::time::Instant::now();
+ let packet_header = build_recovery_slice_header(
+ *recovery_exponent as u32,
+ recovery_data,
+ recovery_set_id,
+ );
+ if let Some(profile) = &mut self.profile {
+ profile.add_duration(
+ CreateProfilePhase::RecoveryPacketSerialization,
+ packet_serialization_start.elapsed(),
+ );
+ }
+
+ let output_write_start = std::time::Instant::now();
+ vol_file
+ .write_all(&packet_header)
+ .and_then(|_| vol_file.write_all(recovery_data))
+ .map_err(|e| packet_write_error("recovery packet", e))?;
+ if let Some(profile) = &mut self.profile {
+ profile.add_duration(
+ CreateProfilePhase::OutputFileWrites,
+ output_write_start.elapsed(),
+ );
+ }
+ } else {
+ write_recovery_slice_packet(
+ &mut vol_file,
+ *recovery_exponent as u32,
+ recovery_data,
+ recovery_set_id,
+ )
+ .map_err(|e| packet_write_error("recovery packet", e))?;
+ }
}
+ let output_write_start = std::time::Instant::now();
vol_file.flush().map_err(|e| CreateError::FileCreateError {
file: vol_path.to_string_lossy().to_string(),
source: e,
})?;
+ if let Some(profile) = &mut self.profile {
+ profile.add_duration(
+ CreateProfilePhase::OutputFileWrites,
+ output_write_start.elapsed(),
+ );
+ }
self.output_files
.push(vol_path.to_string_lossy().to_string());
}
@@ -934,7 +1302,73 @@ impl CreateContext {
#[cfg(test)]
mod tests {
+ use super::super::progress::SilentCreateReporter;
+ use super::super::types::CreateConfig;
use super::*;
+ use proptest::prelude::*;
+ use std::sync::{Mutex, OnceLock};
+
+ fn env_lock() -> std::sync::MutexGuard<'static, ()> {
+ static LOCK: OnceLock> = OnceLock::new();
+ LOCK.get_or_init(|| Mutex::new(())).lock().unwrap()
+ }
+
+ fn test_context(output_name: &str, base_path: Option) -> CreateContext {
+ CreateContext {
+ config: CreateConfig {
+ output_name: output_name.to_string(),
+ base_path,
+ ..CreateConfig::default()
+ },
+ reporter: Box::new(SilentCreateReporter),
+ recovery_set_id: None,
+ source_files: Vec::new(),
+ block_size: BlockSize::new(0),
+ source_block_count: 0,
+ recovery_block_count: 0,
+ recovery_blocks: Vec::new(),
+ output_files: Vec::new(),
+ profile: None,
+ }
+ }
+
+ fn source_info(path: impl Into, size: u64, index: usize) -> SourceFileInfo {
+ SourceFileInfo::new(path.into(), size, index)
+ }
+
+ #[test]
+ fn default_output_base_path_uses_parent_or_current_directory() {
+ assert!(default_output_base_path("out.par2").as_os_str().is_empty());
+ assert_eq!(
+ default_output_base_path("nested/out.par2"),
+ std::path::PathBuf::from("nested")
+ );
+ }
+
+ #[test]
+ fn recovery_slice_header_contains_expected_md5_and_layout() {
+ let set_id = RecoverySetId::new([0xCD; 16]);
+ let recovery_data = [1u8, 2, 3, 4, 5, 6];
+ let header = build_recovery_slice_header(9, &recovery_data, set_id);
+
+ assert_eq!(&header[0..8], crate::packets::MAGIC_BYTES);
+ assert_eq!(
+ u64::from_le_bytes(header[8..16].try_into().unwrap()),
+ 8 + 8 + 16 + 16 + 16 + 4 + recovery_data.len() as u64
+ );
+ assert_eq!(&header[32..48], set_id.as_bytes());
+ assert_eq!(&header[48..64], RECOVERY_PACKET_TYPE);
+ assert_eq!(u32::from_le_bytes(header[64..68].try_into().unwrap()), 9);
+
+ use md5::{Digest, Md5};
+ let mut hasher = Md5::new();
+ hasher.update(set_id.as_bytes());
+ hasher.update(RECOVERY_PACKET_TYPE);
+ hasher.update(9u32.to_le_bytes());
+ hasher.update(recovery_data);
+ let expected_md5 = hasher.finalize();
+ assert_eq!(&header[16..32], &expected_md5[..]);
+ }
// --- calculate_chunk_size_impl ---
@@ -985,6 +1419,51 @@ mod tests {
assert_eq!(result, block_size);
}
+ #[test]
+ fn chunk_size_is_capped_like_turbo() {
+ let block_size = 64 * 1024 * 1024;
+ let recovery_count = 1;
+ let source_count = 4;
+ let memory_limit = block_size * (recovery_count + 30);
+
+ let result =
+ calculate_chunk_size_impl(block_size, source_count, recovery_count, memory_limit);
+
+ assert_eq!(result, MAX_CREATE_CHUNK_SIZE);
+ }
+
+ #[test]
+ fn chunk_size_keeps_large_full_blocks_capped_like_turbo() {
+ let block_size = 64 * 1024 * 1024;
+ let recovery_count = 2;
+ let source_count = 10;
+ let memory_limit = block_size * (recovery_count + 26);
+
+ let result =
+ calculate_chunk_size_impl(block_size, source_count, recovery_count, memory_limit);
+
+ assert_eq!(result, MAX_CREATE_CHUNK_SIZE);
+ }
+
+ #[test]
+ fn process_config_marks_large_capped_blocks_non_deferred() {
+ let block_size = BlockSize::new(64 * 1024 * 1024);
+ let process_config =
+ CreateProcessConfig::new(block_size, 10, 2, block_size.as_usize() * 28);
+
+ assert_eq!(process_config.chunk_size.as_usize(), MAX_CREATE_CHUNK_SIZE);
+ assert!(!process_config.defer_hash_computation);
+ }
+
+ #[test]
+ fn process_config_marks_full_block_chunks_deferred() {
+ let block_size = BlockSize::new(1024 * 1024);
+ let process_config = CreateProcessConfig::new(block_size, 4, 2, DEFAULT_MEMORY_LIMIT);
+
+ assert_eq!(process_config.chunk_size.as_usize(), block_size.as_usize());
+ assert!(process_config.defer_hash_computation);
+ }
+
#[test]
fn chunk_size_respects_explicit_memory_limit() {
let block_size = 1024;
@@ -992,6 +1471,203 @@ mod tests {
assert_eq!(result, 16);
}
+ #[test]
+ fn packet_name_for_path_uses_relative_path_when_base_path_matches() {
+ let tmp = tempfile::tempdir().unwrap();
+ let base = tmp.path().join("base");
+ let nested = base.join("nested");
+ std::fs::create_dir_all(&nested).unwrap();
+ let file = nested.join("file.bin");
+ std::fs::write(&file, b"hello").unwrap();
+
+ let context = test_context("out.par2", Some(base.clone()));
+ assert_eq!(
+ context.packet_name_for_path(&file).unwrap(),
+ normalize_packet_path(std::path::Path::new("nested/file.bin"))
+ );
+ }
+
+ #[test]
+ fn packet_name_for_path_uses_canonical_base_when_literal_prefix_differs() {
+ let tmp = tempfile::tempdir().unwrap();
+ let base = tmp.path().join("base");
+ let nested = base.join("nested");
+ std::fs::create_dir_all(&nested).unwrap();
+ let file = nested.join("file.bin");
+ std::fs::write(&file, b"hello").unwrap();
+
+ let base_with_dotdots = tmp.path().join("base").join("..").join("base");
+ let context = test_context("out.par2", Some(base_with_dotdots));
+ assert_eq!(
+ context.packet_name_for_path(&file).unwrap(),
+ normalize_packet_path(std::path::Path::new("nested/file.bin"))
+ );
+ }
+
+ #[test]
+ fn packet_name_for_path_falls_back_when_outside_base_path() {
+ let tmp = tempfile::tempdir().unwrap();
+ let base = tmp.path().join("base");
+ let other = tmp.path().join("elsewhere");
+ std::fs::create_dir_all(&base).unwrap();
+ std::fs::create_dir_all(&other).unwrap();
+ let file = other.join("file.bin");
+ std::fs::write(&file, b"hello").unwrap();
+
+ let context = test_context("out.par2", Some(base));
+ assert_eq!(
+ context.packet_name_for_path(&file).unwrap(),
+ packet_name_from_path(&file)
+ );
+ }
+
+ #[test]
+ fn packet_base_path_prefers_explicit_base_path_and_otherwise_uses_output_parent() {
+ let explicit = test_context(
+ "nested/out.par2",
+ Some(std::path::PathBuf::from("/tmp/base")),
+ );
+ assert_eq!(
+ explicit.packet_base_path().as_ref(),
+ std::path::Path::new("/tmp/base")
+ );
+
+ let derived = test_context("nested/out.par2", None);
+ assert_eq!(
+ derived.packet_base_path().as_ref(),
+ std::path::Path::new("nested")
+ );
+ }
+
+ #[test]
+ fn scan_source_files_skips_empty_files_and_uses_output_directory_for_packet_names() {
+ let tmp = tempfile::tempdir().unwrap();
+ let nested = tmp.path().join("nested");
+ std::fs::create_dir_all(&nested).unwrap();
+ let empty = tmp.path().join("empty.bin");
+ let data = nested.join("data.bin");
+ let output = tmp.path().join("archive.par2");
+ std::fs::write(&empty, []).unwrap();
+ std::fs::write(&data, b"hello world").unwrap();
+
+ let mut context = test_context(output.to_str().unwrap(), None);
+ context.config.source_files = vec![empty, data.clone()];
+
+ context.scan_source_files().unwrap();
+
+ assert_eq!(context.source_files.len(), 1);
+ assert_eq!(context.source_files[0].path, data);
+ assert_eq!(context.source_files[0].packet_name(), "nested/data.bin");
+ assert_eq!(context.source_files[0].size, 11);
+ }
+
+ #[test]
+ fn scan_source_files_rejects_missing_and_all_empty_inputs() {
+ let tmp = tempfile::tempdir().unwrap();
+ let empty = tmp.path().join("empty.bin");
+ std::fs::write(&empty, []).unwrap();
+
+ let mut empty_only = test_context(tmp.path().join("out.par2").to_str().unwrap(), None);
+ empty_only.config.source_files = vec![empty];
+ assert!(matches!(
+ empty_only.scan_source_files(),
+ Err(CreateError::EmptySourceFiles)
+ ));
+
+ let missing = tmp.path().join("missing.bin");
+ let mut missing_ctx = test_context(tmp.path().join("out.par2").to_str().unwrap(), None);
+ missing_ctx.config.source_files = vec![missing.clone()];
+ assert!(matches!(
+ missing_ctx.scan_source_files(),
+ Err(CreateError::FileNotFound(path)) if path == missing.to_string_lossy()
+ ));
+ }
+
+ #[test]
+ fn calculate_block_size_rejects_target_smaller_than_file_count() {
+ let mut context = test_context("out.par2", None);
+ context.config.source_block_count = Some(SourceBlockCount::new(1));
+ context.source_files = vec![source_info("a.bin", 8, 0), source_info("b.bin", 12, 1)];
+
+ let error = context.calculate_block_size().unwrap_err();
+ assert!(error
+ .to_string()
+ .contains("cannot be smaller than the number of files"));
+ }
+
+ #[test]
+ fn calculate_block_size_uses_largest_file_when_target_equals_file_count() {
+ let mut context = test_context("out.par2", None);
+ context.config.source_block_count = Some(SourceBlockCount::new(2));
+ context.source_files = vec![source_info("a.bin", 5, 0), source_info("b.bin", 10, 1)];
+
+ context.calculate_block_size().unwrap();
+
+ assert_eq!(context.block_size.as_u64(), 12);
+ assert_eq!(context.source_block_count, 2);
+ }
+
+ #[test]
+ fn calculate_block_size_uses_minimum_size_when_requested_blocks_exceed_total_units() {
+ let mut context = test_context("out.par2", None);
+ context.config.source_block_count = Some(SourceBlockCount::new(20));
+ context.source_files = vec![source_info("a.bin", 8, 0), source_info("b.bin", 8, 1)];
+
+ context.calculate_block_size().unwrap();
+
+ assert_eq!(context.block_size.as_u64(), 4);
+ assert_eq!(context.source_block_count, 4);
+ }
+
+ #[test]
+ fn calculate_block_size_honors_explicit_block_size() {
+ let mut context = test_context("out.par2", None);
+ context.config.block_size = Some(16);
+ context.source_files = vec![source_info("a.bin", 5, 0), source_info("b.bin", 20, 1)];
+
+ context.calculate_block_size().unwrap();
+
+ assert_eq!(context.block_size.as_u64(), 16);
+ assert_eq!(context.source_block_count, 3);
+ }
+
+ #[test]
+ fn calculate_block_size_recomputes_final_count_when_lower_bound_meets_upper_bound() {
+ let mut context = test_context("out.par2", None);
+ context.config.source_block_count = Some(SourceBlockCount::new(3));
+ context.source_files = vec![source_info("a.bin", 4, 0), source_info("b.bin", 12, 1)];
+
+ context.calculate_block_size().unwrap();
+
+ assert_eq!(context.block_size.as_u64(), 8);
+ assert_eq!(context.source_block_count, 3);
+ }
+
+ #[test]
+ fn checked_recovery_block_count_rejects_impossible_requests() {
+ let mut context = test_context("out.par2", None);
+ context.config.first_recovery_block = 65_535;
+
+ assert!(context.checked_recovery_block_count(65_537).is_err());
+ assert!(context.checked_recovery_block_count(1).is_err());
+ }
+
+ #[test]
+ fn calculate_recovery_blocks_for_target_size_never_returns_zero() {
+ let mut context = test_context("out.par2", None);
+ context.block_size = BlockSize::new(4);
+ context.source_block_count = 2;
+ context.source_files = vec![source_info("a.bin", 4, 0)];
+ context.config.recovery_file_scheme = crate::create::RecoveryFileScheme::Uniform;
+
+ assert_eq!(
+ context
+ .calculate_recovery_blocks_for_target_size(1)
+ .unwrap(),
+ 1
+ );
+ }
+
#[test]
fn recovery_slice_packet_writer_streams_borrowed_data() {
let set_id = RecoverySetId::new([0xAB; 16]);
@@ -1056,6 +1732,135 @@ mod tests {
assert_eq!(recovery_packet.recovery_data, expected);
}
+ #[test]
+ fn encode_and_hash_files_slim_mode_profiles_prepass_and_hashes_files() {
+ use crate::checksum::{compute_block_checksums_padded, compute_file_id, compute_md5_only};
+
+ let tmp = tempfile::tempdir().unwrap();
+ let file_a_path = tmp.path().join("a.bin");
+ let file_b_path = tmp.path().join("b.bin");
+ let file_a = b"abcdefghijkl".to_vec();
+ let file_b = b"xyz12".to_vec();
+ std::fs::write(&file_a_path, &file_a).unwrap();
+ std::fs::write(&file_b_path, &file_b).unwrap();
+
+ let source_files = vec![
+ SourceFileInfo::new(file_a_path.clone(), file_a.len() as u64, 0),
+ SourceFileInfo::new(file_b_path.clone(), file_b.len() as u64, 1),
+ ];
+ let block_size = 8u64;
+ let source_block_count: u32 = source_files
+ .iter()
+ .map(|file| file.calculate_block_count(block_size))
+ .sum();
+ let encoder = crate::reed_solomon::RecoveryBlockEncoder::new(
+ block_size as usize,
+ source_block_count as usize,
+ );
+
+ let (_recovery_blocks, hash_states, encode_profile) = encode_and_hash_files(
+ &source_files,
+ block_size,
+ 4,
+ source_block_count,
+ encoder.base_values(),
+ 0,
+ 1,
+ 1,
+ &SilentCreateReporter,
+ true,
+ )
+ .unwrap();
+
+ let profile = encode_profile.expect("profile data should be returned when enabled");
+ assert_eq!(
+ profile.counters.source_hash_bytes_read,
+ (file_a.len() + file_b.len()) as u64
+ );
+ assert!(profile.counters.source_seek_count >= source_files.len() as u64);
+ assert!(profile.source_open_hash_prepass > std::time::Duration::ZERO);
+
+ let expected_blocks = [&file_a[..8], &file_a[8..], &file_b[..]];
+ let expected_offsets = [0u32, 2u32];
+ let expected_full_hashes = [compute_md5_only(&file_a), compute_md5_only(&file_b)];
+ let expected_16k_hashes = expected_full_hashes;
+ let expected_file_ids = [
+ compute_file_id(
+ &expected_16k_hashes[0],
+ file_a.len() as u64,
+ source_files[0].packet_name().as_bytes(),
+ ),
+ compute_file_id(
+ &expected_16k_hashes[1],
+ file_b.len() as u64,
+ source_files[1].packet_name().as_bytes(),
+ ),
+ ];
+
+ assert_eq!(hash_states.len(), 2);
+ assert_eq!(hash_states[0].hash_16k, expected_16k_hashes[0]);
+ assert_eq!(hash_states[0].full_md5, expected_full_hashes[0]);
+ assert_eq!(hash_states[0].file_id, expected_file_ids[0]);
+ assert_eq!(hash_states[0].block_count, 2);
+ assert_eq!(hash_states[0].global_block_offset, expected_offsets[0]);
+ assert_eq!(hash_states[1].hash_16k, expected_16k_hashes[1]);
+ assert_eq!(hash_states[1].full_md5, expected_full_hashes[1]);
+ assert_eq!(hash_states[1].file_id, expected_file_ids[1]);
+ assert_eq!(hash_states[1].block_count, 1);
+ assert_eq!(hash_states[1].global_block_offset, expected_offsets[1]);
+
+ let mut flattened = hash_states
+ .into_iter()
+ .flat_map(|state| state.block_checksums)
+ .collect::>();
+ flattened.sort_by_key(|block| block.global_index);
+ assert_eq!(flattened.len(), expected_blocks.len());
+
+ for (idx, (actual, expected_data)) in flattened.iter().zip(expected_blocks).enumerate() {
+ let (expected_md5, expected_crc32) =
+ compute_block_checksums_padded(expected_data, block_size as usize);
+ assert_eq!(actual.global_index, idx as u32);
+ assert_eq!(actual.hash, expected_md5);
+ assert_eq!(actual.crc32, expected_crc32.as_u32());
+ }
+ }
+
+ #[test]
+ fn create_with_profile_enabled_records_counters_and_outputs() {
+ let _guard = env_lock();
+ std::env::set_var("PAR2RS_CREATE_PROFILE", "1");
+
+ let tmp = tempfile::tempdir().unwrap();
+ let source_path = tmp.path().join("profile.bin");
+ let par2_path = tmp.path().join("profile.par2");
+ std::fs::write(&source_path, b"hello profile").unwrap();
+
+ let mut context = crate::create::CreateContextBuilder::new()
+ .output_name(par2_path.to_str().unwrap())
+ .source_files(vec![source_path.clone()])
+ .block_size(4)
+ .recovery_block_count(1)
+ .quiet(true)
+ .build()
+ .unwrap();
+
+ context.create().unwrap();
+ std::env::remove_var("PAR2RS_CREATE_PROFILE");
+
+ assert!(context.output_files().len() >= 2);
+ let profile = context.profile.as_ref().expect("profile should be enabled");
+ let counters = profile.counters();
+ assert_eq!(counters.source_file_count, 1);
+ assert_eq!(counters.source_bytes, 13);
+ assert_eq!(counters.block_size, 4);
+ assert_eq!(counters.source_block_count, 4);
+ assert_eq!(counters.recovery_block_count, 1);
+ assert!(counters.chunk_size >= 4);
+ assert!(counters.source_recovery_bytes_read >= 13);
+ assert!(counters.recovery_chunk_count >= 1);
+ assert!(counters.selected_backend.is_some());
+ }
+
// --- calculate_chunk_size (method) via CreateContextBuilder ---
#[test]
@@ -1075,4 +1880,35 @@ mod tests {
// 4096 * (2 + small_source_count) << 1GB, so chunk = full block
assert_eq!(ctx.block_size(), 4096);
}
+
+ proptest! {
+ #[test]
+ fn chunk_size_impl_preserves_alignment_and_bounds(
+ block_size in 4usize..(8 * 1024 * 1024),
+ source_block_count in 1usize..512,
+ recovery_block_count in 0usize..64,
+ memory_limit in 4usize..(256 * 1024 * 1024),
+ ) {
+ let block_size = block_size & !3;
+ prop_assume!(block_size >= 4);
+
+ let result = calculate_chunk_size_impl(
+ block_size,
+ source_block_count,
+ recovery_block_count,
+ memory_limit,
+ );
+ let max_allowed = block_size.min(MAX_CREATE_CHUNK_SIZE);
+ let block_overhead = 2 + (source_block_count + 1).min(24);
+ let full_block_memory = block_size * (recovery_block_count + block_overhead);
+
+ prop_assert!(result >= 4);
+ prop_assert!(result <= max_allowed);
+ prop_assert_eq!(result % 4, 0);
+
+ if full_block_memory <= memory_limit {
+ prop_assert_eq!(result, max_allowed);
+ }
+ }
+ }
}
diff --git a/src/create/error.rs b/src/create/error.rs
index c86d8f9d..c8cebb86 100644
--- a/src/create/error.rs
+++ b/src/create/error.rs
@@ -66,6 +66,10 @@ pub enum CreateError {
#[error("Failed to generate packet: {0}")]
PacketGenerationError(String),
+ /// XOR-JIT packed checksum validation failed
+ #[error("XOR-JIT packed checksum validation failed")]
+ XorJitChecksumValidationFailed,
+
/// I/O error during creation
#[error("I/O error: {0}")]
IoError(#[from] std::io::Error),
@@ -93,6 +97,7 @@ mod tests {
CreateError::EmptySourceFiles,
CreateError::ReedSolomonError("matrix failed".to_string()),
CreateError::PacketGenerationError("packet failed".to_string()),
+ CreateError::XorJitChecksumValidationFailed,
CreateError::Other("plain error".to_string()),
];
diff --git a/src/create/mod.rs b/src/create/mod.rs
index 78f1c963..d982eb49 100644
--- a/src/create/mod.rs
+++ b/src/create/mod.rs
@@ -36,6 +36,7 @@ pub mod error;
pub mod error_helpers;
pub mod file_naming;
pub mod packet_generator;
+pub(crate) mod profile;
pub mod progress;
pub mod source_file;
pub mod types;
@@ -80,3 +81,31 @@ pub fn create_files(
context.create()?;
Ok(context)
}
+
+#[cfg(test)]
+mod tests {
+ use super::{create_files, SilentCreateReporter};
+
+ #[test]
+ fn create_files_wrapper_creates_output_and_returns_context() {
+ let tmp = tempfile::tempdir().unwrap();
+ let source = tmp.path().join("data.bin");
+ let output = tmp.path().join("archive.par2");
+ std::fs::write(&source, b"hello world").unwrap();
+
+ let context = create_files(
+ output.to_str().unwrap(),
+ vec![source],
+ 10,
+ Box::new(SilentCreateReporter),
+ )
+ .unwrap();
+
+ assert!(!context.output_files().is_empty());
+ assert!(context
+ .output_files()
+ .iter()
+ .any(|path| path.ends_with(".par2")));
+ assert!(context.block_size() >= 4);
+ }
+}
diff --git a/src/create/profile.rs b/src/create/profile.rs
new file mode 100644
index 00000000..15311bae
--- /dev/null
+++ b/src/create/profile.rs
@@ -0,0 +1,255 @@
+use std::time::{Duration, Instant};
+
+const PROFILE_ENV: &str = "PAR2RS_CREATE_PROFILE";
+
+#[derive(Debug, Clone, Copy)]
+pub(crate) enum CreateProfilePhase {
+ SourceScanMetadata,
+ SourceOpenHashPrepass,
+ RecoveryChunkProcessing,
+ CriticalPacketSerialization,
+ RecoveryPacketSerialization,
+ OutputFileWrites,
+}
+
+#[derive(Debug, Default, Clone)]
+pub(crate) struct CreateProfileCounters {
+ pub source_file_count: usize,
+ pub source_bytes: u64,
+ pub block_size: u64,
+ pub source_block_count: u32,
+ pub recovery_block_count: u32,
+ pub chunk_size: usize,
+ pub defer_hash_computation: bool,
+ pub source_hash_bytes_read: u64,
+ pub source_recovery_bytes_read: u64,
+ pub source_seek_count: u64,
+ pub recovery_chunk_count: u64,
+ pub selected_backend: Option,
+}
+
+#[derive(Debug)]
+pub(crate) struct CreateProfile {
+ started_at: Instant,
+ source_scan_metadata: Duration,
+ source_open_hash_prepass: Duration,
+ recovery_chunk_processing: Duration,
+ critical_packet_serialization: Duration,
+ recovery_packet_serialization: Duration,
+ output_file_writes: Duration,
+ counters: CreateProfileCounters,
+}
+
+impl CreateProfile {
+ pub(crate) fn from_env() -> Option {
+ let enabled = std::env::var_os(PROFILE_ENV).is_some_and(|value| {
+ let value = value.to_string_lossy();
+ !matches!(value.as_ref(), "" | "0" | "false" | "FALSE" | "off" | "OFF")
+ });
+ enabled.then(Self::new)
+ }
+
+ fn new() -> Self {
+ Self {
+ started_at: Instant::now(),
+ source_scan_metadata: Duration::ZERO,
+ source_open_hash_prepass: Duration::ZERO,
+ recovery_chunk_processing: Duration::ZERO,
+ critical_packet_serialization: Duration::ZERO,
+ recovery_packet_serialization: Duration::ZERO,
+ output_file_writes: Duration::ZERO,
+ counters: CreateProfileCounters::default(),
+ }
+ }
+
+ pub(crate) fn add_duration(&mut self, phase: CreateProfilePhase, duration: Duration) {
+ match phase {
+ CreateProfilePhase::SourceScanMetadata => self.source_scan_metadata += duration,
+ CreateProfilePhase::SourceOpenHashPrepass => self.source_open_hash_prepass += duration,
+ CreateProfilePhase::RecoveryChunkProcessing => {
+ self.recovery_chunk_processing += duration;
+ }
+ CreateProfilePhase::CriticalPacketSerialization => {
+ self.critical_packet_serialization += duration;
+ }
+ CreateProfilePhase::RecoveryPacketSerialization => {
+ self.recovery_packet_serialization += duration;
+ }
+ CreateProfilePhase::OutputFileWrites => self.output_file_writes += duration,
+ }
+ }
+
+ pub(crate) fn counters_mut(&mut self) -> &mut CreateProfileCounters {
+ &mut self.counters
+ }
+
+ #[cfg(test)]
+ pub(crate) fn counters(&self) -> &CreateProfileCounters {
+ &self.counters
+ }
+
+ pub(crate) fn emit(&self) {
+ eprintln!("PAR2RS_CREATE_PROFILE_BEGIN");
+ eprintln!("phase,seconds");
+ eprintln!(
+ "source_scan_metadata,{:.9}",
+ self.source_scan_metadata.as_secs_f64()
+ );
+ eprintln!(
+ "source_open_hash_prepass,{:.9}",
+ self.source_open_hash_prepass.as_secs_f64()
+ );
+ eprintln!(
+ "recovery_chunk_processing,{:.9}",
+ self.recovery_chunk_processing.as_secs_f64()
+ );
+ eprintln!(
+ "critical_packet_serialization,{:.9}",
+ self.critical_packet_serialization.as_secs_f64()
+ );
+ eprintln!(
+ "recovery_packet_serialization,{:.9}",
+ self.recovery_packet_serialization.as_secs_f64()
+ );
+ eprintln!(
+ "output_file_writes,{:.9}",
+ self.output_file_writes.as_secs_f64()
+ );
+ eprintln!(
+ "total_create_process,{:.9}",
+ self.started_at.elapsed().as_secs_f64()
+ );
+ eprintln!("counter,value");
+ eprintln!("source_file_count,{}", self.counters.source_file_count);
+ eprintln!("source_bytes,{}", self.counters.source_bytes);
+ eprintln!("block_size,{}", self.counters.block_size);
+ eprintln!("source_block_count,{}", self.counters.source_block_count);
+ eprintln!(
+ "recovery_block_count,{}",
+ self.counters.recovery_block_count
+ );
+ eprintln!("chunk_size,{}", self.counters.chunk_size);
+ eprintln!(
+ "defer_hash_computation,{}",
+ self.counters.defer_hash_computation
+ );
+ eprintln!(
+ "source_hash_bytes_read,{}",
+ self.counters.source_hash_bytes_read
+ );
+ eprintln!(
+ "source_recovery_bytes_read,{}",
+ self.counters.source_recovery_bytes_read
+ );
+ eprintln!("source_seek_count,{}", self.counters.source_seek_count);
+ eprintln!(
+ "recovery_chunk_count,{}",
+ self.counters.recovery_chunk_count
+ );
+ eprintln!(
+ "selected_create_backend,{}",
+ self.counters
+ .selected_backend
+ .as_deref()
+ .unwrap_or("unknown")
+ );
+ eprintln!("PAR2RS_CREATE_PROFILE_END");
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::sync::{Mutex, OnceLock};
+
+ fn env_lock() -> std::sync::MutexGuard<'static, ()> {
+ static LOCK: OnceLock> = OnceLock::new();
+ LOCK.get_or_init(|| Mutex::new(())).lock().unwrap()
+ }
+
+ #[test]
+ fn from_env_respects_falsey_and_truthy_values() {
+ let _guard = env_lock();
+ std::env::remove_var(PROFILE_ENV);
+ assert!(CreateProfile::from_env().is_none());
+
+ for value in ["", "0", "false", "FALSE", "off", "OFF"] {
+ std::env::set_var(PROFILE_ENV, value);
+ assert!(
+ CreateProfile::from_env().is_none(),
+ "value {value} should disable profiling"
+ );
+ }
+
+ for value in ["1", "true", "on", "yes"] {
+ std::env::set_var(PROFILE_ENV, value);
+ assert!(
+ CreateProfile::from_env().is_some(),
+ "value {value} should enable profiling"
+ );
+ }
+
+ std::env::remove_var(PROFILE_ENV);
+ }
+
+ #[test]
+ fn profile_accumulates_phase_durations_and_counters() {
+ let mut profile = CreateProfile::new();
+
+ profile.add_duration(
+ CreateProfilePhase::SourceScanMetadata,
+ Duration::from_millis(5),
+ );
+ profile.add_duration(
+ CreateProfilePhase::SourceOpenHashPrepass,
+ Duration::from_millis(7),
+ );
+ profile.add_duration(
+ CreateProfilePhase::RecoveryChunkProcessing,
+ Duration::from_millis(11),
+ );
+ profile.add_duration(
+ CreateProfilePhase::CriticalPacketSerialization,
+ Duration::from_millis(13),
+ );
+ profile.add_duration(
+ CreateProfilePhase::RecoveryPacketSerialization,
+ Duration::from_millis(17),
+ );
+ profile.add_duration(
+ CreateProfilePhase::OutputFileWrites,
+ Duration::from_millis(19),
+ );
+
+ let counters = profile.counters_mut();
+ counters.source_file_count = 2;
+ counters.source_bytes = 123;
+ counters.block_size = 4;
+ counters.source_block_count = 6;
+ counters.recovery_block_count = 1;
+ counters.chunk_size = 4;
+ counters.defer_hash_computation = true;
+ counters.source_hash_bytes_read = 100;
+ counters.source_recovery_bytes_read = 80;
+ counters.source_seek_count = 3;
+ counters.recovery_chunk_count = 2;
+ counters.selected_backend = Some("scalar".to_string());
+
+ assert_eq!(profile.source_scan_metadata, Duration::from_millis(5));
+ assert_eq!(profile.source_open_hash_prepass, Duration::from_millis(7));
+ assert_eq!(profile.recovery_chunk_processing, Duration::from_millis(11));
+ assert_eq!(
+ profile.critical_packet_serialization,
+ Duration::from_millis(13)
+ );
+ assert_eq!(
+ profile.recovery_packet_serialization,
+ Duration::from_millis(17)
+ );
+ assert_eq!(profile.output_file_writes, Duration::from_millis(19));
+ assert_eq!(profile.counters.source_file_count, 2);
+ assert_eq!(profile.counters.selected_backend.as_deref(), Some("scalar"));
+
+ profile.emit();
+ }
+}
diff --git a/src/create/progress.rs b/src/create/progress.rs
index a48dfebc..fc266400 100644
--- a/src/create/progress.rs
+++ b/src/create/progress.rs
@@ -117,7 +117,8 @@ impl CreateReporter for SilentCreateReporter {
#[cfg(test)]
mod tests {
- use super::percent_complete;
+ use super::{percent_complete, ConsoleCreateReporter, CreateReporter, SilentCreateReporter};
+ use proptest::prelude::*;
#[test]
fn percent_complete_treats_empty_work_as_complete() {
@@ -128,4 +129,41 @@ mod tests {
fn percent_complete_clamps_overreported_progress() {
assert_eq!(percent_complete(12, 10), 100);
}
+
+ #[test]
+ fn reporters_are_callable_in_quiet_and_verbose_modes() {
+ for quiet in [true, false] {
+ let reporter = ConsoleCreateReporter::new(quiet);
+ reporter.report_scanning_files(1, 3, "input.bin");
+ reporter.report_file_hashing("input.bin", 5, 10);
+ reporter.report_block_checksums(2, 4);
+ reporter.report_recovery_generation(3, 4);
+ reporter.report_writing_file("output.par2");
+ reporter.report_complete(&["output.par2".to_string()]);
+ reporter.report_error("boom");
+ }
+ }
+
+ #[test]
+ fn silent_reporter_accepts_all_calls() {
+ let reporter = SilentCreateReporter;
+ reporter.report_scanning_files(1, 1, "input.bin");
+ reporter.report_file_hashing("input.bin", 10, 10);
+ reporter.report_block_checksums(1, 1);
+ reporter.report_recovery_generation(1, 1);
+ reporter.report_writing_file("output.par2");
+ reporter.report_complete(&[]);
+ reporter.report_error("boom");
+ }
+
+ proptest! {
+ #[test]
+ fn percent_complete_stays_bounded(completed in 0u64..10_000, total in 1u64..10_000) {
+ let percent = percent_complete(completed, total);
+ let expected = ((completed.min(total) * 100) / total) as u32;
+
+ prop_assert_eq!(percent, expected);
+ prop_assert!(percent <= 100);
+ }
+ }
}
diff --git a/src/domain.rs b/src/domain.rs
index 153dfbb6..cba18f71 100644
--- a/src/domain.rs
+++ b/src/domain.rs
@@ -517,3 +517,118 @@ impl std::fmt::Display for FileSize {
write!(f, "{}", self.0)
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use proptest::prelude::*;
+
+ #[test]
+ fn hash_newtypes_expose_their_underlying_bytes() {
+ let bytes = [0x5Au8; 16];
+
+ let file_id = FileId::new(bytes);
+ let recovery_set_id = RecoverySetId::from(bytes);
+ let md5 = Md5Hash::from(bytes);
+
+ assert_eq!(file_id.as_bytes(), &bytes);
+ assert_eq!(file_id.as_ref(), &bytes);
+ assert_eq!(file_id, bytes);
+ assert_eq!(bytes, file_id);
+
+ assert_eq!(recovery_set_id.as_bytes(), &bytes);
+ assert_eq!(recovery_set_id.as_ref(), &bytes);
+ assert_eq!(recovery_set_id, bytes);
+ assert_eq!(bytes, recovery_set_id);
+
+ assert_eq!(md5.as_bytes(), &bytes);
+ assert_eq!(md5.as_ref(), &bytes);
+ assert_eq!(md5.len(), 16);
+ assert_eq!(md5, bytes);
+ assert_eq!(bytes, md5);
+ }
+
+ #[test]
+ fn index_and_count_newtypes_support_expected_arithmetic() {
+ let global = GlobalSliceIndex::new(11);
+ let local = LocalSliceIndex::new(7);
+ let mut total_blocks = BlockCount::new(5);
+
+ assert_eq!((global + 4).as_usize(), 15);
+ assert_eq!(global - GlobalSliceIndex::new(3), 8);
+ assert_eq!(local.to_global(GlobalSliceIndex::new(20)).as_usize(), 27);
+ assert_eq!(local.as_usize(), 7);
+ assert_eq!(format!("{global}"), "11");
+ assert_eq!(format!("{local}"), "7");
+
+ assert_eq!(SourceBlockCount::new(9).as_u32(), 9);
+ assert_eq!(SourceBlockCount::new(9).as_usize(), 9);
+ assert_eq!(SourceBlockCount::new(9).as_u64(), 9);
+
+ assert_eq!((BlockCount::new(2) + BlockCount::new(3)).as_u32(), 5);
+ assert_eq!((BlockCount::new(9) - BlockCount::new(4)).as_u32(), 5);
+ assert_eq!(BlockCount::new(9) - 4usize, 5);
+ total_blocks += BlockCount::new(6);
+ assert_eq!(total_blocks.as_usize(), 11);
+ assert_eq!(
+ [BlockCount::new(1), BlockCount::new(2), BlockCount::new(3)]
+ .into_iter()
+ .sum::()
+ .as_u32(),
+ 6
+ );
+ assert_eq!(format!("{total_blocks}"), "11");
+ }
+
+ #[test]
+ fn numeric_wrapper_types_preserve_conversion_and_formatting_behavior() {
+ let crc = Crc32Value::new(0x1234_ABCD);
+ let block_size = BlockSize::new(4096);
+ let chunk_size = ChunkSize::new(2048);
+ let file_size = FileSize::new(10_000);
+
+ assert_eq!(crc.as_u32(), 0x1234_ABCD);
+ assert_eq!(crc.to_le_bytes(), 0x1234_ABCDu32.to_le_bytes());
+ assert_eq!(crc, 0x1234_ABCD);
+ assert_eq!(0x1234_ABCD, crc);
+ assert_eq!(format!("{crc}"), "1234abcd");
+
+ assert_eq!(block_size.as_u64(), 4096);
+ assert_eq!(block_size.as_usize(), 4096);
+ assert_eq!(8193u64 % block_size, 1);
+ assert_eq!(block_size - 96, 4000);
+ assert_eq!(block_size, 4096);
+ assert!(block_size > 1024);
+ assert_eq!(format!("{block_size}"), "4096");
+
+ assert_eq!(chunk_size.as_usize(), 2048);
+ assert_eq!(chunk_size, 2048);
+ assert!(chunk_size > 1024);
+ assert_eq!(format!("{chunk_size}"), "2048");
+
+ assert_eq!(file_size.as_u64(), 10_000);
+ assert_eq!(file_size.as_usize(), 10_000);
+ assert_eq!(file_size % block_size, 1808);
+ assert_eq!(
+ [FileSize::new(3), FileSize::new(4), FileSize::new(5)]
+ .into_iter()
+ .sum::()
+ .as_u64(),
+ 12
+ );
+ assert_eq!(file_size, 10_000);
+ assert!(file_size > 9_000);
+ assert_eq!(format!("{file_size}"), "10000");
+ }
+
+ proptest! {
+ #[test]
+ fn local_indices_round_trip_through_global_offsets(local in 0usize..1_000_000, offset in 0usize..1_000_000) {
+ let local_index = LocalSliceIndex::new(local);
+ let global_index = local_index.to_global(GlobalSliceIndex::new(offset));
+
+ prop_assert_eq!(global_index.as_usize(), local + offset);
+ prop_assert_eq!(global_index - GlobalSliceIndex::new(offset), local);
+ }
+ }
+}
diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs
new file mode 100644
index 00000000..49cc227e
--- /dev/null
+++ b/src/ffi/mod.rs
@@ -0,0 +1,206 @@
+//! Rust FFI wrappers around the embedded ParPar hasher sources.
+//!
+//! This module is only built on x86_64 when the `parpar-compare` feature is enabled.
+
+use crate::domain::Md5Hash;
+use std::ffi::c_void;
+
+#[repr(u32)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum HasherInputMethod {
+ Scalar = 0,
+ Simd = 1,
+ Crc = 2,
+ SimdCrc = 3,
+ Bmi1 = 4,
+ Avx512 = 5,
+}
+
+unsafe extern "C" {
+ fn parpar_md5single_new() -> *mut c_void;
+ fn parpar_md5single_update(ctx: *mut c_void, data: *const u8, len: usize);
+ fn parpar_md5single_end(ctx: *mut c_void, out: *mut u8);
+ fn parpar_md5single_free(ctx: *mut c_void);
+ fn parpar_md5single_hash(data: *const u8, len: usize, out: *mut u8);
+
+ fn parpar_hasher_input_new(method: u32) -> *mut c_void;
+ fn parpar_hasher_input_update(ctx: *mut c_void, data: *const u8, len: usize);
+ fn parpar_hasher_input_end(ctx: *mut c_void, out: *mut u8);
+ fn parpar_hasher_input_reset(ctx: *mut c_void);
+ fn parpar_hasher_input_free(ctx: *mut c_void);
+ fn parpar_hasher_input_hash(method: u32, data: *const u8, len: usize, out: *mut u8) -> bool;
+ fn parpar_hasher_input_is_available(method: u32) -> bool;
+
+ fn parpar_crc32_compute(data: *const u8, len: usize) -> u32;
+}
+
+impl HasherInputMethod {
+ pub fn is_available(self) -> bool {
+ unsafe { parpar_hasher_input_is_available(self as u32) }
+ }
+}
+
+pub mod md5 {
+ use super::*;
+
+ pub struct ParParMd5 {
+ ctx: *mut c_void,
+ }
+
+ impl ParParMd5 {
+ pub fn new() -> Option {
+ let ctx = unsafe { parpar_md5single_new() };
+ (!ctx.is_null()).then_some(Self { ctx })
+ }
+
+ pub fn update(&mut self, data: &[u8]) {
+ unsafe { parpar_md5single_update(self.ctx, data.as_ptr(), data.len()) }
+ }
+
+ pub fn finalize(mut self) -> Md5Hash {
+ let mut digest = [0u8; 16];
+ let ctx = self.ctx;
+ self.ctx = std::ptr::null_mut();
+ unsafe {
+ parpar_md5single_end(ctx, digest.as_mut_ptr());
+ parpar_md5single_free(ctx);
+ }
+ Md5Hash::new(digest)
+ }
+ }
+
+ impl Drop for ParParMd5 {
+ fn drop(&mut self) {
+ if !self.ctx.is_null() {
+ unsafe { parpar_md5single_free(self.ctx) };
+ self.ctx = std::ptr::null_mut();
+ }
+ }
+ }
+
+ pub fn md5_hash(data: &[u8]) -> Md5Hash {
+ let mut digest = [0u8; 16];
+ unsafe {
+ parpar_md5single_hash(data.as_ptr(), data.len(), digest.as_mut_ptr());
+ }
+ Md5Hash::new(digest)
+ }
+}
+
+pub mod hasher_input {
+ use super::*;
+
+ pub struct ParParHasherInput {
+ ctx: *mut c_void,
+ }
+
+ impl ParParHasherInput {
+ pub fn new(method: HasherInputMethod) -> Option {
+ let ctx = unsafe { parpar_hasher_input_new(method as u32) };
+ (!ctx.is_null()).then_some(Self { ctx })
+ }
+
+ pub fn update(&mut self, data: &[u8]) {
+ unsafe { parpar_hasher_input_update(self.ctx, data.as_ptr(), data.len()) }
+ }
+
+ pub fn reset(&mut self) {
+ unsafe { parpar_hasher_input_reset(self.ctx) }
+ }
+
+ pub fn finalize(mut self) -> Md5Hash {
+ let mut digest = [0u8; 16];
+ let ctx = self.ctx;
+ self.ctx = std::ptr::null_mut();
+ unsafe {
+ parpar_hasher_input_end(ctx, digest.as_mut_ptr());
+ parpar_hasher_input_free(ctx);
+ }
+ Md5Hash::new(digest)
+ }
+
+ pub fn hash(method: HasherInputMethod, data: &[u8]) -> Option {
+ let mut digest = [0u8; 16];
+ let ok = unsafe {
+ parpar_hasher_input_hash(
+ method as u32,
+ data.as_ptr(),
+ data.len(),
+ digest.as_mut_ptr(),
+ )
+ };
+ ok.then(|| Md5Hash::new(digest))
+ }
+ }
+
+ impl Drop for ParParHasherInput {
+ fn drop(&mut self) {
+ if !self.ctx.is_null() {
+ unsafe { parpar_hasher_input_free(self.ctx) };
+ self.ctx = std::ptr::null_mut();
+ }
+ }
+ }
+}
+
+pub mod crc32 {
+ use super::*;
+
+ pub fn crc32_compute(data: &[u8]) -> u32 {
+ unsafe { parpar_crc32_compute(data.as_ptr(), data.len()) }
+ }
+}
+
+pub use crc32::crc32_compute;
+pub use hasher_input::ParParHasherInput;
+pub use md5::ParParMd5;
+
+#[cfg(all(
+ test,
+ feature = "parpar-compare",
+ target_arch = "x86_64",
+ parpar_compare_embedded
+))]
+mod tests {
+ use super::{hasher_input::ParParHasherInput, HasherInputMethod};
+
+ #[test]
+ fn one_shot_hash_returns_none_for_unavailable_method() {
+ let unavailable = [
+ HasherInputMethod::Avx512,
+ HasherInputMethod::Bmi1,
+ HasherInputMethod::SimdCrc,
+ HasherInputMethod::Crc,
+ HasherInputMethod::Simd,
+ HasherInputMethod::Scalar,
+ ]
+ .into_iter()
+ .find(|method| !method.is_available());
+
+ if let Some(method) = unavailable {
+ assert!(ParParHasherInput::hash(method, b"ffi smoke").is_none());
+ }
+ }
+
+ #[test]
+ fn one_shot_hash_matches_streaming_hash_for_available_method() {
+ let method = [
+ HasherInputMethod::Avx512,
+ HasherInputMethod::Bmi1,
+ HasherInputMethod::SimdCrc,
+ HasherInputMethod::Crc,
+ HasherInputMethod::Simd,
+ HasherInputMethod::Scalar,
+ ]
+ .into_iter()
+ .find(|method| method.is_available())
+ .expect("at least one ParPar method should be available");
+
+ let data = b"ffi smoke";
+ let one_shot = ParParHasherInput::hash(method, data).expect("one-shot hash");
+ let mut streaming = ParParHasherInput::new(method).expect("streaming hash");
+ streaming.update(data);
+
+ assert_eq!(one_shot, streaming.finalize());
+ }
+}
diff --git a/src/ffi/wrapper.cpp b/src/ffi/wrapper.cpp
new file mode 100644
index 00000000..b065881e
--- /dev/null
+++ b/src/ffi/wrapper.cpp
@@ -0,0 +1,164 @@
+#include
+#include
+#include
+#include
+
+using std::max_align_t;
+using std::size_t;
+using std::uint8_t;
+using std::uint16_t;
+using std::uint32_t;
+using std::uint64_t;
+using std::uintptr_t;
+
+#include "par2cmdline-turbo/parpar/hasher/hasher_input.h"
+#include "par2cmdline-turbo/parpar/hasher/hasher_md5crc.h"
+
+namespace {
+enum ParParHasherInputMethod : unsigned {
+ PARPAR_HASHER_INPUT_SCALAR = 0,
+ PARPAR_HASHER_INPUT_SIMD = 1,
+ PARPAR_HASHER_INPUT_CRC = 2,
+ PARPAR_HASHER_INPUT_SIMD_CRC = 3,
+ PARPAR_HASHER_INPUT_BMI1 = 4,
+ PARPAR_HASHER_INPUT_AVX512 = 5,
+};
+
+static void configure_md5single() {
+ static std::once_flag configured;
+ std::call_once(configured, [] {
+ if (!set_hasherMD5CRC(MD5CRCMETH_BMI1)) {
+ (void)set_hasherMD5CRC(MD5CRCMETH_SCALAR);
+ }
+ });
+}
+
+static IHasherInput* create_input(ParParHasherInputMethod method) {
+ switch (method) {
+ case PARPAR_HASHER_INPUT_SCALAR:
+ return HasherInput_Scalar::create();
+ case PARPAR_HASHER_INPUT_SIMD:
+ return HasherInput_SSE::create();
+ case PARPAR_HASHER_INPUT_CRC:
+ return HasherInput_ClMulScalar::create();
+ case PARPAR_HASHER_INPUT_SIMD_CRC:
+ return HasherInput_ClMulSSE::create();
+ case PARPAR_HASHER_INPUT_BMI1:
+ return HasherInput_BMI1::create();
+ case PARPAR_HASHER_INPUT_AVX512:
+ return HasherInput_AVX512::create();
+ default:
+ return nullptr;
+ }
+}
+} // namespace
+
+extern "C" {
+
+void* parpar_md5single_new(void) {
+ configure_md5single();
+ return new (std::nothrow) MD5Single();
+}
+
+void parpar_md5single_update(void* ctx, const unsigned char* data, size_t len) {
+ if (!ctx || (!data && len != 0)) {
+ return;
+ }
+ static_cast(ctx)->update(data, len);
+}
+
+void parpar_md5single_end(void* ctx, unsigned char* out) {
+ if (!ctx || !out) {
+ return;
+ }
+ static_cast(ctx)->end(out);
+}
+
+void parpar_md5single_free(void* ctx) {
+ delete static_cast(ctx);
+}
+
+void parpar_md5single_hash(const unsigned char* data, size_t len, unsigned char* out) {
+ if (!out || (!data && len != 0)) {
+ return;
+ }
+ configure_md5single();
+ MD5Single ctx;
+ ctx.update(data, len);
+ ctx.end(out);
+}
+
+void* parpar_hasher_input_new(unsigned method) {
+ return create_input(static_cast(method));
+}
+
+void parpar_hasher_input_update(void* ctx, const unsigned char* data, size_t len) {
+ if (!ctx || (!data && len != 0)) {
+ return;
+ }
+ static_cast(ctx)->update(data, len);
+}
+
+void parpar_hasher_input_end(void* ctx, unsigned char* out) {
+ if (!ctx || !out) {
+ return;
+ }
+ static_cast(ctx)->end(out);
+}
+
+void parpar_hasher_input_reset(void* ctx) {
+ if (!ctx) {
+ return;
+ }
+ static_cast(ctx)->reset();
+}
+
+bool parpar_hasher_input_is_available(unsigned method) {
+ switch (static_cast(method)) {
+ case PARPAR_HASHER_INPUT_SCALAR:
+ return HasherInput_Scalar::isAvailable;
+ case PARPAR_HASHER_INPUT_SIMD:
+ return HasherInput_SSE::isAvailable;
+ case PARPAR_HASHER_INPUT_CRC:
+ return HasherInput_ClMulScalar::isAvailable;
+ case PARPAR_HASHER_INPUT_SIMD_CRC:
+ return HasherInput_ClMulSSE::isAvailable;
+ case PARPAR_HASHER_INPUT_BMI1:
+ return HasherInput_BMI1::isAvailable;
+ case PARPAR_HASHER_INPUT_AVX512:
+ return HasherInput_AVX512::isAvailable;
+ default:
+ return false;
+ }
+}
+
+void parpar_hasher_input_free(void* ctx) {
+ if (!ctx) {
+ return;
+ }
+ static_cast(ctx)->destroy();
+}
+
+bool parpar_hasher_input_hash(unsigned method, const unsigned char* data, size_t len, unsigned char* out) {
+ if (!out || (!data && len != 0)) {
+ return false;
+ }
+
+ IHasherInput* ctx = create_input(static_cast(method));
+ if (!ctx) {
+ return false;
+ }
+ ctx->update(data, len);
+ ctx->end(out);
+ ctx->destroy();
+ return true;
+}
+
+uint32_t parpar_crc32_compute(const unsigned char* data, size_t len) {
+ if (!data && len != 0) {
+ return 0;
+ }
+ return CRC32_Calc_ClMul(data, len);
+}
+
+}
diff --git a/src/lib.rs b/src/lib.rs
index c94ff8d7..69c1cb70 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -27,11 +27,20 @@ pub mod verify;
pub mod checksum;
pub mod domain;
pub mod packets;
+pub mod parpar_hasher;
pub mod reporters;
// Reed-Solomon is internal implementation detail, but exposed for advanced use
pub mod reed_solomon;
+// FFI bindings to ParPar C++ for comparison/validation (benchmarks only)
+#[cfg(all(
+ feature = "parpar-compare",
+ target_arch = "x86_64",
+ parpar_compare_embedded
+))]
+pub mod ffi;
+
// Re-export commonly used types for convenience (used internally and by binaries)
pub use args::parse_args;
pub use packets::{
diff --git a/src/parpar_hasher/ATTRIBUTION.md b/src/parpar_hasher/ATTRIBUTION.md
new file mode 100644
index 00000000..4e7d1ea8
--- /dev/null
+++ b/src/parpar_hasher/ATTRIBUTION.md
@@ -0,0 +1,102 @@
+# Attribution: par2rs `parpar_hasher` module
+
+The Rust source files in this directory are line-for-line ports of C++
+headers from the **par2cmdline-turbo** / **ParPar** projects, both
+distributed under **GNU GPL v2 or later**. par2rs is also distributed
+under **GNU GPL v2 or later**, so the licenses are compatible.
+
+## Upstream
+
+* Project: par2cmdline-turbo
+ * Repo:
+ * License: GPL-2.0-or-later (`COPYING` at the repo root)
+* Project: ParPar (the upstream of par2cmdline-turbo's `parpar/` subtree)
+ * Repo:
+ * License: GPL-2.0-or-later
+
+## File-by-file mapping
+
+| par2rs Rust source | Upstream C/C++ source | Notes |
+| ----------------------------- | ----------------------------------------- | ---------------------------------------------------- |
+| `md5x2.rs` | `parpar/hasher/md5x2-base.h` macro contract | `Md5x2` trait — Rust analogue of upstream's textual `_FNMD5x2(f)` macro substitution. |
+| `md5x2_scalar.rs` | `parpar/hasher/md5x2-x86-asm.h` (non-BMI1 branches) | Two-lane scalar (GPR) MD5 via `asm!`. Always available on x86_64. |
+| `md5x2_bmi1.rs` | `parpar/hasher/md5x2-x86-asm.h` (`_MD5_USE_BMI1_` branches), gated through `parpar/hasher/hasher_bmi1.cpp` | Same shape as scalar; G/I/I_LAST round bodies use `andnl` and the `K-1`/`subl` identity to save a NOT (and an OR in I). Requires runtime BMI1 check. |
+| `md5x2_sse2.rs` | `parpar/hasher/md5x2-sse.h` + `parpar/hasher/md5-base.h` | Two-lane SSE2 MD5: each lane in xmm lanes 0/2 of `[__m128i; 4]` state, rotate via `srli_epi64(shuffle<2200>, 32-r)` trick. |
+| `crc_clmul.rs` | `parpar/hasher/crc_clmul.h` + `parpar/hasher/tables.cpp` (`pshufb_shf_table`) | x86_64 PCLMULQDQ CRC32 (4-fold). |
+| `hasher_input.rs` | `parpar/hasher/hasher_input_base.h`, | Fused 64-byte driver (block-MD5 + file-MD5 + CRC32). Generic over `Md5x2` backend. |
+| | `parpar/hasher/hasher_input.cpp`, | |
+| | `parpar/hasher/hasher_input_impl.h` | |
+| `md5x2_avx512.rs` *(future)* | `parpar/hasher/md5-avx512-asm.h` | AVX-512 ternary-logic-accelerated path. |
+| `md5x2_neon.rs` *(future)* | `parpar/hasher/md5x2-neon-asm.h`, | aarch64 NEON two-lane MD5. |
+| | `parpar/hasher/md5-arm64-asm.h` | |
+| `crc_arm.rs` *(future)* | `parpar/hasher/crc_arm.h` | aarch64 PMULL CRC32 fold (deferred — `crc32fast` covers ARM via the same fallback). |
+
+## CRC32 backend decision (T2.b, then revised)
+
+Initial decision (T2.b, kept after `benches/crc_compare.rs` and the
+first round of `benches/md5x2_crc_fused.rs`): use `crc32fast` rather
+than porting `parpar/hasher/crc_clmul.h`. A microbench compared
+`crc32fast` against the alternative
+[`crc-fast`](https://crates.io/crates/crc-fast) crate (which folds
+8-at-a-time vs. parpar's 4-at-a-time) on the access patterns par2rs
+actually uses:
+
+* 64 B one-shot — `crc32fast` ~50% faster.
+* 16 KiB streamed in 64 B chunks — `crc32fast` ~17% faster.
+* 4 MiB streamed in 64 B chunks — `crc32fast` ~13% faster.
+* 4–64 MiB single-shot — `crc-fast` ~6–8% faster (irrelevant: not our
+ access pattern, since the fused HasherInput driver feeds 64 B at a time
+ to interleave with MD5x2).
+
+Inside the actual `HasherInput` access pattern (MD5x2 + CRC at 64 B
+granularity over `(file-MD5, block-MD5, block-CRC32)`), the two CRC
+backends are within noise of each other (816 vs 808 MiB/s at 16 KiB;
+819 vs 823 MiB/s at 4 MiB) — MD5x2's GPR work hides the CRC backend
+cost difference between the two crates entirely.
+
+### Revisit: per-call SIMD setup overhead (T2.b → T2.b')
+
+Both `crc32fast::Hasher::update` and `crc_fast::Digest::update` carry
+a per-call alignment / SIMD-setup prologue. When the create path's
+fused driver calls them once per 64 B (65 536 calls per 4 MiB block)
+that prologue dominates: variant 7 in `benches/md5x2_crc_fused.rs`
+(MD5x2 + this CLMul port fused at 64 B) reaches 925 / 933 MiB/s vs
+817 / 815 for variant 3 (`crc32fast` per 64 B), and the lower-bound
+"MD5x2 only" variant runs at 965 MiB/s — so the remaining 32–40 MiB/s
+gap is the actual CLMul folding cost rather than per-call overhead.
+
+Decision: port `crc_clmul.h` (this commit, `crc_clmul.rs`) so the
+fused driver can call PCLMULQDQ CRC32 inline at 64 B granularity with
+no setup-prologue cost. `crc32fast` is retained as the bulk-CRC
+backend everywhere outside the fused driver (verify path, partial
+last-block tail under non-block-order create) where the access
+pattern *does* match its strong regime.
+
+`crc-fast` is retained as a `dev-dep` purely so `benches/crc_compare.rs`
+and `benches/md5x2_crc_fused.rs` can be re-run by future contributors
+who want to revisit either decision.
+
+## Outstanding follow-up (T2.c)
+
+The currently-shipped Tier-1 helper
+`checksum::update_file_md5_block_md5_crc32_fused` runs at ~494 MiB/s
+vs ~954 MiB/s for a naive 3-pass sequential at 16 KiB / 4 MiB. Cache
+traffic improved (per `perf stat`), wall-clock did not. The bench
+shows that an MD5x2 + CLMul fused 64-B driver hits ~925–933 MiB/s, so
+the path forward is to build a `HasherInput`-style driver around
+`md5x2_scalar` + `crc_clmul` and replace this helper on the create
+path. Tracked as T2.c; not included in this commit.
+
+## What was copied vs. re-derived
+
+* The instruction sequences inside each `asm!` block are direct
+ translations of the corresponding upstream `asm volatile` blocks: same
+ ordering, same register roles, same constant tables.
+* The Rust glue around them (struct layout, function signatures, error
+ handling, `#[cfg]` gating) is fresh par2rs code.
+* MD5 round constants and the 64 K table are the standard values from
+ RFC 1321 and were not copied as such.
+
+Each Rust file carries a header comment naming its specific upstream
+source file and giving the upstream project's copyright line so the
+provenance is greppable from inside the repo.
diff --git a/src/parpar_hasher/crc_armcrc.rs b/src/parpar_hasher/crc_armcrc.rs
new file mode 100644
index 00000000..81d19779
--- /dev/null
+++ b/src/parpar_hasher/crc_armcrc.rs
@@ -0,0 +1,33 @@
+//! ARM CRC32 backend for aarch64 (placeholder for future porting).
+//!
+//! This module will port the ARMv8 CRC32 implementation from
+//! par2cmdline-turbo/ParPar when resources allow. For now, it's a stub.
+//!
+//! Upstream source: parpar/hasher/crc_arm.h (61 SLOC)
+//!
+//! ## Plan
+//! Uses ARMv8 CRC32 instructions:
+//! - `__crc32d` — 64-bit CRC update (aarch64)
+//! - `__crc32w` — 32-bit CRC update (both architectures)
+//! - `__crc32h` — 16-bit CRC update
+//! - `__crc32b` — 8-bit CRC update
+//!
+//! These are available via `std::arch::aarch64` intrinsics and require
+//! the ARMv8 CRC feature (`__ARM_FEATURE_CRC32`).
+
+#![cfg(target_arch = "aarch64")]
+
+/// CRC32 backend using ARMv8 CRC instructions (placeholder).
+pub struct ArmCrc;
+
+// TODO: Implement when porting is ready
+//
+// pub struct State {
+// crc: u32,
+// }
+//
+// impl ArmCrc {
+// pub fn init() -> State { ... }
+// pub fn process_block(&mut self, src: &[u8; 64]) { ... }
+// pub fn finish(&self, tail: &[u8]) -> u32 { ... }
+// }
diff --git a/src/parpar_hasher/crc_clmul.rs b/src/parpar_hasher/crc_clmul.rs
new file mode 100644
index 00000000..6b2942ed
--- /dev/null
+++ b/src/parpar_hasher/crc_clmul.rs
@@ -0,0 +1,431 @@
+// Copyright (C) Anomaly Industries Inc. and par2rs contributors.
+// SPDX-License-Identifier: GPL-2.0-or-later
+//
+// PCLMULQDQ-based CRC32 (IEEE / zlib polynomial 0xEDB88320) using the
+// 4-fold parallel folding approach from Intel's white paper:
+// "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ"
+// http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
+//
+// This is a direct Rust port of:
+// parpar/hasher/crc_clmul.h
+// from par2cmdline-turbo / ParPar (https://github.com/animetosho/ParPar),
+// both GPL-2.0-or-later. Upstream took it from zlib-ng / Intel's zlib
+// patch; the magic constants (rk1/rk2/rk5/rk6/rk7/rk8 and the fold
+// constant) come straight from that Intel reference implementation.
+//
+// The state is four xmm registers ([State; 4] = 64 bytes), processed
+// 64 B per block — exactly matching the MD5x2 cadence in the create
+// path's HasherInput so we can fuse them at one cache-line read.
+//
+// Initial state: state[0] = 0x9db42487 packed into the low 32 bits of an
+// xmm; this constant is the precomputed fold of the 0xFFFFFFFF IEEE
+// initial XOR through the four-fold pipeline so the very first data
+// block can be folded in raw.
+//
+// Final output: standard IEEE CRC32 (already XOR'd with 0xFFFFFFFF
+// inside the finish reduction).
+
+use core::arch::x86_64::{
+ __m128i, _mm_and_si128, _mm_clmulepi64_si128, _mm_cvtsi32_si128, _mm_extract_epi32,
+ _mm_load_si128, _mm_loadu_si128, _mm_or_si128, _mm_set1_epi8, _mm_set_epi32, _mm_setzero_si128,
+ _mm_shuffle_epi8, _mm_slli_si128, _mm_srli_si128, _mm_xor_si128,
+};
+
+/// 64-byte CRC32 fold state.
+///
+/// Mirrors `__m128i state[4]` from upstream `crc_clmul.h`.
+#[repr(C, align(16))]
+#[derive(Clone, Copy)]
+pub struct State {
+ pub xmm: [u128; 4],
+}
+
+impl State {
+ /// Standard IEEE CRC32 initial fold state (matches `crc_init_clmul`).
+ #[inline]
+ pub fn new() -> Self {
+ let mut s = Self { xmm: [0; 4] };
+ // SAFETY: requires pclmul/sse2 only for the store; these are
+ // baseline x86_64 features for the load/store, but we keep
+ // initialisation in plain integer code to avoid pulling in any
+ // SIMD requirement just to construct a State.
+ s.xmm[0] = 0x9db42487u128; // low 32 bits of xmm0
+ s
+ }
+}
+
+impl Default for State {
+ #[inline]
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+/// `pshufb` shuffle/shift table, mirrors `parpar/hasher/tables.cpp`'s
+/// `pshufb_shf_table[60]`. Indexed by `(len - 1)` for `1 <= len <= 15`.
+///
+/// Each row is 16 bytes (one xmm). The high bit (`0x80`) flagged bytes
+/// in `pshufb` produce a zero, which is how upstream implements the
+/// combined "shift-left by `len`" / "shift-right by `16 - len`" trick.
+#[repr(C, align(16))]
+pub(super) struct ShfTable(pub(super) [u32; 60]);
+
+pub(super) static PSHUFB_SHF_TABLE: ShfTable = ShfTable([
+ 0x84838281, 0x88878685, 0x8c8b8a89, 0x008f8e8d, // shl 15 / shr 1
+ 0x85848382, 0x89888786, 0x8d8c8b8a, 0x01008f8e, // shl 14 / shr 2
+ 0x86858483, 0x8a898887, 0x8e8d8c8b, 0x0201008f, // shl 13 / shr 3
+ 0x87868584, 0x8b8a8988, 0x8f8e8d8c, 0x03020100, // shl 12 / shr 4
+ 0x88878685, 0x8c8b8a89, 0x008f8e8d, 0x04030201, // shl 11 / shr 5
+ 0x89888786, 0x8d8c8b8a, 0x01008f8e, 0x05040302, // shl 10 / shr 6
+ 0x8a898887, 0x8e8d8c8b, 0x0201008f, 0x06050403, // shl 9 / shr 7
+ 0x8b8a8988, 0x8f8e8d8c, 0x03020100, 0x07060504, // shl 8 / shr 8
+ 0x8c8b8a89, 0x008f8e8d, 0x04030201, 0x08070605, // shl 7 / shr 9
+ 0x8d8c8b8a, 0x01008f8e, 0x05040302, 0x09080706, // shl 6 / shr 10
+ 0x8e8d8c8b, 0x0201008f, 0x06050403, 0x0a090807, // shl 5 / shr 11
+ 0x8f8e8d8c, 0x03020100, 0x07060504, 0x0b0a0908, // shl 4 / shr 12
+ 0x008f8e8d, 0x04030201, 0x08070605, 0x0c0b0a09, // shl 3 / shr 13
+ 0x01008f8e, 0x05040302, 0x09080706, 0x0d0c0b0a, // shl 2 / shr 14
+ 0x0201008f, 0x06050403, 0x0a090807, 0x0e0d0c0b, // shl 1 / shr 15
+]);
+
+#[inline]
+#[target_feature(enable = "sse2,pclmulqdq")]
+unsafe fn double_xor(a: __m128i, b: __m128i, c: __m128i) -> __m128i {
+ _mm_xor_si128(_mm_xor_si128(a, b), c)
+}
+
+#[inline]
+#[target_feature(enable = "sse2,pclmulqdq")]
+unsafe fn do_one_fold_merge(src: __m128i, data: __m128i) -> __m128i {
+ // `xmm_fold4` packs rk3 (0xc6e41596) in the low 64 bits and rk4
+ // (0x54442bd4) in the high 64 bits. These come from upstream's
+ // `_mm_set_epi32(0x00000001, 0x54442bd4, 0x00000001, 0xc6e41596)`.
+ let xmm_fold4 = _mm_set_epi32(
+ 0x00000001,
+ 0x54442bd4u32 as i32,
+ 0x00000001,
+ 0xc6e41596u32 as i32,
+ );
+ double_xor(
+ _mm_clmulepi64_si128(src, xmm_fold4, 0x01),
+ data,
+ _mm_clmulepi64_si128(src, xmm_fold4, 0x10),
+ )
+}
+
+/// Reset state to the standard IEEE CRC32 initial fold.
+///
+/// # Safety
+/// Caller must have verified `is_x86_feature_detected!("sse2")` (a
+/// baseline x86_64 feature) before invocation. `state` must be a valid
+/// mutable reference; nothing else is read.
+#[inline]
+#[target_feature(enable = "sse2")]
+pub unsafe fn init(state: &mut State) {
+ let xmm0 = _mm_cvtsi32_si128(0x9db42487u32 as i32);
+ let zero = _mm_setzero_si128();
+ let p = state.xmm.as_mut_ptr() as *mut __m128i;
+ core::arch::x86_64::_mm_store_si128(p, xmm0);
+ core::arch::x86_64::_mm_store_si128(p.add(1), zero);
+ core::arch::x86_64::_mm_store_si128(p.add(2), zero);
+ core::arch::x86_64::_mm_store_si128(p.add(3), zero);
+}
+
+/// Fold one 64-byte block into the state.
+///
+/// `src` must point to at least 64 readable bytes. Mirrors
+/// `crc_process_block_clmul`.
+///
+/// # Safety
+/// `src` must be valid for a 64-byte read. Caller must have checked
+/// `is_x86_feature_detected!("pclmulqdq")` and `"sse4.1"` (for the
+/// finish path's `_mm_shuffle_epi8`; the block path itself only needs
+/// `pclmulqdq` + `sse2`, but we group features together).
+#[inline]
+#[target_feature(enable = "sse2,sse4.1,pclmulqdq")]
+pub unsafe fn process_block(state: &mut State, src: *const u8) {
+ let s = state.xmm.as_mut_ptr() as *mut __m128i;
+ let p = src as *const __m128i;
+
+ let xmm_t0 = _mm_loadu_si128(p);
+ let xmm_t1 = _mm_loadu_si128(p.add(1));
+ let xmm_t2 = _mm_loadu_si128(p.add(2));
+ let xmm_t3 = _mm_loadu_si128(p.add(3));
+
+ let c0 = _mm_load_si128(s);
+ let c1 = _mm_load_si128(s.add(1));
+ let c2 = _mm_load_si128(s.add(2));
+ let c3 = _mm_load_si128(s.add(3));
+
+ core::arch::x86_64::_mm_store_si128(s, do_one_fold_merge(c0, xmm_t0));
+ core::arch::x86_64::_mm_store_si128(s.add(1), do_one_fold_merge(c1, xmm_t1));
+ core::arch::x86_64::_mm_store_si128(s.add(2), do_one_fold_merge(c2, xmm_t2));
+ core::arch::x86_64::_mm_store_si128(s.add(3), do_one_fold_merge(c3, xmm_t3));
+}
+
+/// Finish: absorb `len` (≤ 63) trailing bytes and reduce the four-fold
+/// state to a single 32-bit IEEE CRC32. Mirrors `crc_finish_clmul`.
+///
+/// # Safety
+/// `src` must be valid for `len` readable bytes. `len` must be ≤ 63.
+#[inline]
+#[target_feature(enable = "sse2,sse4.1,pclmulqdq")]
+pub unsafe fn finish(state: &mut State, src: *const u8, len: usize) -> u32 {
+ debug_assert!(len <= 63);
+ let s = state.xmm.as_mut_ptr() as *mut __m128i;
+ let mut src = src;
+ let mut len = len;
+
+ // Stage 1: absorb full 16-byte chunks 48 / 32 / 16 of the tail.
+ if len >= 48 {
+ let xmm_t0 = _mm_loadu_si128(src as *const __m128i);
+ let xmm_t1 = _mm_loadu_si128((src as *const __m128i).add(1));
+ let xmm_t2 = _mm_loadu_si128((src as *const __m128i).add(2));
+
+ let c0 = _mm_load_si128(s);
+ let c1 = _mm_load_si128(s.add(1));
+ let c2 = _mm_load_si128(s.add(2));
+ let c3 = _mm_load_si128(s.add(3));
+
+ let new3 = do_one_fold_merge(c2, xmm_t2);
+ let new2 = do_one_fold_merge(c1, xmm_t1);
+ let new1 = do_one_fold_merge(c0, xmm_t0);
+
+ core::arch::x86_64::_mm_store_si128(s, c3);
+ core::arch::x86_64::_mm_store_si128(s.add(1), new1);
+ core::arch::x86_64::_mm_store_si128(s.add(2), new2);
+ core::arch::x86_64::_mm_store_si128(s.add(3), new3);
+ } else if len >= 32 {
+ let xmm_t0 = _mm_loadu_si128(src as *const __m128i);
+ let xmm_t1 = _mm_loadu_si128((src as *const __m128i).add(1));
+
+ let c0 = _mm_load_si128(s);
+ let c1 = _mm_load_si128(s.add(1));
+ let c2 = _mm_load_si128(s.add(2));
+ let c3 = _mm_load_si128(s.add(3));
+
+ let new3 = do_one_fold_merge(c1, xmm_t1);
+ let new2 = do_one_fold_merge(c0, xmm_t0);
+
+ core::arch::x86_64::_mm_store_si128(s, c2);
+ core::arch::x86_64::_mm_store_si128(s.add(1), c3);
+ core::arch::x86_64::_mm_store_si128(s.add(2), new2);
+ core::arch::x86_64::_mm_store_si128(s.add(3), new3);
+ } else if len >= 16 {
+ let xmm_t0 = _mm_loadu_si128(src as *const __m128i);
+
+ let c0 = _mm_load_si128(s);
+ let c1 = _mm_load_si128(s.add(1));
+ let c2 = _mm_load_si128(s.add(2));
+ let c3 = _mm_load_si128(s.add(3));
+
+ let new3 = do_one_fold_merge(c0, xmm_t0);
+
+ core::arch::x86_64::_mm_store_si128(s, c1);
+ core::arch::x86_64::_mm_store_si128(s.add(1), c2);
+ core::arch::x86_64::_mm_store_si128(s.add(2), c3);
+ core::arch::x86_64::_mm_store_si128(s.add(3), new3);
+ }
+ src = src.add(len & 48);
+ len &= 15;
+
+ // Stage 2: absorb the final 1..=15 byte fragment.
+ if len > 0 {
+ // Load the (len-1)th row of the shuffle table (16 bytes).
+ let shf_row = (PSHUFB_SHF_TABLE.0.as_ptr() as *const __m128i).add(len - 1);
+ let xmm_shl = _mm_load_si128(shf_row);
+ let xmm_shr = _mm_xor_si128(xmm_shl, _mm_set1_epi8(-128));
+
+ // Zero-padded load of the tail.
+ let mut tail_bytes = [0u8; 16];
+ core::ptr::copy_nonoverlapping(src, tail_bytes.as_mut_ptr(), len);
+ let xmm_t0 = _mm_loadu_si128(tail_bytes.as_ptr() as *const __m128i);
+
+ let c0 = _mm_load_si128(s);
+ let c1 = _mm_load_si128(s.add(1));
+ let c2 = _mm_load_si128(s.add(2));
+ let c3 = _mm_load_si128(s.add(3));
+
+ let xmm_t1 = _mm_shuffle_epi8(c0, xmm_shl);
+
+ let new0 = _mm_or_si128(_mm_shuffle_epi8(c0, xmm_shr), _mm_shuffle_epi8(c1, xmm_shl));
+ let new1 = _mm_or_si128(_mm_shuffle_epi8(c1, xmm_shr), _mm_shuffle_epi8(c2, xmm_shl));
+ let new2 = _mm_or_si128(_mm_shuffle_epi8(c2, xmm_shr), _mm_shuffle_epi8(c3, xmm_shl));
+ let new3_intermediate = _mm_or_si128(
+ _mm_shuffle_epi8(c3, xmm_shr),
+ _mm_shuffle_epi8(xmm_t0, xmm_shl),
+ );
+ let new3 = do_one_fold_merge(xmm_t1, new3_intermediate);
+
+ core::arch::x86_64::_mm_store_si128(s, new0);
+ core::arch::x86_64::_mm_store_si128(s.add(1), new1);
+ core::arch::x86_64::_mm_store_si128(s.add(2), new2);
+ core::arch::x86_64::_mm_store_si128(s.add(3), new3);
+ }
+
+ // Stage 3: reduce four xmm to one (rk1/rk2 fold).
+ let crc_fold_rk12 = _mm_set_epi32(
+ 0x00000001,
+ 0x751997d0u32 as i32, // rk2
+ 0x00000000,
+ 0xccaa009eu32 as i32, // rk1
+ );
+
+ let c0 = _mm_load_si128(s);
+ let c1 = _mm_load_si128(s.add(1));
+ let c2 = _mm_load_si128(s.add(2));
+ let c3 = _mm_load_si128(s.add(3));
+
+ let mut t = double_xor(
+ c1,
+ _mm_clmulepi64_si128(c0, crc_fold_rk12, 0x10),
+ _mm_clmulepi64_si128(c0, crc_fold_rk12, 0x01),
+ );
+ t = double_xor(
+ c2,
+ _mm_clmulepi64_si128(t, crc_fold_rk12, 0x10),
+ _mm_clmulepi64_si128(t, crc_fold_rk12, 0x01),
+ );
+ t = double_xor(
+ c3,
+ _mm_clmulepi64_si128(t, crc_fold_rk12, 0x10),
+ _mm_clmulepi64_si128(t, crc_fold_rk12, 0x01),
+ );
+
+ // Stage 4: 128 -> 64 fold (rk5/rk6).
+ let crc_fold_rk56 = _mm_set_epi32(
+ 0x00000001,
+ 0x63cd6124u32 as i32, // rk6
+ 0x00000000,
+ 0xccaa009eu32 as i32, // rk5
+ );
+
+ let mut xmm_t1 = _mm_xor_si128(
+ _mm_clmulepi64_si128(t, crc_fold_rk56, 0),
+ _mm_srli_si128(t, 8),
+ );
+
+ let mut xmm_t0 = _mm_slli_si128(xmm_t1, 4);
+ xmm_t0 = _mm_clmulepi64_si128(xmm_t0, crc_fold_rk56, 0x10);
+ let mask = _mm_set_epi32(0, -1, -1, 0);
+ xmm_t1 = _mm_and_si128(xmm_t1, mask);
+ xmm_t0 = _mm_xor_si128(xmm_t0, xmm_t1);
+
+ // Stage 5: Barrett reduction (rk7/rk8) for the final 32-bit CRC.
+ let crc_fold_rk78 = _mm_set_epi32(
+ 0x00000001,
+ 0xdb710640u32 as i32, // rk8
+ 0x00000000,
+ 0xf7011641u32 as i32, // rk7
+ );
+
+ let xmm_q1 = _mm_clmulepi64_si128(xmm_t0, crc_fold_rk78, 0);
+ let xmm_q2 = _mm_clmulepi64_si128(xmm_q1, crc_fold_rk78, 0x10);
+ // result = NOT(q2 XOR t0) folded with the 0xFFFFFFFF init absorption mask.
+ let xmm_xor_mask = _mm_set_epi32(0, -1, -1, 0);
+ let xmm_t0_xor = _mm_xor_si128(xmm_t0, xmm_xor_mask);
+ let xmm_final = _mm_xor_si128(xmm_q2, xmm_t0_xor);
+ _mm_extract_epi32(xmm_final, 2) as u32
+}
+
+/// Convenience: detect once at startup, then call this on each block.
+#[inline]
+pub fn is_supported() -> bool {
+ is_x86_feature_detected!("sse4.1") && is_x86_feature_detected!("pclmulqdq")
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// Drive the CLMul CRC32 over a buffer the same way the create path
+ /// will: 64 B blocks, then a tail.
+ fn clmul_crc(data: &[u8]) -> u32 {
+ assert!(is_supported(), "test host must have sse4.1 + pclmulqdq");
+ let mut state = State::new();
+ unsafe {
+ init(&mut state);
+ let mut p = data.as_ptr();
+ let mut remaining = data.len();
+ while remaining >= 64 {
+ process_block(&mut state, p);
+ p = p.add(64);
+ remaining -= 64;
+ }
+ finish(&mut state, p, remaining)
+ }
+ }
+
+ fn reference_crc(data: &[u8]) -> u32 {
+ let mut h = crc32fast::Hasher::new();
+ h.update(data);
+ h.finalize()
+ }
+
+ #[test]
+ fn matches_crc32fast_empty() {
+ assert_eq!(clmul_crc(&[]), reference_crc(&[]));
+ }
+
+ #[test]
+ fn matches_crc32fast_short_lengths() {
+ // Cover every length 1..=63 so every branch in finish() is hit
+ // (1..=15 fragment, 16..=31 + fragment, 32..=47 + fragment,
+ // 48..=63 + fragment) plus the no-tail case.
+ let pattern: Vec = (0..256u16)
+ .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7))
+ .collect();
+ for len in 0..=63 {
+ let buf = &pattern[..len];
+ assert_eq!(clmul_crc(buf), reference_crc(buf), "mismatch at len={len}");
+ }
+ }
+
+ #[test]
+ fn matches_crc32fast_block_aligned() {
+ // Exact multiples of 64.
+ let pattern: Vec = (0..4096u32)
+ .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7))
+ .collect();
+ for blocks in [1usize, 2, 3, 4, 16, 64] {
+ let len = blocks * 64;
+ let buf = &pattern[..len];
+ assert_eq!(clmul_crc(buf), reference_crc(buf), "mismatch at len={len}");
+ }
+ }
+
+ #[test]
+ fn matches_crc32fast_block_plus_tail() {
+ let pattern: Vec = (0..8192u32)
+ .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7))
+ .collect();
+ // Lots of (full-blocks, tail) combinations.
+ for &full in &[0usize, 1, 2, 5, 16, 100] {
+ for tail in [1usize, 7, 15, 16, 17, 31, 32, 33, 47, 48, 49, 63] {
+ let len = full * 64 + tail;
+ if len > pattern.len() {
+ continue;
+ }
+ let buf = &pattern[..len];
+ assert_eq!(
+ clmul_crc(buf),
+ reference_crc(buf),
+ "mismatch at full={full} tail={tail} (len={len})"
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn matches_crc32fast_known_vectors() {
+ // Sanity vs published CRC32 test vectors.
+ assert_eq!(clmul_crc(b""), 0x00000000);
+ assert_eq!(clmul_crc(b"a"), 0xe8b7be43);
+ assert_eq!(clmul_crc(b"abc"), 0x352441c2);
+ assert_eq!(clmul_crc(b"123456789"), 0xcbf43926);
+ assert_eq!(
+ clmul_crc(b"The quick brown fox jumps over the lazy dog"),
+ 0x414fa339
+ );
+ }
+}
diff --git a/src/parpar_hasher/crc_clmul_avx512.rs b/src/parpar_hasher/crc_clmul_avx512.rs
new file mode 100644
index 00000000..7ff26b98
--- /dev/null
+++ b/src/parpar_hasher/crc_clmul_avx512.rs
@@ -0,0 +1,335 @@
+// Copyright (C) Anomaly Industries Inc. and par2rs contributors.
+// SPDX-License-Identifier: GPL-2.0-or-later
+//
+// AVX-512VL flavour of the 4-fold PCLMULQDQ CRC32 driver. Mirrors the
+// `_CRC_USE_AVX512_` branch in `parpar/hasher/crc_clmul.h`: the only
+// substantive change vs the SSE4.1 baseline is `double_xor` collapsing
+// from two `pxor` instructions into a single `vpternlogd` with truth
+// table 0x96 (`a ^ b ^ c`). Everything else — the rk1..rk8 constants,
+// the four-fold pipeline shape, the pshufb tail trick — is identical,
+// so we reuse [`super::crc_clmul::State`], `init`, and the
+// `PSHUFB_SHF_TABLE` from the SSE module.
+//
+// `HasherInput` selects between this and the SSE baseline at compile
+// time via the `Md5x2::USE_AVX512_CRC` associated const, monomorphised
+// per backend. There is no separate CRC backend trait.
+
+#![cfg(target_arch = "x86_64")]
+
+use core::arch::x86_64::{
+ __m128i, _mm_and_si128, _mm_clmulepi64_si128, _mm_extract_epi32, _mm_load_si128,
+ _mm_loadu_si128, _mm_or_si128, _mm_set1_epi8, _mm_set_epi32, _mm_shuffle_epi8, _mm_slli_si128,
+ _mm_srli_si128, _mm_ternarylogic_epi32, _mm_xor_si128,
+};
+
+use super::crc_clmul::{State, PSHUFB_SHF_TABLE};
+
+/// `vpternlogd` with truth table 0x96 (`a ^ b ^ c`) — replaces the two
+/// SSE2 xor instructions with a single AVX-512VL op. See upstream
+/// `crc_clmul.h:24`.
+#[inline]
+#[target_feature(enable = "avx512f,avx512vl")]
+unsafe fn double_xor_avx512(a: __m128i, b: __m128i, c: __m128i) -> __m128i {
+ _mm_ternarylogic_epi32::<0x96>(a, b, c)
+}
+
+#[inline]
+#[target_feature(enable = "avx512f,avx512vl,pclmulqdq")]
+unsafe fn do_one_fold_merge_avx512(src: __m128i, data: __m128i) -> __m128i {
+ let xmm_fold4 = _mm_set_epi32(
+ 0x00000001,
+ 0x54442bd4u32 as i32,
+ 0x00000001,
+ 0xc6e41596u32 as i32,
+ );
+ double_xor_avx512(
+ _mm_clmulepi64_si128(src, xmm_fold4, 0x01),
+ data,
+ _mm_clmulepi64_si128(src, xmm_fold4, 0x10),
+ )
+}
+
+/// Fold one 64-byte block. AVX-512VL flavour of `crc_clmul::process_block`.
+///
+/// # Safety
+/// Caller must have verified `is_x86_feature_detected!("avx512f")`,
+/// `"avx512vl"`, and `"pclmulqdq"`. `src` must be valid for 64 readable bytes.
+#[inline]
+#[target_feature(enable = "avx512f,avx512vl,sse4.1,pclmulqdq")]
+pub unsafe fn process_block(state: &mut State, src: *const u8) {
+ let s = state.xmm.as_mut_ptr() as *mut __m128i;
+ let p = src as *const __m128i;
+
+ let xmm_t0 = _mm_loadu_si128(p);
+ let xmm_t1 = _mm_loadu_si128(p.add(1));
+ let xmm_t2 = _mm_loadu_si128(p.add(2));
+ let xmm_t3 = _mm_loadu_si128(p.add(3));
+
+ let c0 = _mm_load_si128(s);
+ let c1 = _mm_load_si128(s.add(1));
+ let c2 = _mm_load_si128(s.add(2));
+ let c3 = _mm_load_si128(s.add(3));
+
+ core::arch::x86_64::_mm_store_si128(s, do_one_fold_merge_avx512(c0, xmm_t0));
+ core::arch::x86_64::_mm_store_si128(s.add(1), do_one_fold_merge_avx512(c1, xmm_t1));
+ core::arch::x86_64::_mm_store_si128(s.add(2), do_one_fold_merge_avx512(c2, xmm_t2));
+ core::arch::x86_64::_mm_store_si128(s.add(3), do_one_fold_merge_avx512(c3, xmm_t3));
+}
+
+/// Finish: absorb `len` (≤ 63) trailing bytes; reduce 4×xmm → 32-bit IEEE CRC.
+///
+/// AVX-512VL flavour of `crc_clmul::finish`. Uses `vpternlogd`-backed
+/// `double_xor` in stages 1 and 3 (rk1/rk2 fold). Stages 4 (rk5/rk6) and
+/// 5 (Barrett rk7/rk8) only use single xors and are unchanged.
+///
+/// # Safety
+/// Caller must have verified avx512f + avx512vl + sse4.1 + pclmulqdq.
+/// `src` must be valid for `len` readable bytes; `len ≤ 63`.
+#[inline]
+#[target_feature(enable = "avx512f,avx512vl,sse4.1,pclmulqdq")]
+pub unsafe fn finish(state: &mut State, src: *const u8, len: usize) -> u32 {
+ debug_assert!(len <= 63);
+ let s = state.xmm.as_mut_ptr() as *mut __m128i;
+ let mut src = src;
+ let mut len = len;
+
+ if len >= 48 {
+ let xmm_t0 = _mm_loadu_si128(src as *const __m128i);
+ let xmm_t1 = _mm_loadu_si128((src as *const __m128i).add(1));
+ let xmm_t2 = _mm_loadu_si128((src as *const __m128i).add(2));
+
+ let c0 = _mm_load_si128(s);
+ let c1 = _mm_load_si128(s.add(1));
+ let c2 = _mm_load_si128(s.add(2));
+ let c3 = _mm_load_si128(s.add(3));
+
+ let new3 = do_one_fold_merge_avx512(c2, xmm_t2);
+ let new2 = do_one_fold_merge_avx512(c1, xmm_t1);
+ let new1 = do_one_fold_merge_avx512(c0, xmm_t0);
+
+ core::arch::x86_64::_mm_store_si128(s, c3);
+ core::arch::x86_64::_mm_store_si128(s.add(1), new1);
+ core::arch::x86_64::_mm_store_si128(s.add(2), new2);
+ core::arch::x86_64::_mm_store_si128(s.add(3), new3);
+ } else if len >= 32 {
+ let xmm_t0 = _mm_loadu_si128(src as *const __m128i);
+ let xmm_t1 = _mm_loadu_si128((src as *const __m128i).add(1));
+
+ let c0 = _mm_load_si128(s);
+ let c1 = _mm_load_si128(s.add(1));
+ let c2 = _mm_load_si128(s.add(2));
+ let c3 = _mm_load_si128(s.add(3));
+
+ let new3 = do_one_fold_merge_avx512(c1, xmm_t1);
+ let new2 = do_one_fold_merge_avx512(c0, xmm_t0);
+
+ core::arch::x86_64::_mm_store_si128(s, c2);
+ core::arch::x86_64::_mm_store_si128(s.add(1), c3);
+ core::arch::x86_64::_mm_store_si128(s.add(2), new2);
+ core::arch::x86_64::_mm_store_si128(s.add(3), new3);
+ } else if len >= 16 {
+ let xmm_t0 = _mm_loadu_si128(src as *const __m128i);
+
+ let c0 = _mm_load_si128(s);
+ let c1 = _mm_load_si128(s.add(1));
+ let c2 = _mm_load_si128(s.add(2));
+ let c3 = _mm_load_si128(s.add(3));
+
+ let new3 = do_one_fold_merge_avx512(c0, xmm_t0);
+
+ core::arch::x86_64::_mm_store_si128(s, c1);
+ core::arch::x86_64::_mm_store_si128(s.add(1), c2);
+ core::arch::x86_64::_mm_store_si128(s.add(2), c3);
+ core::arch::x86_64::_mm_store_si128(s.add(3), new3);
+ }
+ src = src.add(len & 48);
+ len &= 15;
+
+ if len > 0 {
+ let shf_row = (PSHUFB_SHF_TABLE.0.as_ptr() as *const __m128i).add(len - 1);
+ let xmm_shl = _mm_load_si128(shf_row);
+ let xmm_shr = _mm_xor_si128(xmm_shl, _mm_set1_epi8(-128));
+
+ let mut tail_bytes = [0u8; 16];
+ core::ptr::copy_nonoverlapping(src, tail_bytes.as_mut_ptr(), len);
+ let xmm_t0 = _mm_loadu_si128(tail_bytes.as_ptr() as *const __m128i);
+
+ let c0 = _mm_load_si128(s);
+ let c1 = _mm_load_si128(s.add(1));
+ let c2 = _mm_load_si128(s.add(2));
+ let c3 = _mm_load_si128(s.add(3));
+
+ let xmm_t1 = _mm_shuffle_epi8(c0, xmm_shl);
+
+ let new0 = _mm_or_si128(_mm_shuffle_epi8(c0, xmm_shr), _mm_shuffle_epi8(c1, xmm_shl));
+ let new1 = _mm_or_si128(_mm_shuffle_epi8(c1, xmm_shr), _mm_shuffle_epi8(c2, xmm_shl));
+ let new2 = _mm_or_si128(_mm_shuffle_epi8(c2, xmm_shr), _mm_shuffle_epi8(c3, xmm_shl));
+ let new3_intermediate = _mm_or_si128(
+ _mm_shuffle_epi8(c3, xmm_shr),
+ _mm_shuffle_epi8(xmm_t0, xmm_shl),
+ );
+ let new3 = do_one_fold_merge_avx512(xmm_t1, new3_intermediate);
+
+ core::arch::x86_64::_mm_store_si128(s, new0);
+ core::arch::x86_64::_mm_store_si128(s.add(1), new1);
+ core::arch::x86_64::_mm_store_si128(s.add(2), new2);
+ core::arch::x86_64::_mm_store_si128(s.add(3), new3);
+ }
+
+ // Stage 3: reduce four xmm to one (rk1/rk2 fold) — vpternlogd path.
+ let crc_fold_rk12 = _mm_set_epi32(
+ 0x00000001,
+ 0x751997d0u32 as i32,
+ 0x00000000,
+ 0xccaa009eu32 as i32,
+ );
+
+ let c0 = _mm_load_si128(s);
+ let c1 = _mm_load_si128(s.add(1));
+ let c2 = _mm_load_si128(s.add(2));
+ let c3 = _mm_load_si128(s.add(3));
+
+ let mut t = double_xor_avx512(
+ c1,
+ _mm_clmulepi64_si128(c0, crc_fold_rk12, 0x10),
+ _mm_clmulepi64_si128(c0, crc_fold_rk12, 0x01),
+ );
+ t = double_xor_avx512(
+ c2,
+ _mm_clmulepi64_si128(t, crc_fold_rk12, 0x10),
+ _mm_clmulepi64_si128(t, crc_fold_rk12, 0x01),
+ );
+ t = double_xor_avx512(
+ c3,
+ _mm_clmulepi64_si128(t, crc_fold_rk12, 0x10),
+ _mm_clmulepi64_si128(t, crc_fold_rk12, 0x01),
+ );
+
+ // Stage 4: 128 -> 64 fold (rk5/rk6).
+ let crc_fold_rk56 = _mm_set_epi32(
+ 0x00000001,
+ 0x63cd6124u32 as i32,
+ 0x00000000,
+ 0xccaa009eu32 as i32,
+ );
+
+ let mut xmm_t1 = _mm_xor_si128(
+ _mm_clmulepi64_si128(t, crc_fold_rk56, 0),
+ _mm_srli_si128(t, 8),
+ );
+
+ let mut xmm_t0 = _mm_slli_si128(xmm_t1, 4);
+ xmm_t0 = _mm_clmulepi64_si128(xmm_t0, crc_fold_rk56, 0x10);
+ let mask = _mm_set_epi32(0, -1, -1, 0);
+ xmm_t1 = _mm_and_si128(xmm_t1, mask);
+ xmm_t0 = _mm_xor_si128(xmm_t0, xmm_t1);
+
+ // Stage 5: Barrett reduction (rk7/rk8).
+ let crc_fold_rk78 = _mm_set_epi32(
+ 0x00000001,
+ 0xdb710640u32 as i32,
+ 0x00000000,
+ 0xf7011641u32 as i32,
+ );
+
+ let xmm_q1 = _mm_clmulepi64_si128(xmm_t0, crc_fold_rk78, 0);
+ let xmm_q2 = _mm_clmulepi64_si128(xmm_q1, crc_fold_rk78, 0x10);
+ let xmm_xor_mask = _mm_set_epi32(0, -1, -1, 0);
+ let xmm_t0_xor = _mm_xor_si128(xmm_t0, xmm_xor_mask);
+ let xmm_final = _mm_xor_si128(xmm_q2, xmm_t0_xor);
+ _mm_extract_epi32(xmm_final, 2) as u32
+}
+
+/// Convenience: detect support once at startup.
+#[inline]
+pub fn is_supported() -> bool {
+ is_x86_feature_detected!("avx512f")
+ && is_x86_feature_detected!("avx512vl")
+ && is_x86_feature_detected!("pclmulqdq")
+ && is_x86_feature_detected!("sse4.1")
+}
+
+#[cfg(test)]
+mod tests {
+ use super::super::crc_clmul;
+ use super::*;
+
+ fn avx512_crc(data: &[u8]) -> u32 {
+ let mut state = State::new();
+ unsafe {
+ crc_clmul::init(&mut state);
+ let mut p = data.as_ptr();
+ let mut remaining = data.len();
+ while remaining >= 64 {
+ process_block(&mut state, p);
+ p = p.add(64);
+ remaining -= 64;
+ }
+ finish(&mut state, p, remaining)
+ }
+ }
+
+ fn sse_crc(data: &[u8]) -> u32 {
+ let mut state = State::new();
+ unsafe {
+ crc_clmul::init(&mut state);
+ let mut p = data.as_ptr();
+ let mut remaining = data.len();
+ while remaining >= 64 {
+ crc_clmul::process_block(&mut state, p);
+ p = p.add(64);
+ remaining -= 64;
+ }
+ crc_clmul::finish(&mut state, p, remaining)
+ }
+ }
+
+ #[test]
+ #[ignore = "requires AVX-512VL + PCLMULQDQ hardware"]
+ fn avx512_matches_sse_known_vectors() {
+ assert!(is_supported(), "host lacks avx512vl + pclmulqdq");
+ for s in [
+ &b""[..],
+ &b"a"[..],
+ &b"abc"[..],
+ &b"123456789"[..],
+ &b"The quick brown fox jumps over the lazy dog"[..],
+ ] {
+ assert_eq!(avx512_crc(s), sse_crc(s), "mismatch on {:?}", s);
+ }
+ }
+
+ #[test]
+ #[ignore = "requires AVX-512VL + PCLMULQDQ hardware"]
+ fn avx512_matches_sse_short_lengths() {
+ assert!(is_supported());
+ let pat: Vec = (0..256u16)
+ .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7))
+ .collect();
+ for len in 0..=63 {
+ assert_eq!(avx512_crc(&pat[..len]), sse_crc(&pat[..len]));
+ }
+ }
+
+ #[test]
+ #[ignore = "requires AVX-512VL + PCLMULQDQ hardware"]
+ fn avx512_matches_sse_block_plus_tail() {
+ assert!(is_supported());
+ let pat: Vec = (0..8192u32)
+ .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7))
+ .collect();
+ for &full in &[0usize, 1, 2, 5, 16, 100] {
+ for tail in [0usize, 1, 7, 15, 16, 17, 31, 32, 33, 47, 48, 49, 63] {
+ let len = full * 64 + tail;
+ if len > pat.len() {
+ continue;
+ }
+ assert_eq!(
+ avx512_crc(&pat[..len]),
+ sse_crc(&pat[..len]),
+ "mismatch full={full} tail={tail}"
+ );
+ }
+ }
+ }
+}
diff --git a/src/parpar_hasher/hasher_input.rs b/src/parpar_hasher/hasher_input.rs
new file mode 100644
index 00000000..3487ffbe
--- /dev/null
+++ b/src/parpar_hasher/hasher_input.rs
@@ -0,0 +1,753 @@
+// Copyright (C) Anomaly Industries Inc. and par2rs contributors.
+// SPDX-License-Identifier: GPL-2.0-or-later
+//
+// Fused per-file HasherInput driver — block-MD5 + file-MD5 + block-CRC32
+// from a single 64-byte cache-line read.
+//
+// Direct Rust port of:
+// parpar/hasher/hasher_input_base.h (the driver: update / getBlock / end)
+// parpar/hasher/hasher_input.cpp (the per-impl instantiation glue)
+// parpar/hasher/md5-final.c (md5_final_block tail finalisation)
+// parpar/hasher/crc_zeropad.c (crc_zeroPad GF(2) multiply)
+// from par2cmdline-turbo / ParPar (https://github.com/animetosho/ParPar),
+// both GPL-2.0-or-later. par2rs is GPL-2.0-or-later, licenses compatible.
+//
+// Lane convention (matches upstream HASH2X_BLOCK / HASH2X_FILE):
+//
+// * `md5_state[0..4]` = lane 0 = per-block MD5 (reset between blocks)
+// * `md5_state[4..8]` = lane 1 = per-file MD5 (rolls across all blocks)
+//
+// Staggered offset bookkeeping (the trick that lets two MD5 lanes share
+// one walk over the input bytes when they don't start at the same byte):
+//
+// * `tmp[0..64]` — staging area for the file-MD5 lane.
+// * `tmp[pos_offset..pos_offset+64]` — staging area for the block-MD5
+// lane. `pos_offset` ∈ [0, 63].
+// * After each `get_block`, `pos_offset := tmp_len` so the new block
+// lane's logical start point realigns with where the file lane is.
+//
+// This is the always-available scalar+CLMul tier (Tier 1): GPR MD5x2 +
+// PCLMULQDQ CRC32. Future tiers (SSE2/AVX-512 MD5x2, ARM NEON+PMULL) plug
+// in via runtime dispatch on top of this contract.
+
+#![cfg(target_arch = "x86_64")]
+
+use super::crc_clmul;
+use super::crc_clmul_avx512;
+use super::md5x2::Md5x2;
+
+/// Default backend on x86_64.
+///
+/// Upstream picks `HasherInput_ClMulScalar` (= scalar MD5x2 + CLMul
+/// CRC32) on big cores and `HasherInput_ClMulSSE` (= SSE2 MD5x2 + CLMul
+/// CRC32) only on small cores (Tremont / Gracemont / Zen1 — see
+/// `parpar/hasher/hasher.cpp:107-117` for the exact dispatch rules).
+/// The reason: SSE2 MD5x2 packs both lanes' 32-bit MD5 words into one
+/// xmm register, which serialises the per-round dependency chain that
+/// scalar GPR code would have run in parallel via OoO execution. On
+/// modern Skylake-class / Zen3+ big cores, the scalar version is
+/// **faster**.
+///
+/// We don't yet have runtime CPU dispatch, so the default mirrors the
+/// most common upstream outcome: scalar. The SSE2 backend remains
+/// available via `HasherInput` for explicit selection / for the
+/// future small-core dispatch path.
+pub type DefaultBackend = super::md5x2_scalar::Scalar;
+
+const MD5_BLOCKSIZE: usize = 64;
+
+/// Per-backend CRC `process_block` dispatch — picks the AVX-512VL
+/// `vpternlogd` fold when the MD5x2 backend's `USE_AVX512_CRC` is true,
+/// else the SSE4.1+PCLMULQDQ baseline. The branch is on a const so it
+/// monomorphises away.
+///
+/// # Safety
+/// `src` must be valid for 64 readable bytes. Caller must have verified
+/// the relevant CPU features (the same set that gates selection of the
+/// MD5x2 backend `B`).
+#[inline(always)]
+unsafe fn crc_process_block(state: &mut crc_clmul::State, src: *const u8) {
+ if B::USE_AVX512_CRC {
+ crc_clmul_avx512::process_block(state, src)
+ } else {
+ crc_clmul::process_block(state, src)
+ }
+}
+
+/// Per-backend CRC `finish` dispatch — same const-branch pattern as
+/// [`crc_process_block`].
+///
+/// # Safety
+/// `src` must be valid for `len` (≤ 63) readable bytes. Caller must have
+/// verified the relevant CPU features.
+#[inline(always)]
+unsafe fn crc_finish(state: &mut crc_clmul::State, src: *const u8, len: usize) -> u32 {
+ if B::USE_AVX512_CRC {
+ crc_clmul_avx512::finish(state, src, len)
+ } else {
+ crc_clmul::finish(state, src, len)
+ }
+}
+
+/// Fused hasher state for one source file. Computes file-MD5
+/// continuously while emitting per-block (MD5, CRC32) pairs at every
+/// `get_block` call.
+///
+/// Mirrors upstream `HasherInput` (`hasher_input_base.h`). Generic over
+/// the MD5x2 backend `B`; the default is [`DefaultBackend`] (SSE2).
+pub struct HasherInput {
+ md5_state: B::State,
+ crc_state: crc_clmul::State,
+ tmp: [u8; 128],
+ tmp_len: usize,
+ pos_offset: usize,
+ data_len_block: u64,
+ data_len_file: u64,
+}
+
+/// Output of `get_block`: per-block MD5 and per-block CRC32 (IEEE).
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct BlockHash {
+ pub md5: [u8; 16],
+ pub crc32: u32,
+}
+
+impl HasherInput {
+ /// Create a new per-file hasher state. Equivalent to upstream
+ /// `HasherInput::HasherInput()` (which calls `reset()`).
+ #[inline]
+ pub fn new() -> Self {
+ let mut h = Self {
+ md5_state: B::init_state(),
+ crc_state: crc_clmul::State::new(),
+ tmp: [0; 128],
+ tmp_len: 0,
+ pos_offset: 0,
+ data_len_block: 0,
+ data_len_file: 0,
+ };
+ h.reset();
+ h
+ }
+
+ /// Reset to the post-construction state.
+ #[inline]
+ pub fn reset(&mut self) {
+ self.md5_state = B::init_state();
+ self.tmp_len = 0;
+ self.pos_offset = 0;
+ // SAFETY: x86_64 with sse2 always available; init touches state only.
+ unsafe { crc_clmul::init(&mut self.crc_state) };
+ self.data_len_block = 0;
+ self.data_len_file = 0;
+ }
+
+ /// Feed `data` into the hasher. Mirrors upstream
+ /// `HasherInput::update`. Maintains:
+ ///
+ /// * file-MD5 (lane 1) over the full byte stream.
+ /// * block-MD5 (lane 0) over bytes since last `get_block`.
+ /// * block-CRC32 over the same window as block-MD5.
+ pub fn update(&mut self, data: &[u8]) {
+ let len_in = data.len() as u64;
+ self.data_len_block += len_in;
+ self.data_len_file += len_in;
+
+ let mut data_ = data.as_ptr();
+ let mut len = data.len();
+
+ // ---------- Drain anything currently buffered in `tmp`. ----------
+ // Upstream loops 1-2 times here. The block lane lives at
+ // tmp[pos_offset..], the file lane at tmp[0..]. We only have
+ // useful work to do here if there's something staged; otherwise
+ // skip straight to the steady-state loop.
+ if self.tmp_len != 0 {
+ loop {
+ debug_assert!(self.tmp_len >= self.pos_offset);
+ // Bytes still wanted to complete the *block lane's* slot
+ // at tmp[pos_offset..pos_offset+64]. Equivalently, how
+ // many more bytes before tmp_len reaches 64+pos_offset.
+ let mut wanted = MD5_BLOCKSIZE + self.pos_offset - self.tmp_len;
+
+ if len < wanted {
+ // Not enough new data to complete a block. Stash and
+ // return — caller will feed more later.
+ unsafe {
+ core::ptr::copy_nonoverlapping(
+ data_,
+ self.tmp.as_mut_ptr().add(self.tmp_len),
+ len,
+ );
+ }
+ self.tmp_len += len;
+ return;
+ }
+
+ // We have enough to process one block.
+ //
+ // If tmp_len <= pos_offset, the block lane's slot
+ // starts past where tmp_len reaches — meaning the block
+ // hash will read entirely from incoming `data_`, not
+ // from `tmp`. Recompute `wanted` to fill the file
+ // lane's slot only (tmp[..64]); pull block-lane bytes
+ // straight from data_.
+ let block_data: *const u8 = if self.tmp_len <= self.pos_offset {
+ wanted = MD5_BLOCKSIZE - self.tmp_len;
+ data_
+ } else {
+ // Block lane fully in tmp.
+ unsafe { self.tmp.as_ptr().add(self.pos_offset) }
+ };
+
+ // Append the just-decided `wanted` bytes of new data
+ // into tmp at tmp_len so the file lane's tmp[..64] slot
+ // is complete by the time we call MD5x2.
+ unsafe {
+ core::ptr::copy_nonoverlapping(
+ data_,
+ self.tmp.as_mut_ptr().add(self.tmp_len),
+ wanted,
+ );
+
+ // Process the 64 B block: CRC over the block-lane
+ // window; MD5x2 with block_data on lane 0, tmp[0..64]
+ // on lane 1. (Note upstream argument order:
+ // md5_update_block_x2(state, block_data, tmp) — the
+ // FIRST data arg goes to lane HASH2X_BLOCK = 0 and
+ // the SECOND to HASH2X_FILE = 1.)
+ crc_process_block::(&mut self.crc_state, block_data);
+ B::process_block(&mut self.md5_state, block_data, self.tmp.as_ptr());
+ }
+ len -= wanted;
+ data_ = unsafe { data_.add(wanted) };
+
+ // If both halves came out of tmp, shift the file-lane
+ // half down (tmp[64..64+pos_offset] -> tmp[..pos_offset])
+ // so the next iteration sees a clean buffer with only
+ // the file-lane bytes carried over.
+ if self.tmp_len > self.pos_offset {
+ unsafe {
+ core::ptr::copy(
+ self.tmp.as_ptr().add(MD5_BLOCKSIZE),
+ self.tmp.as_mut_ptr(),
+ self.pos_offset,
+ );
+ }
+ self.tmp_len = self.pos_offset;
+ continue;
+ }
+ break;
+ }
+ // Note: we deliberately do NOT reset `self.tmp_len` here.
+ // Upstream leaves `tmpLen` stale at this point; the
+ // unconditional `tmp_len = len` write at the end of `update`
+ // overwrites it. Matches upstream hasher_input_base.h:75-87.
+ }
+
+ // ---------- Steady state: process full blocks straight from data. ----------
+ // Block lane: data_ + pos_offset; file lane: data_.
+ // Need at least 64 + pos_offset bytes to satisfy both lanes.
+ while len >= MD5_BLOCKSIZE + self.pos_offset {
+ unsafe {
+ let block_ptr = data_.add(self.pos_offset);
+ crc_process_block::(&mut self.crc_state, block_ptr);
+ B::process_block(&mut self.md5_state, block_ptr, data_);
+ }
+ data_ = unsafe { data_.add(MD5_BLOCKSIZE) };
+ len -= MD5_BLOCKSIZE;
+ }
+
+ // Stash the tail in tmp for next update / get_block / end.
+ unsafe {
+ core::ptr::copy_nonoverlapping(data_, self.tmp.as_mut_ptr(), len);
+ }
+ self.tmp_len = len;
+ }
+
+ /// Finalise the current block: emit (block-MD5, block-CRC32),
+ /// optionally extending it by `zero_pad` virtual zero bytes (PAR2
+ /// pads short last blocks up to the slice/block size). Then reset
+ /// the block lane + CRC for the next block; the file lane
+ /// continues unchanged.
+ ///
+ /// Mirrors upstream `HasherInput::getBlock`.
+ pub fn get_block(&mut self, zero_pad: u64) -> BlockHash {
+ // Extract block-MD5 lane state into 16 raw bytes, then run the
+ // upstream md5_final_block over tmp[pos_offset..pos_offset + (data_len_block & 63)],
+ // adding `zero_pad` virtual zero bytes after the real tail.
+ let mut md5_out = B::extract_lane(&self.md5_state, 0);
+ let block_tail_start = self.pos_offset;
+ let block_tail_len = (self.data_len_block & 63) as usize;
+ md5_final_block(
+ &mut md5_out,
+ &self.tmp[block_tail_start..block_tail_start + block_tail_len],
+ self.data_len_block,
+ zero_pad,
+ );
+
+ // CRC: finish over the same partial tail, then GF(2)-extend by
+ // zero_pad bytes via crc_zeropad.
+ let crc_partial = unsafe {
+ crc_finish::(
+ &mut self.crc_state,
+ self.tmp.as_ptr().add(block_tail_start),
+ block_tail_len,
+ )
+ };
+ let crc = crc_zero_pad(crc_partial, zero_pad);
+
+ // If tmp_len >= 64, the block-lane consumed less than the
+ // file-lane staged. Push that extra block through the file lane
+ // only (block lane will be reset immediately after, so its
+ // input value here is irrelevant — we can pass tmp for both).
+ if self.tmp_len >= MD5_BLOCKSIZE {
+ unsafe {
+ B::process_block(&mut self.md5_state, self.tmp.as_ptr(), self.tmp.as_ptr());
+ }
+ self.tmp_len -= MD5_BLOCKSIZE;
+ // Shift the carry down: tmp[64..64+tmp_len] -> tmp[..tmp_len].
+ unsafe {
+ core::ptr::copy(
+ self.tmp.as_ptr().add(MD5_BLOCKSIZE),
+ self.tmp.as_mut_ptr(),
+ self.tmp_len,
+ );
+ }
+ }
+
+ // Reset block lane MD5 + CRC; file lane keeps rolling.
+ B::init_lane(&mut self.md5_state, 0);
+ unsafe { crc_clmul::init(&mut self.crc_state) };
+ self.pos_offset = self.tmp_len;
+ self.data_len_block = 0;
+
+ BlockHash {
+ md5: md5_out,
+ crc32: crc,
+ }
+ }
+
+ /// Finalise the file-MD5 lane and return the 16-byte file-level MD5.
+ /// Mirrors upstream `HasherInput::end`. Consumes self because no
+ /// further operations are valid afterwards.
+ pub fn end(mut self) -> [u8; 16] {
+ // Defensive: usually getBlock already drained tmp_len < 64.
+ if self.tmp_len >= MD5_BLOCKSIZE {
+ unsafe {
+ B::process_block(&mut self.md5_state, self.tmp.as_ptr(), self.tmp.as_ptr());
+ }
+ // Upstream doesn't shift here because `end` won't iterate;
+ // the remaining tmp bytes for the file lane are always at
+ // offset 0 (file lane uses tmp[0..]) but bytes 64..tmp_len
+ // are leftover from the block lane's larger window. The
+ // file-lane finalise below uses `tmp[0..tmp_len & 63]` as
+ // its partial tail, but only the first `data_len_file & 63`
+ // bytes are meaningful (md5_final_block uses dataLen). We
+ // don't need to shift — the staging buffer already has the
+ // right file-lane bytes at offset 0. (Same as upstream:
+ // upstream's `end` just calls md5_final_block(md5, tmp,
+ // dataLen[FILE], 0) without any shift.)
+ }
+
+ let mut md5_out = B::extract_lane(&self.md5_state, 1);
+ let file_tail_len = (self.data_len_file & 63) as usize;
+ md5_final_block(
+ &mut md5_out,
+ &self.tmp[..file_tail_len],
+ self.data_len_file,
+ 0,
+ );
+ md5_out
+ }
+}
+
+impl Default for HasherInput {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+/// Pack a 16-byte MD5 state back into 4 little-endian state words.
+#[inline]
+fn pack_state(out: &mut [u32; 4], bytes: &[u8; 16]) {
+ for i in 0..4 {
+ out[i] = u32::from_le_bytes(bytes[i * 4..i * 4 + 4].try_into().unwrap());
+ }
+}
+
+// ----------------------------------------------------------------------
+// md5_final_block — direct port of parpar/hasher/md5-final.c.
+//
+// Take an in-progress MD5 state (16 raw bytes representing four LE u32
+// words), a partial tail (`tail_data`) of length `tail_data.len() < 64`
+// already absorbed into `total_length` but not yet pushed through the
+// compressor, the running total length (BYTES, not bits — same as
+// upstream), and a virtual zero-pad count. Append 0x80, zero-pad to 56
+// mod 64, write 64-bit LE bit length, run as many blocks as needed, and
+// write the final 16-byte digest back into `state`.
+// ----------------------------------------------------------------------
+fn md5_final_block(state: &mut [u8; 16], tail_data: &[u8], total_length: u64, zero_pad: u64) {
+ let mut block = [0u8; 64];
+ let mut remaining = (total_length & 63) as usize;
+ debug_assert_eq!(remaining, tail_data.len());
+ block[..remaining].copy_from_slice(tail_data);
+ // (block[remaining..] is already zero from initialiser.)
+
+ let total_length = total_length + zero_pad;
+ let mut zero_pad = zero_pad;
+ // loop_state mirrors upstream's funky state machine:
+ // 0 -> still draining a multi-block zero-pad with leftover real tail.
+ // 1 -> draining further full zero-pad blocks (block already zeroed).
+ // 2 -> place the 0x80 sentinel.
+ // 3 -> separator block emitted, need a follow-up length-only block.
+ // 4 -> final block (with 64-bit LE bit length at the end).
+ let mut loop_state: u8 = if (remaining as u64) + zero_pad < 64 {
+ 2
+ } else {
+ 0
+ };
+
+ let mut md5_words = [0u32; 4];
+ pack_state(&mut md5_words, state);
+
+ loop {
+ if loop_state == 1 && zero_pad < 64 {
+ loop_state = 2;
+ }
+ if loop_state == 2 {
+ remaining = (total_length & 63) as usize;
+ block[remaining] = 0x80;
+ remaining += 1;
+ if remaining <= 64 - 8 {
+ loop_state = 4;
+ } else {
+ loop_state = 3;
+ remaining = 0;
+ }
+ }
+ if loop_state == 4 {
+ for b in &mut block[remaining..64 - 8] {
+ *b = 0;
+ }
+ let bit_len = total_length.wrapping_shl(3);
+ block[56..64].copy_from_slice(&bit_len.to_le_bytes());
+ }
+
+ // Compress this 64-byte block into md5_words (single-lane MD5).
+ md5_compress_one(&mut md5_words, &block);
+
+ if loop_state == 4 {
+ break;
+ } else if loop_state == 3 {
+ loop_state = 4;
+ } else if loop_state == 1 {
+ zero_pad -= 64;
+ } else if loop_state == 0 {
+ // We just compressed a block whose contents were the real
+ // tail bytes followed by zeros (filling out the first
+ // partial block of the zero-pad). Subsequent blocks are
+ // pure zeros until we run out of zero_pad.
+ block.fill(0);
+ zero_pad -= 64 - (remaining as u64);
+ loop_state = 1;
+ }
+ }
+
+ // Pack words back to little-endian bytes.
+ for i in 0..4 {
+ state[i * 4..i * 4 + 4].copy_from_slice(&md5_words[i].to_le_bytes());
+ }
+}
+
+/// Single-block, single-lane MD5 compressor (RFC 1321 reference).
+/// Used only on the final 1-2 blocks per file/per get_block — not on
+/// the hot 64 B steady-state path. Kept here to avoid pulling in the
+/// `md-5` crate at runtime.
+fn md5_compress_one(state: &mut [u32; 4], block: &[u8; 64]) {
+ const K: [u32; 64] = [
+ 0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee, 0xf57c0faf, 0x4787c62a, 0xa8304613,
+ 0xfd469501, 0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be, 0x6b901122, 0xfd987193,
+ 0xa679438e, 0x49b40821, 0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa, 0xd62f105d,
+ 0x02441453, 0xd8a1e681, 0xe7d3fbc8, 0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed,
+ 0xa9e3e905, 0xfcefa3f8, 0x676f02d9, 0x8d2a4c8a, 0xfffa3942, 0x8771f681, 0x6d9d6122,
+ 0xfde5380c, 0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70, 0x289b7ec6, 0xeaa127fa,
+ 0xd4ef3085, 0x04881d05, 0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665, 0xf4292244,
+ 0x432aff97, 0xab9423a7, 0xfc93a039, 0x655b59c3, 0x8f0ccc92, 0xffeff47d, 0x85845dd1,
+ 0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1, 0xf7537e82, 0xbd3af235, 0x2ad7d2bb,
+ 0xeb86d391,
+ ];
+ const R: [u32; 64] = [
+ 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 5, 9, 14, 20, 5, 9, 14, 20, 5,
+ 9, 14, 20, 5, 9, 14, 20, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 6, 10,
+ 15, 21, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21,
+ ];
+ let mut m = [0u32; 16];
+ for i in 0..16 {
+ m[i] = u32::from_le_bytes(block[i * 4..i * 4 + 4].try_into().unwrap());
+ }
+ let mut a = state[0];
+ let mut b = state[1];
+ let mut c = state[2];
+ let mut d = state[3];
+ for i in 0..64 {
+ let (f, g) = match i {
+ 0..=15 => ((b & c) | (!b & d), i),
+ 16..=31 => ((d & b) | (!d & c), (5 * i + 1) % 16),
+ 32..=47 => (b ^ c ^ d, (3 * i + 5) % 16),
+ _ => (c ^ (b | !d), (7 * i) % 16),
+ };
+ let t = d;
+ d = c;
+ c = b;
+ b = b.wrapping_add(
+ a.wrapping_add(f)
+ .wrapping_add(K[i])
+ .wrapping_add(m[g])
+ .rotate_left(R[i]),
+ );
+ a = t;
+ }
+ state[0] = state[0].wrapping_add(a);
+ state[1] = state[1].wrapping_add(b);
+ state[2] = state[2].wrapping_add(c);
+ state[3] = state[3].wrapping_add(d);
+}
+
+// ----------------------------------------------------------------------
+// crc_zero_pad — direct port of parpar/hasher/crc_zeropad.c.
+// Multiply (NOT crc) by 2^(8 * zero_pad) over GF(2)/IEEE polynomial,
+// using the precomputed `crc_power` table.
+// ----------------------------------------------------------------------
+
+fn crc_multiply(mut a: u32, mut b: u32) -> u32 {
+ let mut res = 0u32;
+ for _ in 0..31 {
+ // mask = -((b >> 31) as i32) as u32 -> all-ones if top bit set, else 0
+ let mask = ((b >> 31) as i32).wrapping_neg() as u32;
+ res ^= mask & a;
+ // a = (a >> 1) ^ (0xEDB88320 & -(a & 1))
+ let a_lsb_mask = ((a & 1) as i32).wrapping_neg() as u32;
+ a = (a >> 1) ^ (0xEDB88320 & a_lsb_mask);
+ b = b.wrapping_shl(1);
+ }
+ let mask = ((b >> 31) as i32).wrapping_neg() as u32;
+ res ^= mask & a;
+ res
+}
+
+const CRC_POWER: [u32; 32] = [
+ 0x00800000, 0x00008000, 0xedb88320, 0xb1e6b092, 0xa06a2517, 0xed627dae, 0x88d14467, 0xd7bbfe6a,
+ 0xec447f11, 0x8e7ea170, 0x6427800e, 0x4d47bae0, 0x09fe548f, 0x83852d0f, 0x30362f1a, 0x7b5a9cc3,
+ 0x31fec169, 0x9fec022a, 0x6c8dedc4, 0x15d6874d, 0x5fde7a4e, 0xbad90e37, 0x2e4e5eef, 0x4eaba214,
+ 0xa8a472c0, 0x429a969e, 0x148d302a, 0xc40ba6d0, 0xc4e22c3c, 0x40000000, 0x20000000, 0x08000000,
+];
+
+fn crc_zero_pad(crc: u32, mut zero_pad: u64) -> u32 {
+ let mut crc = !crc;
+ let mut power: usize = 0;
+ while zero_pad != 0 {
+ if zero_pad & 1 != 0 {
+ crc = crc_multiply(crc, CRC_POWER[power]);
+ }
+ zero_pad >>= 1;
+ power = (power + 1) & 31;
+ }
+ !crc
+}
+
+// ----------------------------------------------------------------------
+// Tests
+// ----------------------------------------------------------------------
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::parpar_hasher::md5x2_bmi1::Bmi1;
+ use crate::parpar_hasher::md5x2_scalar::Scalar;
+ use crate::parpar_hasher::md5x2_sse2::Sse2;
+ use md5::Digest as _;
+ use md5::Md5;
+
+ /// Run a `drive(...)` invocation against every backend so every
+ /// functional test simultaneously exercises Scalar, SSE2, and (when
+ /// the host CPU supports it) BMI1. Any divergence trips here before
+ /// it reaches production callers.
+ macro_rules! run_both {
+ ($data:expr, $bs:expr, $chunks:expr) => {{
+ drive::($data, $bs, $chunks);
+ drive::($data, $bs, $chunks);
+ if std::is_x86_feature_detected!("bmi1") {
+ drive::($data, $bs, $chunks);
+ }
+ }};
+ }
+
+ /// Naive 3-pass reference: returns (file_md5, [(block_md5, block_crc), ...]).
+ fn reference(data: &[u8], block_size: usize) -> ([u8; 16], Vec<([u8; 16], u32)>) {
+ let mut file = Md5::new();
+ file.update(data);
+ let file_md5: [u8; 16] = file.finalize().into();
+
+ let mut blocks = Vec::new();
+ let mut off = 0;
+ while off < data.len() {
+ let end = (off + block_size).min(data.len());
+ let real = &data[off..end];
+ let mut bm = Md5::new();
+ bm.update(real);
+ // Zero-pad short last block to block_size.
+ let pad = block_size - real.len();
+ if pad > 0 {
+ let zeros = vec![0u8; pad];
+ bm.update(&zeros);
+ }
+ let bmd5: [u8; 16] = bm.finalize().into();
+ let mut bc = crc32fast::Hasher::new();
+ bc.update(real);
+ if pad > 0 {
+ let zeros = vec![0u8; pad];
+ bc.update(&zeros);
+ }
+ let bcrc = bc.finalize();
+ blocks.push((bmd5, bcrc));
+ off = end;
+ }
+ (file_md5, blocks)
+ }
+
+ fn synth(len: usize, seed: u8) -> Vec {
+ (0..len)
+ .map(|i| (i as u8).wrapping_mul(31).wrapping_add(seed))
+ .collect()
+ }
+
+ /// Drive a `HasherInput` through a stream split into arbitrary
+ /// chunks, then compare its outputs to the naive reference.
+ fn drive(data: &[u8], block_size: usize, chunks: &[usize]) {
+ let mut h: HasherInput = HasherInput::new();
+ let mut got_blocks = Vec::new();
+ let mut written_in_block = 0usize;
+
+ // Walk `data` in irregular chunks (for `update`), and call
+ // `get_block` whenever we cross a block boundary.
+ let mut cursor = 0;
+ let mut chunk_idx = 0usize;
+
+ while cursor < data.len() {
+ let chunk_size = if chunks.is_empty() {
+ data.len() - cursor
+ } else {
+ chunks[chunk_idx % chunks.len()]
+ };
+ let mut piece_size = chunk_size.min(data.len() - cursor);
+
+ // Don't cross a block boundary in a single update — split.
+ let block_remaining = block_size - written_in_block;
+ if piece_size >= block_remaining {
+ piece_size = block_remaining;
+ }
+ h.update(&data[cursor..cursor + piece_size]);
+ cursor += piece_size;
+ written_in_block += piece_size;
+ if written_in_block == block_size {
+ let bh = h.get_block(0);
+ got_blocks.push((bh.md5, bh.crc32));
+ written_in_block = 0;
+ }
+ chunk_idx += 1;
+ }
+ // Tail block: may be short — call get_block with zero-pad to
+ // match the reference's "pad to block_size".
+ if written_in_block > 0 {
+ let pad = (block_size - written_in_block) as u64;
+ let bh = h.get_block(pad);
+ got_blocks.push((bh.md5, bh.crc32));
+ }
+
+ let file_md5 = h.end();
+ let (ref_file, ref_blocks) = reference(data, block_size);
+ assert_eq!(file_md5, ref_file, "file MD5 mismatch");
+ assert_eq!(got_blocks.len(), ref_blocks.len(), "block count mismatch");
+ for (i, (g, r)) in got_blocks.iter().zip(ref_blocks.iter()).enumerate() {
+ assert_eq!(g.0, r.0, "block {i} MD5 mismatch");
+ assert_eq!(g.1, r.1, "block {i} CRC mismatch");
+ }
+ }
+
+ #[test]
+ fn empty() {
+ for backend in ["scalar", "sse2"] {
+ let file = match backend {
+ "scalar" => HasherInput::::new().end(),
+ _ => HasherInput::::new().end(),
+ };
+ let mut ref_md5 = Md5::new();
+ ref_md5.update([]);
+ let ref_md5: [u8; 16] = ref_md5.finalize().into();
+ assert_eq!(file, ref_md5, "{backend} empty file MD5 mismatch");
+ }
+ }
+
+ #[test]
+ fn single_block_aligned() {
+ // 1 block exactly, no zero-pad.
+ let data = synth(4096, 7);
+ run_both!(&data, 4096, &[]);
+ }
+
+ #[test]
+ fn multiple_blocks_aligned() {
+ let data = synth(4096 * 5, 11);
+ run_both!(&data, 4096, &[]);
+ }
+
+ #[test]
+ fn multiple_blocks_short_tail() {
+ // Last block is shorter than block_size; get_block must zero-pad.
+ let data = synth(4096 * 3 + 123, 17);
+ run_both!(&data, 4096, &[]);
+ }
+
+ #[test]
+ fn small_chunked_updates() {
+ let data = synth(4096 * 2 + 50, 23);
+ run_both!(&data, 4096, &[1, 7, 13, 64, 65, 100, 1024]);
+ }
+
+ #[test]
+ fn one_byte_at_a_time() {
+ let data = synth(4096 + 10, 41);
+ run_both!(&data, 4096, &[1]);
+ }
+
+ #[test]
+ fn chunks_crossing_blocks() {
+ // Big chunks that would naturally cross block boundaries — the
+ // drive helper splits them at the boundary, but each piece is
+ // still large multi-cache-line.
+ let data = synth(4096 * 4 + 7, 53);
+ run_both!(&data, 4096, &[1234, 5678, 9999]);
+ }
+
+ #[test]
+ fn small_block_size() {
+ // 64-byte blocks — exercises the staggered-offset code paths
+ // heavily (pos_offset cycles through values rapidly).
+ let data = synth(64 * 17 + 9, 67);
+ run_both!(&data, 64, &[1, 33, 65, 100]);
+ }
+
+ #[test]
+ fn unaligned_block_size() {
+ // 73-byte blocks (not multiple of 64) — pos_offset takes many
+ // distinct values, including the staggered carry case.
+ let data = synth(73 * 11 + 19, 71);
+ run_both!(&data, 73, &[1, 5, 60, 73, 200]);
+ }
+
+ #[test]
+ fn large_buffer() {
+ let data = synth(64 * 1024 + 17, 89);
+ run_both!(&data, 4096, &[64 * 1024]);
+ }
+}
diff --git a/src/parpar_hasher/hasher_input_arm64.rs b/src/parpar_hasher/hasher_input_arm64.rs
new file mode 100644
index 00000000..02832513
--- /dev/null
+++ b/src/parpar_hasher/hasher_input_arm64.rs
@@ -0,0 +1,370 @@
+// Copyright (C) Anomaly Industries Inc. and par2rs contributors.
+// SPDX-License-Identifier: GPL-2.0-or-later
+//
+// Portable fused per-file HasherInput driver — block-MD5 + file-MD5 + block-CRC32.
+//
+// Software fallback equivalent of hasher_input.rs, using:
+// * md5x2_scalar for both MD5 lanes (portable, always-available)
+// * crc32fast Hasher for streaming CRC32
+//
+// The staggered-offset algorithm, md5_final_block, and crc_zero_pad are
+// identical to the x86_64 version — only the CRC accumulator type differs.
+//
+// aarch64 uses this as its default implementation today. x86_64 also uses it
+// as the safe fallback when CLMUL CRC prerequisites are unavailable.
+
+use super::md5x2::Md5x2;
+use crc32fast::Hasher as CrcHasher;
+
+/// Default backend on aarch64 — portable scalar MD5x2.
+pub type DefaultBackend = super::md5x2_scalar::Scalar;
+
+const MD5_BLOCKSIZE: usize = 64;
+
+/// Per-block output: block MD5 and block CRC32.
+/// Field names match hasher_input.rs for architecture-agnostic consumers.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct BlockHash {
+ pub md5: [u8; 16],
+ pub crc32: u32,
+}
+
+/// Fused hasher state for one source file.
+pub struct HasherInput {
+ md5_state: B::State,
+ crc_hasher: CrcHasher,
+ tmp: [u8; 128],
+ tmp_len: usize,
+ pos_offset: usize,
+ data_len_block: u64,
+ data_len_file: u64,
+}
+
+impl HasherInput {
+ #[inline]
+ pub fn new() -> Self {
+ let mut h = Self {
+ md5_state: B::init_state(),
+ crc_hasher: CrcHasher::new(),
+ tmp: [0; 128],
+ tmp_len: 0,
+ pos_offset: 0,
+ data_len_block: 0,
+ data_len_file: 0,
+ };
+ h.reset();
+ h
+ }
+
+ #[inline]
+ pub fn reset(&mut self) {
+ self.md5_state = B::init_state();
+ self.crc_hasher = CrcHasher::new();
+ self.tmp_len = 0;
+ self.pos_offset = 0;
+ self.data_len_block = 0;
+ self.data_len_file = 0;
+ }
+
+ pub fn update(&mut self, data: &[u8]) {
+ let len_in = data.len() as u64;
+ self.data_len_block += len_in;
+ self.data_len_file += len_in;
+
+ let mut data_ = data.as_ptr();
+ let mut len = data.len();
+
+ // Drain buffered bytes in tmp.
+ if self.tmp_len != 0 {
+ loop {
+ debug_assert!(self.tmp_len >= self.pos_offset);
+ let mut wanted = MD5_BLOCKSIZE + self.pos_offset - self.tmp_len;
+
+ if len < wanted {
+ unsafe {
+ core::ptr::copy_nonoverlapping(
+ data_,
+ self.tmp.as_mut_ptr().add(self.tmp_len),
+ len,
+ );
+ }
+ self.tmp_len += len;
+ return;
+ }
+
+ let block_data: *const u8 = if self.tmp_len <= self.pos_offset {
+ wanted = MD5_BLOCKSIZE - self.tmp_len;
+ data_
+ } else {
+ unsafe { self.tmp.as_ptr().add(self.pos_offset) }
+ };
+
+ unsafe {
+ core::ptr::copy_nonoverlapping(
+ data_,
+ self.tmp.as_mut_ptr().add(self.tmp_len),
+ wanted,
+ );
+ let block_slice = core::slice::from_raw_parts(block_data, MD5_BLOCKSIZE);
+ self.crc_hasher.update(block_slice);
+ B::process_block(&mut self.md5_state, block_data, self.tmp.as_ptr());
+ }
+ len -= wanted;
+ data_ = unsafe { data_.add(wanted) };
+
+ if self.tmp_len > self.pos_offset {
+ unsafe {
+ core::ptr::copy(
+ self.tmp.as_ptr().add(MD5_BLOCKSIZE),
+ self.tmp.as_mut_ptr(),
+ self.pos_offset,
+ );
+ }
+ self.tmp_len = self.pos_offset;
+ continue;
+ }
+ break;
+ }
+ }
+
+ // Steady-state: process full blocks straight from data.
+ while len >= MD5_BLOCKSIZE + self.pos_offset {
+ unsafe {
+ let block_ptr = data_.add(self.pos_offset);
+ let block_slice = core::slice::from_raw_parts(block_ptr, MD5_BLOCKSIZE);
+ self.crc_hasher.update(block_slice);
+ B::process_block(&mut self.md5_state, block_ptr, data_);
+ }
+ data_ = unsafe { data_.add(MD5_BLOCKSIZE) };
+ len -= MD5_BLOCKSIZE;
+ }
+
+ // Stash the tail.
+ unsafe {
+ core::ptr::copy_nonoverlapping(data_, self.tmp.as_mut_ptr(), len);
+ }
+ self.tmp_len = len;
+ }
+
+ pub fn get_block(&mut self, zero_pad: u64) -> BlockHash {
+ let mut md5_out = B::extract_lane(&self.md5_state, 0);
+ let block_tail_start = self.pos_offset;
+ let block_tail_len = (self.data_len_block & 63) as usize;
+ md5_final_block(
+ &mut md5_out,
+ &self.tmp[block_tail_start..block_tail_start + block_tail_len],
+ self.data_len_block,
+ zero_pad,
+ );
+
+ // CRC: feed the partial tail, finalize, then GF(2)-extend for zero_pad.
+ let tail = &self.tmp[block_tail_start..block_tail_start + block_tail_len];
+ let mut crc_final = self.crc_hasher.clone();
+ crc_final.update(tail);
+ let crc_partial = crc_final.finalize();
+ let crc = crc_zero_pad(crc_partial, zero_pad);
+
+ if self.tmp_len >= MD5_BLOCKSIZE {
+ unsafe {
+ B::process_block(&mut self.md5_state, self.tmp.as_ptr(), self.tmp.as_ptr());
+ }
+ self.tmp_len -= MD5_BLOCKSIZE;
+ unsafe {
+ core::ptr::copy(
+ self.tmp.as_ptr().add(MD5_BLOCKSIZE),
+ self.tmp.as_mut_ptr(),
+ self.tmp_len,
+ );
+ }
+ }
+
+ B::init_lane(&mut self.md5_state, 0);
+ self.crc_hasher = CrcHasher::new(); // Reset CRC for next block
+ self.pos_offset = self.tmp_len;
+ self.data_len_block = 0;
+
+ BlockHash {
+ md5: md5_out,
+ crc32: crc,
+ }
+ }
+
+ pub fn end(mut self) -> [u8; 16] {
+ if self.tmp_len >= MD5_BLOCKSIZE {
+ unsafe {
+ B::process_block(&mut self.md5_state, self.tmp.as_ptr(), self.tmp.as_ptr());
+ }
+ }
+ let mut md5_out = B::extract_lane(&self.md5_state, 1);
+ let file_tail_len = (self.data_len_file & 63) as usize;
+ md5_final_block(
+ &mut md5_out,
+ &self.tmp[..file_tail_len],
+ self.data_len_file,
+ 0,
+ );
+ md5_out
+ }
+}
+
+impl Default for HasherInput {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+// ============================================================================
+// md5_final_block, md5_compress_one, crc_zero_pad
+// (Copied from hasher_input.rs — platform-independent pure Rust)
+// ============================================================================
+
+fn md5_final_block(state: &mut [u8; 16], tail_data: &[u8], total_length: u64, zero_pad: u64) {
+ let mut block = [0u8; 64];
+ let mut remaining = (total_length & 63) as usize;
+ debug_assert_eq!(remaining, tail_data.len());
+ block[..remaining].copy_from_slice(tail_data);
+
+ let total_length = total_length + zero_pad;
+ let mut zero_pad = zero_pad;
+ let mut loop_state: u8 = if (remaining as u64) + zero_pad < 64 {
+ 2
+ } else {
+ 0
+ };
+
+ let mut md5_words = [0u32; 4];
+ pack_state(&mut md5_words, state);
+
+ loop {
+ if loop_state == 1 && zero_pad < 64 {
+ loop_state = 2;
+ }
+ if loop_state == 2 {
+ remaining = (total_length & 63) as usize;
+ block[remaining] = 0x80;
+ remaining += 1;
+ if remaining <= 64 - 8 {
+ loop_state = 4;
+ } else {
+ loop_state = 3;
+ remaining = 0;
+ }
+ }
+ if loop_state == 4 {
+ for b in &mut block[remaining..64 - 8] {
+ *b = 0;
+ }
+ let bit_len = total_length.wrapping_shl(3);
+ block[56..64].copy_from_slice(&bit_len.to_le_bytes());
+ }
+
+ md5_compress_one(&mut md5_words, &block);
+
+ if loop_state == 4 {
+ break;
+ } else if loop_state == 3 {
+ loop_state = 4;
+ } else if loop_state == 1 {
+ zero_pad -= 64;
+ } else if loop_state == 0 {
+ block.fill(0);
+ zero_pad -= 64 - (remaining as u64);
+ loop_state = 1;
+ }
+ }
+
+ for i in 0..4 {
+ state[i * 4..i * 4 + 4].copy_from_slice(&md5_words[i].to_le_bytes());
+ }
+}
+
+#[inline]
+fn pack_state(out: &mut [u32; 4], bytes: &[u8; 16]) {
+ for i in 0..4 {
+ out[i] = u32::from_le_bytes(bytes[i * 4..i * 4 + 4].try_into().unwrap());
+ }
+}
+
+fn md5_compress_one(state: &mut [u32; 4], block: &[u8; 64]) {
+ const K: [u32; 64] = [
+ 0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee, 0xf57c0faf, 0x4787c62a, 0xa8304613,
+ 0xfd469501, 0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be, 0x6b901122, 0xfd987193,
+ 0xa679438e, 0x49b40821, 0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa, 0xd62f105d,
+ 0x02441453, 0xd8a1e681, 0xe7d3fbc8, 0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed,
+ 0xa9e3e905, 0xfcefa3f8, 0x676f02d9, 0x8d2a4c8a, 0xfffa3942, 0x8771f681, 0x6d9d6122,
+ 0xfde5380c, 0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70, 0x289b7ec6, 0xeaa127fa,
+ 0xd4ef3085, 0x04881d05, 0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665, 0xf4292244,
+ 0x432aff97, 0xab9423a7, 0xfc93a039, 0x655b59c3, 0x8f0ccc92, 0xffeff47d, 0x85845dd1,
+ 0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1, 0xf7537e82, 0xbd3af235, 0x2ad7d2bb,
+ 0xeb86d391,
+ ];
+ const R: [u32; 64] = [
+ 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 5, 9, 14, 20, 5, 9, 14, 20, 5,
+ 9, 14, 20, 5, 9, 14, 20, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 6, 10,
+ 15, 21, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21,
+ ];
+ let mut m = [0u32; 16];
+ for i in 0..16 {
+ m[i] = u32::from_le_bytes(block[i * 4..i * 4 + 4].try_into().unwrap());
+ }
+ let mut a = state[0];
+ let mut b = state[1];
+ let mut c = state[2];
+ let mut d = state[3];
+ for i in 0..64 {
+ let (f, g) = match i {
+ 0..=15 => ((b & c) | (!b & d), i),
+ 16..=31 => ((d & b) | (!d & c), (5 * i + 1) % 16),
+ 32..=47 => (b ^ c ^ d, (3 * i + 5) % 16),
+ _ => (c ^ (b | !d), (7 * i) % 16),
+ };
+ let t = d;
+ d = c;
+ c = b;
+ b = b.wrapping_add(
+ a.wrapping_add(f)
+ .wrapping_add(K[i])
+ .wrapping_add(m[g])
+ .rotate_left(R[i]),
+ );
+ a = t;
+ }
+ state[0] = state[0].wrapping_add(a);
+ state[1] = state[1].wrapping_add(b);
+ state[2] = state[2].wrapping_add(c);
+ state[3] = state[3].wrapping_add(d);
+}
+
+fn crc_multiply(mut a: u32, mut b: u32) -> u32 {
+ let mut res = 0u32;
+ for _ in 0..31 {
+ let mask = ((b >> 31) as i32).wrapping_neg() as u32;
+ res ^= mask & a;
+ let a_lsb_mask = ((a & 1) as i32).wrapping_neg() as u32;
+ a = (a >> 1) ^ (0xEDB88320 & a_lsb_mask);
+ b = b.wrapping_shl(1);
+ }
+ let mask = ((b >> 31) as i32).wrapping_neg() as u32;
+ res ^= mask & a;
+ res
+}
+
+const CRC_POWER: [u32; 32] = [
+ 0x00800000, 0x00008000, 0xedb88320, 0xb1e6b092, 0xa06a2517, 0xed627dae, 0x88d14467, 0xd7bbfe6a,
+ 0xec447f11, 0x8e7ea170, 0x6427800e, 0x4d47bae0, 0x09fe548f, 0x83852d0f, 0x30362f1a, 0x7b5a9cc3,
+ 0x31fec169, 0x9fec022a, 0x6c8dedc4, 0x15d6874d, 0x5fde7a4e, 0xbad90e37, 0x2e4e5eef, 0x4eaba214,
+ 0xa8a472c0, 0x429a969e, 0x148d302a, 0xc40ba6d0, 0xc4e22c3c, 0x40000000, 0x20000000, 0x08000000,
+];
+
+fn crc_zero_pad(crc: u32, mut zero_pad: u64) -> u32 {
+ let mut crc = !crc;
+ let mut power: usize = 0;
+ while zero_pad != 0 {
+ if zero_pad & 1 != 0 {
+ crc = crc_multiply(crc, CRC_POWER[power]);
+ }
+ zero_pad >>= 1;
+ power = (power + 1) & 31;
+ }
+ !crc
+}
diff --git a/src/parpar_hasher/hasher_input_dyn.rs b/src/parpar_hasher/hasher_input_dyn.rs
new file mode 100644
index 00000000..6005e82d
--- /dev/null
+++ b/src/parpar_hasher/hasher_input_dyn.rs
@@ -0,0 +1,379 @@
+//! Runtime-CPU-dispatched `HasherInput`.
+//!
+//! Mirrors par2cmdline-turbo's `HasherInput_Create()` factory — picks the
+//! best concrete `HasherInput` for the CPU at construction time
+//! and delegates every method to it.
+//!
+//! This dispatcher is architecture-agnostic; selection logic branches on
+//! `target_arch`. x86_64 dispatches to Scalar/SSE2/BMI1/AVX-512; aarch64
+//! dispatches to Scalar (with NEON and ARM CRC placeholders for future).
+
+#[cfg(target_arch = "x86_64")]
+use super::hasher_input::{BlockHash, HasherInput};
+
+#[cfg(target_arch = "aarch64")]
+use super::hasher_input_portable::{BlockHash, HasherInput};
+
+// ============================================================================
+// x86_64 Dispatcher
+// ============================================================================
+
+#[cfg(target_arch = "x86_64")]
+pub enum HasherInputDyn {
+ Scalar(HasherInput),
+ Sse2(HasherInput),
+ Bmi1(HasherInput),
+ Avx512(HasherInput),
+ Portable(super::hasher_input_portable::HasherInput),
+}
+
+#[cfg(target_arch = "x86_64")]
+impl HasherInputDyn {
+ /// Pick the best backend for the current CPU and construct it.
+ ///
+ /// Selection (preferred → fallback):
+ /// 1. **AVX-512VL** — when avx512f+vl+pclmulqdq+sse4.1 are present
+ /// AND the CPU is NOT an AMD Zen4/Zen5 part.
+ /// 2. **BMI1** — fast non-AVX-512 fallback when the CLMUL CRC path is available.
+ /// 3. **Scalar** — scalar MD5x2 + CLMUL CRC when `sse4.1` + `pclmulqdq` are available.
+ /// 4. **Portable** — scalar MD5x2 + software CRC32 fallback.
+ pub fn new() -> Self {
+ if Self::should_use_avx512() {
+ HasherInputDyn::Avx512(HasherInput::new())
+ } else if Self::bmi1_crc_supported() {
+ HasherInputDyn::Bmi1(HasherInput::new())
+ } else if Self::scalar_crc_supported() {
+ HasherInputDyn::Scalar(HasherInput::new())
+ } else {
+ HasherInputDyn::Portable(super::hasher_input_portable::HasherInput::new())
+ }
+ }
+
+ fn clmul_crc_supported() -> bool {
+ is_x86_feature_detected!("pclmulqdq") && is_x86_feature_detected!("sse4.1")
+ }
+
+ fn scalar_crc_supported() -> bool {
+ Self::clmul_crc_supported()
+ }
+
+ fn sse2_crc_supported() -> bool {
+ Self::clmul_crc_supported()
+ }
+
+ fn bmi1_crc_supported() -> bool {
+ Self::clmul_crc_supported() && is_x86_feature_detected!("bmi1")
+ }
+
+ fn avx512_supported() -> bool {
+ is_x86_feature_detected!("avx512f")
+ && is_x86_feature_detected!("avx512vl")
+ && is_x86_feature_detected!("pclmulqdq")
+ && is_x86_feature_detected!("sse4.1")
+ }
+
+ fn should_use_avx512() -> bool {
+ Self::avx512_supported() && !is_amd_vec_rot_slow()
+ }
+
+ pub fn new_scalar() -> Self {
+ if Self::scalar_crc_supported() {
+ HasherInputDyn::Scalar(HasherInput::new())
+ } else {
+ HasherInputDyn::Portable(super::hasher_input_portable::HasherInput::new())
+ }
+ }
+
+ pub fn new_sse2() -> Self {
+ assert!(
+ Self::sse2_crc_supported(),
+ "SSE2 HasherInput requires sse4.1 + pclmulqdq CRC support"
+ );
+ HasherInputDyn::Sse2(HasherInput::new())
+ }
+
+ pub fn new_bmi1() -> Self {
+ assert!(
+ Self::bmi1_crc_supported(),
+ "BMI1 HasherInput requires bmi1 + sse4.1 + pclmulqdq support"
+ );
+ HasherInputDyn::Bmi1(HasherInput::new())
+ }
+
+ pub unsafe fn new_avx512() -> Self {
+ HasherInputDyn::Avx512(HasherInput::new())
+ }
+
+ pub fn try_new_avx512() -> Option {
+ if Self::avx512_supported() {
+ Some(unsafe { Self::new_avx512() })
+ } else {
+ None
+ }
+ }
+
+ pub fn backend_name(&self) -> &'static str {
+ match self {
+ HasherInputDyn::Scalar(_) => "scalar",
+ HasherInputDyn::Sse2(_) => "sse2",
+ HasherInputDyn::Bmi1(_) => "bmi1",
+ HasherInputDyn::Avx512(_) => "avx512",
+ HasherInputDyn::Portable(_) => "portable",
+ }
+ }
+
+ #[inline]
+ pub fn update(&mut self, data: &[u8]) {
+ match self {
+ HasherInputDyn::Scalar(h) => h.update(data),
+ HasherInputDyn::Sse2(h) => h.update(data),
+ HasherInputDyn::Bmi1(h) => h.update(data),
+ HasherInputDyn::Avx512(h) => h.update(data),
+ HasherInputDyn::Portable(h) => h.update(data),
+ }
+ }
+
+ #[inline]
+ pub fn get_block(&mut self, zero_pad: u64) -> BlockHash {
+ match self {
+ HasherInputDyn::Scalar(h) => h.get_block(zero_pad),
+ HasherInputDyn::Sse2(h) => h.get_block(zero_pad),
+ HasherInputDyn::Bmi1(h) => h.get_block(zero_pad),
+ HasherInputDyn::Avx512(h) => h.get_block(zero_pad),
+ HasherInputDyn::Portable(h) => {
+ let block = h.get_block(zero_pad);
+ BlockHash {
+ md5: block.md5,
+ crc32: block.crc32,
+ }
+ }
+ }
+ }
+
+ #[inline]
+ pub fn end(self) -> [u8; 16] {
+ match self {
+ HasherInputDyn::Scalar(h) => h.end(),
+ HasherInputDyn::Sse2(h) => h.end(),
+ HasherInputDyn::Bmi1(h) => h.end(),
+ HasherInputDyn::Avx512(h) => h.end(),
+ HasherInputDyn::Portable(h) => h.end(),
+ }
+ }
+}
+
+#[cfg(target_arch = "x86_64")]
+#[allow(unused_unsafe)]
+fn is_amd_vec_rot_slow() -> bool {
+ use core::arch::x86_64::__cpuid;
+ unsafe {
+ let v = __cpuid(0);
+ let vendor = [
+ v.ebx.to_le_bytes(),
+ v.edx.to_le_bytes(),
+ v.ecx.to_le_bytes(),
+ ];
+ let vendor_flat: [u8; 12] = [
+ vendor[0][0],
+ vendor[0][1],
+ vendor[0][2],
+ vendor[0][3],
+ vendor[1][0],
+ vendor[1][1],
+ vendor[1][2],
+ vendor[1][3],
+ vendor[2][0],
+ vendor[2][1],
+ vendor[2][2],
+ vendor[2][3],
+ ];
+ if &vendor_flat != b"AuthenticAMD" {
+ return false;
+ }
+ let eax = __cpuid(1).eax;
+ let base_family = ((eax >> 8) & 0xF) as u32;
+ let ext_family = ((eax >> 20) & 0xFF) as u32;
+ let family = base_family + ext_family;
+ matches!(family, 0x19 | 0x1A)
+ }
+}
+
+// ============================================================================
+// aarch64 Dispatcher
+// ============================================================================
+
+#[cfg(target_arch = "aarch64")]
+pub enum HasherInputDyn {
+ Scalar(HasherInput),
+ // TODO: Neon(HasherInput),
+ // TODO: ArmCrc(HasherInput),
+}
+
+#[cfg(target_arch = "aarch64")]
+impl HasherInputDyn {
+ /// Pick the best backend for the current CPU and construct it.
+ ///
+ /// For now, always uses Scalar (portable, always-available).
+ /// Future: Will dispatch to NEON when available.
+ pub fn new() -> Self {
+ HasherInputDyn::Scalar(HasherInput::new())
+ }
+
+ pub fn update(&mut self, data: &[u8]) {
+ match self {
+ HasherInputDyn::Scalar(h) => h.update(data),
+ }
+ }
+
+ pub fn get_block(&mut self, zero_pad: u64) -> BlockHash {
+ match self {
+ HasherInputDyn::Scalar(h) => h.get_block(zero_pad),
+ }
+ }
+
+ pub fn end(self) -> [u8; 16] {
+ match self {
+ HasherInputDyn::Scalar(h) => h.end(),
+ }
+ }
+
+ pub fn backend_name(&self) -> &'static str {
+ match self {
+ HasherInputDyn::Scalar(_) => "scalar",
+ }
+ }
+}
+
+// ============================================================================
+// Shared Implementations
+// ============================================================================
+
+impl Default for HasherInputDyn {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn picks_a_backend() {
+ let h = HasherInputDyn::new();
+ let name = h.backend_name();
+ #[cfg(target_arch = "x86_64")]
+ assert!(matches!(
+ name,
+ "scalar" | "sse2" | "bmi1" | "avx512" | "portable"
+ ));
+ #[cfg(target_arch = "aarch64")]
+ assert!(matches!(name, "scalar")); // or "neon" when ported
+ }
+
+ #[test]
+ fn dyn_matches_scalar() {
+ let data: Vec = (0..4096u32).map(|i| (i * 31 + 7) as u8).collect();
+ let block_size = 1024usize;
+
+ let mut a = HasherInputDyn::new();
+ #[cfg(target_arch = "x86_64")]
+ let mut b = HasherInputDyn::new_scalar();
+ #[cfg(target_arch = "aarch64")]
+ let mut b = HasherInputDyn::new();
+
+ let mut a_blocks = Vec::new();
+ let mut b_blocks = Vec::new();
+ for chunk in data.chunks(block_size) {
+ a.update(chunk);
+ b.update(chunk);
+ a_blocks.push(a.get_block(0));
+ b_blocks.push(b.get_block(0));
+ }
+ assert_eq!(a_blocks, b_blocks);
+ assert_eq!(a.end(), b.end());
+ }
+
+ #[test]
+ fn zero_pad_behavior() {
+ let block_size = 64usize;
+ let mut data = vec![0u8; 100];
+ for (i, b) in data.iter_mut().enumerate() {
+ *b = (i as u8).wrapping_mul(13);
+ }
+
+ let mut h = HasherInputDyn::new();
+ h.update(&data[..block_size]);
+ let _b0 = h.get_block(0);
+ h.update(&data[block_size..]);
+ let _b1 = h.get_block((block_size - (data.len() - block_size)) as u64);
+ let file_md5 = h.end();
+
+ use md5::{Digest, Md5};
+ let mut m = Md5::new();
+ m.update(&data);
+ let expected = m.finalize();
+ assert_eq!(&file_md5[..], &expected[..]);
+ }
+
+ #[test]
+ fn default_uses_the_same_backend_family_as_new() {
+ assert_eq!(
+ HasherInputDyn::default().backend_name(),
+ HasherInputDyn::new().backend_name()
+ );
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn scalar_and_sse2_backends_match_each_other() {
+ let data: Vec = (0..2048u32).map(|i| (i * 17 + 3) as u8).collect();
+
+ let mut scalar = HasherInputDyn::new_scalar();
+ let mut sse2 = HasherInputDyn::new_sse2();
+
+ scalar.update(&data[..1024]);
+ sse2.update(&data[..1024]);
+ assert_eq!(scalar.get_block(0), sse2.get_block(0));
+
+ scalar.update(&data[1024..]);
+ sse2.update(&data[1024..]);
+ assert_eq!(scalar.get_block(0), sse2.get_block(0));
+ assert_eq!(scalar.end(), sse2.end());
+ assert_eq!(HasherInputDyn::new_scalar().backend_name(), "scalar");
+ assert_eq!(HasherInputDyn::new_sse2().backend_name(), "sse2");
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn optional_x86_backends_are_guarded_by_runtime_feature_checks() {
+ if is_x86_feature_detected!("bmi1") {
+ let mut bmi1 = HasherInputDyn::new_bmi1();
+ bmi1.update(b"hello world");
+ let _ = bmi1.get_block(0);
+ assert_eq!(bmi1.end().len(), 16);
+ assert_eq!(HasherInputDyn::new_bmi1().backend_name(), "bmi1");
+ }
+
+ let maybe_avx512 = HasherInputDyn::try_new_avx512();
+ assert_eq!(maybe_avx512.is_some(), HasherInputDyn::avx512_supported());
+ if let Some(mut avx512) = maybe_avx512 {
+ avx512.update(b"hello world");
+ let _ = avx512.get_block(0);
+ assert_eq!(avx512.end().len(), 16);
+ assert_eq!(
+ unsafe { HasherInputDyn::new_avx512() }.backend_name(),
+ "avx512"
+ );
+ }
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn amd_vector_rotate_probe_returns_a_stable_boolean() {
+ let first = is_amd_vec_rot_slow();
+ let second = is_amd_vec_rot_slow();
+
+ assert_eq!(first, second);
+ }
+}
diff --git a/src/parpar_hasher/md5x2.rs b/src/parpar_hasher/md5x2.rs
new file mode 100644
index 00000000..5620ef8a
--- /dev/null
+++ b/src/parpar_hasher/md5x2.rs
@@ -0,0 +1,67 @@
+// Copyright (C) Anomaly Industries Inc. and par2rs contributors.
+// SPDX-License-Identifier: GPL-2.0-or-later
+//
+// MD5x2 backend trait — the contract every two-lane MD5 implementation
+// implements so `HasherInput` can be generic over the SIMD/scalar tier
+// chosen at compile or run time.
+//
+// Upstream's equivalent is the macro `_FN(md5_*)` machinery in
+// `parpar/hasher/md5x2-base.h`, which textually re-substitutes the
+// active backend's name into the round bodies. Rust uses a trait + a
+// concrete impl per backend, which gives us monomorphisation without
+// repeating the driver code per backend.
+
+/// Two-lane MD5 block compressor.
+///
+/// The state type is an associated type because each backend chooses
+/// the layout that's most natural for its instruction set:
+///
+/// * scalar uses `[u32; 8]` (lanes laid out flat: `[a0,b0,c0,d0,a1,b1,c1,d1]`).
+/// * SSE2 uses `[__m128i; 4]` with each register holding both lanes'
+/// matching state word in the funky `[lane0, GARBAGE, lane1, GARBAGE]`
+/// layout from `md5x2-sse.h`.
+///
+/// Backends keep the state in the form they prefer, and only convert
+/// at the boundaries the driver actually observes (`init_state`,
+/// `init_lane`, `extract_lane`).
+pub trait Md5x2 {
+ /// Backend-private state representation.
+ type State;
+
+ /// Whether this backend's host CPU also supports the AVX-512VL
+ /// `vpternlogd` form of the CLMul CRC32 fold (`crc_clmul_avx512`).
+ ///
+ /// Default `false` — the SSE4.1 + PCLMULQDQ baseline is always safe.
+ /// Backends that already require AVX-512VL (e.g. the AVX-512 MD5x2
+ /// path) override this to `true` so `HasherInput` picks the
+ /// `vpternlogd`-collapsed fold without needing a second generic
+ /// parameter. Mirrors upstream's coupling: the `_CRC_USE_AVX512_`
+ /// switch in `crc_clmul.h` is gated by the same CPU feature set as
+ /// the AVX-512 MD5 path, so they're never selected independently.
+ const USE_AVX512_CRC: bool = false;
+
+ /// Build a fresh state with both lanes initialised to the standard
+ /// MD5 IV.
+ fn init_state() -> Self::State;
+
+ /// Reset just one of the two lanes to the standard MD5 IV. Used by
+ /// `HasherInput::get_block` to restart the block-MD5 lane while the
+ /// file-MD5 lane keeps accumulating.
+ fn init_lane(state: &mut Self::State, lane: usize);
+
+ /// Read one lane's current state out as a 16-byte little-endian
+ /// MD5 digest. This is the canonical wire form the finalizer
+ /// (`md5_final_block`) consumes.
+ fn extract_lane(state: &Self::State, lane: usize) -> [u8; 16];
+
+ /// Compress one 64-byte block on each lane.
+ ///
+ /// `data1` feeds lane 0 (the block-MD5 lane in `HasherInput`).
+ /// `data2` feeds lane 1 (the file-MD5 lane).
+ ///
+ /// # Safety
+ /// Both pointers must be valid for 64-byte reads. They may overlap
+ /// (the upstream API is happy with `data1 == data2`, which happens
+ /// on the very first block of every source file).
+ unsafe fn process_block(state: &mut Self::State, data1: *const u8, data2: *const u8);
+}
diff --git a/src/parpar_hasher/md5x2_avx512.rs b/src/parpar_hasher/md5x2_avx512.rs
new file mode 100644
index 00000000..712eef31
--- /dev/null
+++ b/src/parpar_hasher/md5x2_avx512.rs
@@ -0,0 +1,391 @@
+// Copyright (C) Anomaly Industries Inc. and par2rs contributors.
+// SPDX-License-Identifier: GPL-2.0-or-later
+//
+// AVX-512VL two-lane MD5 block compression for x86_64.
+//
+// 1:1 Rust port of the `__AVX512VL__` specialisation produced by:
+// parpar/hasher/md5x2-sse.h lines ~109-134 (`#ifdef __AVX512VL__` block)
+// parpar/hasher/md5x2-base.h (round expansion machinery — unchanged)
+// parpar/hasher/md5-base.h (the actual 64-round body — unchanged)
+// from par2cmdline-turbo (https://github.com/animetosho/par2cmdline-turbo)
+// and upstream ParPar (https://github.com/animetosho/ParPar), both
+// GPL-2.0-or-later.
+//
+// ## What's different vs the SSE2 backend
+//
+// **State layout: identical.** Upstream aliases
+// #define md5_extract_x2_avx512 md5_extract_x2_sse
+// so the lane layout in xmm registers (`[lane0_word, GARBAGE,
+// lane1_word, GARBAGE]`) is shared verbatim. We re-use `State`,
+// `from_flat`, `to_flat`, and `load4` from `md5x2_sse2` so any future
+// layout fix lands in both backends at once.
+//
+// **Bool functions: vpternlogd.** AVX-512VL's `vpternlogd` collapses
+// any 3-input boolean expression into one instruction. Upstream
+// (md5x2-sse.h:122-128 in the `__AVX512VL__` block) redefines:
+//
+// #define F(b,c,d) _mm_ternarylogic_epi32(d,c,b,0xD8)
+// #define G(b,c,d) _mm_ternarylogic_epi32(d,c,b,0xAC)
+// #define H(b,c,d) _mm_ternarylogic_epi32(d,c,b,0x96)
+// #define I(b,c,d) _mm_ternarylogic_epi32(d,c,b,0x63)
+//
+// Note ADDF is *not* defined in this block, so md5-base.h's fallback
+// `_ADDF(f,a,b,c,d) = ADD(a, f(b,c,d))` is used — i.e. round step is
+// a = a + ik
+// a = a + f(b,c,d) // single vpternlogd then vpaddd
+// a = ROL(a, r)
+// a = a + b
+//
+// Truth-table sanity check (vpternlogd indexes by `(d_bit<<2)|(c_bit<<1)|b_bit`
+// and the 8-bit IMM is the lookup table indexed MSB→LSB):
+//
+// F = ((c ^ d) & b) ^ d → indices 011,100,110,111 are 1 → 0xD8 ✓
+// H = b ^ c ^ d → indices 001,010,100,111 are 1 → 0x96 ✓
+// G = (~d & c) | (d & b) (eq. (~d & c) + (d & b) since disjoint)
+// → indices 010,011,101,111 are 1 → 0xAC ✓
+// I = (~d | b) ^ c → indices 000,001,011,101,110 are 1 → 0x63 ✓
+//
+// **Rotate: vprold.** `_mm_rol_epi32::(a)` is a real per-dword
+// 32-bit rotate, so the SSE2 shuffle+srli_epi64 trick (and its 32-r
+// pre-subtraction) goes away. The macro callers pass `r` directly.
+//
+// ## Why we don't widen to YMM/ZMM
+//
+// Upstream's `HasherInput_AVX512` keeps the 128-bit footprint — the
+// AVX-512 block is purely about replacing 2-3 ops with 1, not about
+// processing 2× or 4× the data. Widening would require reworking the
+// fused driver in `hasher_input.rs` and is explicitly out of scope.
+
+#![cfg(target_arch = "x86_64")]
+#![allow(non_snake_case)] // mirror upstream variable names A, B, C, D, XX0..XX15
+
+use core::arch::x86_64::{
+ __m128i, _mm_add_epi32, _mm_rol_epi32, _mm_set1_epi32, _mm_ternarylogic_epi32,
+};
+
+use crate::parpar_hasher::md5x2_sse2::{load4 as load4_xmm, State};
+
+/// 32-bit constant broadcast into all four 32-bit lanes. Direct port
+/// of upstream `VAL(k) = _mm_set1_epi32(k)`.
+#[inline(always)]
+#[allow(unused_unsafe)]
+unsafe fn val(k: u32) -> __m128i {
+ _mm_set1_epi32(k as i32)
+}
+
+// ----- Bool functions, one vpternlogd each -----
+//
+// `_mm_ternarylogic_epi32::(a, b, c)` evaluates the boolean
+// function whose 8-entry truth table is `IMM`, indexed MSB-first by
+// `(a_bit << 2) | (b_bit << 1) | c_bit`. Upstream calls these as
+// `_mm_ternarylogic_epi32(d, c, b, IMM)`, so we mirror the `(d, c, b)`
+// argument order verbatim.
+
+#[inline(always)]
+#[allow(unused_unsafe)]
+unsafe fn addf_f(a: __m128i, b: __m128i, c: __m128i, d: __m128i) -> __m128i {
+ let f = _mm_ternarylogic_epi32::<0xD8>(d, c, b);
+ _mm_add_epi32(a, f)
+}
+
+#[inline(always)]
+#[allow(unused_unsafe)]
+unsafe fn addf_g(a: __m128i, b: __m128i, c: __m128i, d: __m128i) -> __m128i {
+ let g = _mm_ternarylogic_epi32::<0xAC>(d, c, b);
+ _mm_add_epi32(a, g)
+}
+
+#[inline(always)]
+#[allow(unused_unsafe)]
+unsafe fn addf_h(a: __m128i, b: __m128i, c: __m128i, d: __m128i) -> __m128i {
+ let h = _mm_ternarylogic_epi32::<0x96>(d, c, b);
+ _mm_add_epi32(a, h)
+}
+
+#[inline(always)]
+#[allow(unused_unsafe)]
+unsafe fn addf_i(a: __m128i, b: __m128i, c: __m128i, d: __m128i) -> __m128i {
+ let i = _mm_ternarylogic_epi32::<0x63>(d, c, b);
+ _mm_add_epi32(a, i)
+}
+
+// ----------------------------------------------------------------------
+// Round body. Same shape as the SSE2 macro, but `vprold` takes the
+// rotate amount directly (no 32-r pre-subtraction).
+// ----------------------------------------------------------------------
+
+macro_rules! rx {
+ ($f_helper:ident, $a:ident, $b:ident, $c:ident, $d:ident, $xx:expr, $r:literal, $k:expr) => {{
+ $a = _mm_add_epi32($a, _mm_add_epi32($xx, val($k)));
+ $a = $f_helper($a, $b, $c, $d);
+ $a = _mm_rol_epi32::<$r>($a);
+ $a = _mm_add_epi32($a, $b);
+ }};
+}
+
+/// Process one 64-byte block per lane, MD5x2 AVX-512VL.
+///
+/// 1:1 expansion of `md5_process_block_x2_avx512` from the
+/// `__AVX512VL__` block of upstream `md5x2-sse.h` (after re-substitution
+/// via `md5x2-base.h` → `md5-base.h`).
+///
+/// # Safety
+/// Caller must ensure CPU supports `avx512f` + `avx512vl`. Both
+/// pointers must be valid for 64-byte reads. They may alias.
+#[target_feature(enable = "avx512f,avx512vl")]
+pub unsafe fn process_block_x2_avx512(state: &mut State, data1: *const u8, data2: *const u8) {
+ let oA = state.0[0];
+ let oB = state.0[1];
+ let oC = state.0[2];
+ let oD = state.0[3];
+
+ let mut A = oA;
+ let mut B = oB;
+ let mut C = oC;
+ let mut D = oD;
+
+ // load4 reuses upstream LOAD4 — same in both AVX-512 and SSE2 paths.
+ let (XX0, XX1, XX2, XX3) = load4_xmm(data1, data2, 0);
+ let (XX4, XX5, XX6, XX7) = load4_xmm(data1, data2, 4);
+ let (XX8, XX9, XX10, XX11) = load4_xmm(data1, data2, 8);
+ let (XX12, XX13, XX14, XX15) = load4_xmm(data1, data2, 12);
+
+ // ----- Round 0 (F) — md5-base.h:148-174 -----
+ rx!(addf_f, A, B, C, D, XX0, 7, 0xd76aa478);
+ rx!(addf_f, D, A, B, C, XX1, 12, 0xe8c7b756);
+ rx!(addf_f, C, D, A, B, XX2, 17, 0x242070db);
+ rx!(addf_f, B, C, D, A, XX3, 22, 0xc1bdceee);
+ rx!(addf_f, A, B, C, D, XX4, 7, 0xf57c0faf);
+ rx!(addf_f, D, A, B, C, XX5, 12, 0x4787c62a);
+ rx!(addf_f, C, D, A, B, XX6, 17, 0xa8304613);
+ rx!(addf_f, B, C, D, A, XX7, 22, 0xfd469501);
+ rx!(addf_f, A, B, C, D, XX8, 7, 0x698098d8);
+ rx!(addf_f, D, A, B, C, XX9, 12, 0x8b44f7af);
+ rx!(addf_f, C, D, A, B, XX10, 17, 0xffff5bb1);
+ rx!(addf_f, B, C, D, A, XX11, 22, 0x895cd7be);
+ rx!(addf_f, A, B, C, D, XX12, 7, 0x6b901122);
+ rx!(addf_f, D, A, B, C, XX13, 12, 0xfd987193);
+ rx!(addf_f, C, D, A, B, XX14, 17, 0xa679438e);
+ rx!(addf_f, B, C, D, A, XX15, 22, 0x49b40821);
+
+ // ----- Round 1 (G) — md5-base.h:192-207 -----
+ rx!(addf_g, A, B, C, D, XX1, 5, 0xf61e2562);
+ rx!(addf_g, D, A, B, C, XX6, 9, 0xc040b340);
+ rx!(addf_g, C, D, A, B, XX11, 14, 0x265e5a51);
+ rx!(addf_g, B, C, D, A, XX0, 20, 0xe9b6c7aa);
+ rx!(addf_g, A, B, C, D, XX5, 5, 0xd62f105d);
+ rx!(addf_g, D, A, B, C, XX10, 9, 0x02441453);
+ rx!(addf_g, C, D, A, B, XX15, 14, 0xd8a1e681);
+ rx!(addf_g, B, C, D, A, XX4, 20, 0xe7d3fbc8);
+ rx!(addf_g, A, B, C, D, XX9, 5, 0x21e1cde6);
+ rx!(addf_g, D, A, B, C, XX14, 9, 0xc33707d6);
+ rx!(addf_g, C, D, A, B, XX3, 14, 0xf4d50d87);
+ rx!(addf_g, B, C, D, A, XX8, 20, 0x455a14ed);
+ rx!(addf_g, A, B, C, D, XX13, 5, 0xa9e3e905);
+ rx!(addf_g, D, A, B, C, XX2, 9, 0xfcefa3f8);
+ rx!(addf_g, C, D, A, B, XX7, 14, 0x676f02d9);
+ rx!(addf_g, B, C, D, A, XX12, 20, 0x8d2a4c8a);
+
+ // ----- Round 2 (H) — md5-base.h:225-240 -----
+ rx!(addf_h, A, B, C, D, XX5, 4, 0xfffa3942);
+ rx!(addf_h, D, A, B, C, XX8, 11, 0x8771f681);
+ rx!(addf_h, C, D, A, B, XX11, 16, 0x6d9d6122);
+ rx!(addf_h, B, C, D, A, XX14, 23, 0xfde5380c);
+ rx!(addf_h, A, B, C, D, XX1, 4, 0xa4beea44);
+ rx!(addf_h, D, A, B, C, XX4, 11, 0x4bdecfa9);
+ rx!(addf_h, C, D, A, B, XX7, 16, 0xf6bb4b60);
+ rx!(addf_h, B, C, D, A, XX10, 23, 0xbebfbc70);
+ rx!(addf_h, A, B, C, D, XX13, 4, 0x289b7ec6);
+ rx!(addf_h, D, A, B, C, XX0, 11, 0xeaa127fa);
+ rx!(addf_h, C, D, A, B, XX3, 16, 0xd4ef3085);
+ rx!(addf_h, B, C, D, A, XX6, 23, 0x04881d05);
+ rx!(addf_h, A, B, C, D, XX9, 4, 0xd9d4d039);
+ rx!(addf_h, D, A, B, C, XX12, 11, 0xe6db99e5);
+ rx!(addf_h, C, D, A, B, XX15, 16, 0x1fa27cf8);
+ rx!(addf_h, B, C, D, A, XX2, 23, 0xc4ac5665);
+
+ // ----- Round 3 (I) — md5-base.h:263-278 -----
+ //
+ // IMPORTANT — IOFFSET handling: upstream's AVX (and only AVX)
+ // `__AVX__` block sets `IOFFSET = -1` because its `ADDF` redefines
+ // I as `_mm_sub_epi32(a, c ^ andnot(b, d))`, and the K constants
+ // need a -1 compensation. The `__AVX512VL__` block does NOT set
+ // IOFFSET (md5x2-sse.h:128-130 — `# ifdef IOFFSET / # undef
+ // IOFFSET / # endif` clears it before include). So our AVX-512
+ // path uses the SSE2-style I (= `((~d|b) ^ c)`) and the
+ // unmodified K constants. Same constants as the SSE2 backend.
+ rx!(addf_i, A, B, C, D, XX0, 6, 0xf4292244);
+ rx!(addf_i, D, A, B, C, XX7, 10, 0x432aff97);
+ rx!(addf_i, C, D, A, B, XX14, 15, 0xab9423a7);
+ rx!(addf_i, B, C, D, A, XX5, 21, 0xfc93a039);
+ rx!(addf_i, A, B, C, D, XX12, 6, 0x655b59c3);
+ rx!(addf_i, D, A, B, C, XX3, 10, 0x8f0ccc92);
+ rx!(addf_i, C, D, A, B, XX10, 15, 0xffeff47d);
+ rx!(addf_i, B, C, D, A, XX1, 21, 0x85845dd1);
+ rx!(addf_i, A, B, C, D, XX8, 6, 0x6fa87e4f);
+ rx!(addf_i, D, A, B, C, XX15, 10, 0xfe2ce6e0);
+ rx!(addf_i, C, D, A, B, XX6, 15, 0xa3014314);
+ rx!(addf_i, B, C, D, A, XX13, 21, 0x4e0811a1);
+ rx!(addf_i, A, B, C, D, XX4, 6, 0xf7537e82);
+ rx!(addf_i, D, A, B, C, XX11, 10, 0xbd3af235);
+ rx!(addf_i, C, D, A, B, XX2, 15, 0x2ad7d2bb);
+ rx!(addf_i, B, C, D, A, XX9, 21, 0xeb86d391);
+
+ state.0[0] = _mm_add_epi32(oA, A);
+ state.0[1] = _mm_add_epi32(oB, B);
+ state.0[2] = _mm_add_epi32(oC, C);
+ state.0[3] = _mm_add_epi32(oD, D);
+}
+
+/// `Md5x2` backend wrapper for the AVX-512VL implementation.
+///
+/// Caller is responsible for runtime feature detection — typically
+/// done via `HasherInputDyn::new()` which checks `is_x86_feature_detected!`
+/// before instantiating this backend.
+pub struct Avx512;
+
+impl crate::parpar_hasher::md5x2::Md5x2 for Avx512 {
+ type State = State;
+
+ const USE_AVX512_CRC: bool = true;
+
+ #[inline(always)]
+ fn init_state() -> Self::State {
+ // Same XMM lane layout as SSE2 — `from_flat` is shared verbatim.
+ let flat: [u32; 8] = [
+ 0x67452301, 0xefcdab89, 0x98badcfe, 0x10325476, 0x67452301, 0xefcdab89, 0x98badcfe,
+ 0x10325476,
+ ];
+ unsafe { State::from_flat(&flat) }
+ }
+
+ #[inline(always)]
+ fn init_lane(state: &mut Self::State, lane: usize) {
+ debug_assert!(lane < 2);
+ let mut flat = unsafe { state.to_flat() };
+ let off = lane * 4;
+ flat[off] = 0x67452301;
+ flat[off + 1] = 0xefcdab89;
+ flat[off + 2] = 0x98badcfe;
+ flat[off + 3] = 0x10325476;
+ *state = unsafe { State::from_flat(&flat) };
+ }
+
+ #[inline(always)]
+ fn extract_lane(state: &Self::State, lane: usize) -> [u8; 16] {
+ debug_assert!(lane < 2);
+ let flat = unsafe { state.to_flat() };
+ let off = lane * 4;
+ let mut out = [0u8; 16];
+ for i in 0..4 {
+ out[i * 4..i * 4 + 4].copy_from_slice(&flat[off + i].to_le_bytes());
+ }
+ out
+ }
+
+ #[inline(always)]
+ unsafe fn process_block(state: &mut Self::State, data1: *const u8, data2: *const u8) {
+ process_block_x2_avx512(state, data1, data2)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::parpar_hasher::md5x2::Md5x2;
+ use crate::parpar_hasher::md5x2_scalar::Scalar;
+
+ fn avx512_supported() -> bool {
+ is_x86_feature_detected!("avx512f") && is_x86_feature_detected!("avx512vl")
+ }
+
+ /// Hash an N-block message via scalar and AVX-512 backends and
+ /// assert lane digests match exactly. Mirrors `md5x2_sse2::tests`.
+ ///
+ /// Panics if AVX-512 is unsupported on this CPU — callers are
+ /// expected to be `#[ignore]`-marked tests so non-AVX-512 hosts
+ /// don't run them at all (rather than silently passing).
+ fn cross_check(blocks: &[u8]) {
+ assert!(
+ avx512_supported(),
+ "AVX-512 cross-check invoked on CPU without avx512f+vl — \
+ this test is #[ignore]'d; run with --ignored on AVX-512 hardware"
+ );
+ assert_eq!(blocks.len() % 64, 0);
+ let n = blocks.len() / 64;
+
+ let mut s_state = Scalar::init_state();
+ let mut v_state = Avx512::init_state();
+
+ for i in 0..n {
+ let d1 = &blocks[i * 64..i * 64 + 64];
+ let d2 = &blocks[(n - 1 - i) * 64..(n - 1 - i) * 64 + 64];
+ unsafe {
+ Scalar::process_block(&mut s_state, d1.as_ptr(), d2.as_ptr());
+ Avx512::process_block(&mut v_state, d1.as_ptr(), d2.as_ptr());
+ }
+ }
+
+ let s0 = Scalar::extract_lane(&s_state, 0);
+ let s1 = Scalar::extract_lane(&s_state, 1);
+ let v0 = Avx512::extract_lane(&v_state, 0);
+ let v1 = Avx512::extract_lane(&v_state, 1);
+
+ assert_eq!(v0, s0, "lane 0 mismatch after {n} blocks");
+ assert_eq!(v1, s1, "lane 1 mismatch after {n} blocks");
+ }
+
+ fn synth(len: usize, seed: u64) -> Vec {
+ let mut s = seed;
+ let mut out = vec![0u8; len];
+ for byte in out.iter_mut() {
+ s ^= s << 13;
+ s ^= s >> 7;
+ s ^= s << 17;
+ *byte = s as u8;
+ }
+ out
+ }
+
+ #[test]
+ #[ignore = "requires AVX-512VL hardware"]
+ fn one_block() {
+ cross_check(&synth(64, 1));
+ }
+
+ #[test]
+ #[ignore = "requires AVX-512VL hardware"]
+ fn two_blocks() {
+ cross_check(&synth(128, 2));
+ }
+
+ #[test]
+ #[ignore = "requires AVX-512VL hardware"]
+ fn many_blocks() {
+ cross_check(&synth(64 * 17, 3));
+ }
+
+ #[test]
+ #[ignore = "requires AVX-512VL hardware"]
+ fn lane_reset_round_trip() {
+ assert!(
+ avx512_supported(),
+ "lane_reset_round_trip invoked on non-AVX-512 host"
+ );
+ let mut state = Avx512::init_state();
+ let block = synth(64, 4);
+ unsafe {
+ Avx512::process_block(&mut state, block.as_ptr(), block.as_ptr());
+ }
+ let lane1_before = Avx512::extract_lane(&state, 1);
+ Avx512::init_lane(&mut state, 0);
+ let lane1_after = Avx512::extract_lane(&state, 1);
+ let lane0_after = Avx512::extract_lane(&state, 0);
+ assert_eq!(lane1_before, lane1_after, "init_lane(0) clobbered lane 1");
+ let iv: [u8; 16] = [
+ 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54,
+ 0x32, 0x10,
+ ];
+ assert_eq!(lane0_after, iv, "init_lane(0) didn't restore IV");
+ }
+}
diff --git a/src/parpar_hasher/md5x2_bmi1.rs b/src/parpar_hasher/md5x2_bmi1.rs
new file mode 100644
index 00000000..f0487751
--- /dev/null
+++ b/src/parpar_hasher/md5x2_bmi1.rs
@@ -0,0 +1,481 @@
+// Copyright (C) Anomaly Industries Inc. and par2rs contributors.
+// SPDX-License-Identifier: GPL-2.0-or-later
+//
+// Two-lane MD5 block compression for x86_64 using BMI1's `andn`.
+//
+// This is a Rust line-for-line port of:
+// parpar/hasher/md5x2-x86-asm.h (the `_MD5_USE_BMI1_` branches),
+// wired through parpar/hasher/hasher_bmi1.cpp
+// from par2cmdline-turbo (https://github.com/animetosho/par2cmdline-turbo)
+// and the upstream ParPar (https://github.com/animetosho/ParPar), both
+// licensed GPL-2.0-or-later.
+//
+// Relative to `md5x2_scalar.rs`, this file differs in only the G and I
+// (and ROUND_I_LAST) round bodies — F and H are bit-for-bit identical.
+// The substitutions track upstream's `#ifdef _MD5_USE_BMI1_` branches:
+//
+// * ROUND_G: legacy `(D & B) | (~D & C)` rewritten as
+// `andnl C, D, TMP` ; TMP = (~D) & C
+// `addl TMP, A`
+// `movl D, TMP`
+// `andl B, TMP` ; TMP = D & B
+// `addl TMP, A`
+// The two summands are bitwise-disjoint, so `+` ≡ `|` exactly. Saves a
+// `notl` (and lets the OoO engine schedule two independent `addl`s
+// instead of an `or`-then-add chain).
+//
+// * ROUND_I / ROUND_I_LAST: legacy `((~D) | B) ^ C` with `+K, +TMP`
+// rewritten via the identity
+// `K + (((~D)|B) ^ C) ≡ (K-1) - (((~B)&D) ^ C) (mod 2^32)`
+// (because `~(~D|B) = D & ~B`, and complement-pairs sum to -1; XORing
+// each with C preserves the complement relationship). This saves a
+// `notl` and removes the `or` from the dep chain:
+// `addl $K-1, A`
+// `andnl D, B, TMP` ; TMP = (~B) & D
+// `xorl C, TMP`
+// `subl TMP, A` ; A += K - 1 - TMP, equivalent to legacy formula
+//
+// The function is gated by `#[target_feature(enable = "bmi1")]`. Callers
+// MUST verify BMI1 support (e.g. `is_x86_feature_detected!("bmi1")`)
+// before invoking, otherwise behaviour is undefined.
+
+#![cfg(target_arch = "x86_64")]
+
+use core::arch::asm;
+
+/// Process one 64-byte block for each of two MD5 lanes, using BMI1
+/// `andn` in the G and I round bodies.
+///
+/// `state[0..4]` is lane 1 (A1, B1, C1, D1); `state[4..8]` is lane 2.
+///
+/// # Safety
+/// * `data1` and `data2` must each be valid for 64-byte reads (may overlap).
+/// * The CPU must support BMI1. Verify with `is_x86_feature_detected!("bmi1")`.
+#[target_feature(enable = "bmi1")]
+#[inline]
+pub unsafe fn process_block_x2_bmi1(state: &mut [u32; 8], data1: *const u8, data2: *const u8) {
+ let mut a1 = state[0];
+ let mut b1 = state[1];
+ let mut c1 = state[2];
+ let mut d1 = state[3];
+ let mut a2 = state[4];
+ let mut b2 = state[5];
+ let mut c2 = state[6];
+ let mut d2 = state[7];
+
+ // Pre-add input word 0 into A (mirrors upstream's pre-loop fold-in).
+ a1 = a1.wrapping_add(read32(data1, 0));
+ a2 = a2.wrapping_add(read32(data2, 0));
+
+ // ---- F: identical to scalar ---------------------------------------
+ macro_rules! round_f {
+ ($a1:ident, $b1:ident, $c1:ident, $d1:ident,
+ $a2:ident, $b2:ident, $c2:ident, $d2:ident,
+ $i_off:expr, $k:expr, $r:expr) => {
+ asm!(
+ "movl {d1:e}, {tmp1:e}",
+ "movl {d2:e}, {tmp2:e}",
+ "addl ${k}, {a1:e}",
+ "addl ${k}, {a2:e}",
+ "xorl {c1:e}, {tmp1:e}",
+ "xorl {c2:e}, {tmp2:e}",
+ "andl {b1:e}, {tmp1:e}",
+ "andl {b2:e}, {tmp2:e}",
+ "xorl {d1:e}, {tmp1:e}",
+ "xorl {d2:e}, {tmp2:e}",
+ "addl {i_off}({base1}), {d1:e}",
+ "addl {i_off}({base2}), {d2:e}",
+ "addl {tmp1:e}, {a1:e}",
+ "addl {tmp2:e}, {a2:e}",
+ "roll ${r}, {a1:e}",
+ "roll ${r}, {a2:e}",
+ "addl {b1:e}, {a1:e}",
+ "addl {b2:e}, {a2:e}",
+ k = const ($k as u32 as i32),
+ r = const $r,
+ i_off = const ($i_off * 4),
+ a1 = inout(reg) $a1,
+ b1 = in(reg) $b1,
+ c1 = in(reg) $c1,
+ d1 = inout(reg) $d1,
+ a2 = inout(reg) $a2,
+ b2 = in(reg) $b2,
+ c2 = in(reg) $c2,
+ d2 = inout(reg) $d2,
+ tmp1 = out(reg) _,
+ tmp2 = out(reg) _,
+ base1 = in(reg) data1,
+ base2 = in(reg) data2,
+ options(att_syntax, nostack, readonly),
+ );
+ };
+ }
+
+ // ---- G: BMI1 form — andnl C,D,TMP ; addl TMP,A ; movl D,TMP ;
+ // addl Mi,D ; andl B,TMP ; addl TMP,A ; rol ; +B
+ macro_rules! round_g {
+ ($a1:ident, $b1:ident, $c1:ident, $d1:ident,
+ $a2:ident, $b2:ident, $c2:ident, $d2:ident,
+ $i_off:expr, $k:expr, $r:expr) => {
+ asm!(
+ "addl ${k}, {a1:e}",
+ "addl ${k}, {a2:e}",
+ // TMP = (~D) & C
+ "andnl {c1:e}, {d1:e}, {tmp1:e}",
+ "andnl {c2:e}, {d2:e}, {tmp2:e}",
+ "addl {tmp1:e}, {a1:e}",
+ "addl {tmp2:e}, {a2:e}",
+ // TMP = D_old, then D += Mi (input fold), then TMP &= B → D & B
+ "movl {d1:e}, {tmp1:e}",
+ "movl {d2:e}, {tmp2:e}",
+ "addl {i_off}({base1}), {d1:e}",
+ "addl {i_off}({base2}), {d2:e}",
+ "andl {b1:e}, {tmp1:e}",
+ "andl {b2:e}, {tmp2:e}",
+ "addl {tmp1:e}, {a1:e}",
+ "addl {tmp2:e}, {a2:e}",
+ "roll ${r}, {a1:e}",
+ "roll ${r}, {a2:e}",
+ "addl {b1:e}, {a1:e}",
+ "addl {b2:e}, {a2:e}",
+ k = const ($k as u32 as i32),
+ r = const $r,
+ i_off = const ($i_off * 4),
+ a1 = inout(reg) $a1,
+ b1 = in(reg) $b1,
+ c1 = in(reg) $c1,
+ d1 = inout(reg) $d1,
+ a2 = inout(reg) $a2,
+ b2 = in(reg) $b2,
+ c2 = in(reg) $c2,
+ d2 = inout(reg) $d2,
+ tmp1 = out(reg) _,
+ tmp2 = out(reg) _,
+ base1 = in(reg) data1,
+ base2 = in(reg) data2,
+ options(att_syntax, nostack, readonly),
+ );
+ };
+ }
+
+ // ---- H: identical to scalar ---------------------------------------
+ macro_rules! round_h {
+ ($a1:ident, $b1:ident, $c1:ident, $d1:ident,
+ $a2:ident, $b2:ident, $c2:ident, $d2:ident,
+ $i_off:expr, $k:expr, $r:expr) => {
+ asm!(
+ "movl {d1:e}, {tmp1:e}",
+ "movl {d2:e}, {tmp2:e}",
+ "addl ${k}, {a1:e}",
+ "addl ${k}, {a2:e}",
+ "xorl {c1:e}, {tmp1:e}",
+ "xorl {c2:e}, {tmp2:e}",
+ "addl {i_off}({base1}), {d1:e}",
+ "addl {i_off}({base2}), {d2:e}",
+ "xorl {b1:e}, {tmp1:e}",
+ "xorl {b2:e}, {tmp2:e}",
+ "addl {tmp1:e}, {a1:e}",
+ "addl {tmp2:e}, {a2:e}",
+ "roll ${r}, {a1:e}",
+ "roll ${r}, {a2:e}",
+ "addl {b1:e}, {a1:e}",
+ "addl {b2:e}, {a2:e}",
+ k = const ($k as u32 as i32),
+ r = const $r,
+ i_off = const ($i_off * 4),
+ a1 = inout(reg) $a1,
+ b1 = in(reg) $b1,
+ c1 = in(reg) $c1,
+ d1 = inout(reg) $d1,
+ a2 = inout(reg) $a2,
+ b2 = in(reg) $b2,
+ c2 = in(reg) $c2,
+ d2 = inout(reg) $d2,
+ tmp1 = out(reg) _,
+ tmp2 = out(reg) _,
+ base1 = in(reg) data1,
+ base2 = in(reg) data2,
+ options(att_syntax, nostack, readonly),
+ );
+ };
+ }
+
+ // ---- I: BMI1 form — addl K-1,A ; andnl D,B,TMP ; xorl C,TMP ;
+ // addl Mi,D ; subl TMP,A ; rol ; +B
+ macro_rules! round_i {
+ ($a1:ident, $b1:ident, $c1:ident, $d1:ident,
+ $a2:ident, $b2:ident, $c2:ident, $d2:ident,
+ $i_off:expr, $k:expr, $r:expr) => {
+ asm!(
+ "addl ${km1}, {a1:e}",
+ "addl ${km1}, {a2:e}",
+ // TMP = (~B) & D
+ "andnl {d1:e}, {b1:e}, {tmp1:e}",
+ "andnl {d2:e}, {b2:e}, {tmp2:e}",
+ "xorl {c1:e}, {tmp1:e}",
+ "xorl {c2:e}, {tmp2:e}",
+ "addl {i_off}({base1}), {d1:e}",
+ "addl {i_off}({base2}), {d2:e}",
+ "subl {tmp1:e}, {a1:e}",
+ "subl {tmp2:e}, {a2:e}",
+ "roll ${r}, {a1:e}",
+ "roll ${r}, {a2:e}",
+ "addl {b1:e}, {a1:e}",
+ "addl {b2:e}, {a2:e}",
+ km1 = const (($k as u32 as i32).wrapping_sub(1)),
+ r = const $r,
+ i_off = const ($i_off * 4),
+ a1 = inout(reg) $a1,
+ b1 = in(reg) $b1,
+ c1 = in(reg) $c1,
+ d1 = inout(reg) $d1,
+ a2 = inout(reg) $a2,
+ b2 = in(reg) $b2,
+ c2 = in(reg) $c2,
+ d2 = inout(reg) $d2,
+ tmp1 = out(reg) _,
+ tmp2 = out(reg) _,
+ base1 = in(reg) data1,
+ base2 = in(reg) data2,
+ options(att_syntax, nostack, readonly),
+ );
+ };
+ }
+
+ // ---- I_LAST: same as I but no input fold into D ------------------
+ macro_rules! round_i_last {
+ ($a1:ident, $b1:ident, $c1:ident, $d1:ident,
+ $a2:ident, $b2:ident, $c2:ident, $d2:ident,
+ $k:expr, $r:expr) => {
+ asm!(
+ "addl ${km1}, {a1:e}",
+ "addl ${km1}, {a2:e}",
+ "andnl {d1:e}, {b1:e}, {tmp1:e}",
+ "andnl {d2:e}, {b2:e}, {tmp2:e}",
+ "xorl {c1:e}, {tmp1:e}",
+ "xorl {c2:e}, {tmp2:e}",
+ "subl {tmp1:e}, {a1:e}",
+ "subl {tmp2:e}, {a2:e}",
+ "roll ${r}, {a1:e}",
+ "roll ${r}, {a2:e}",
+ "addl {b1:e}, {a1:e}",
+ "addl {b2:e}, {a2:e}",
+ km1 = const (($k as u32 as i32).wrapping_sub(1)),
+ r = const $r,
+ a1 = inout(reg) $a1,
+ b1 = in(reg) $b1,
+ c1 = in(reg) $c1,
+ d1 = in(reg) $d1,
+ a2 = inout(reg) $a2,
+ b2 = in(reg) $b2,
+ c2 = in(reg) $c2,
+ d2 = in(reg) $d2,
+ tmp1 = out(reg) _,
+ tmp2 = out(reg) _,
+ options(att_syntax, nostack, readonly),
+ );
+ };
+ }
+
+ // Round 1 (F) — identical schedule to scalar
+ round_f!(a1, b1, c1, d1, a2, b2, c2, d2, 1, -0x28955b88_i32, 7);
+ round_f!(d1, a1, b1, c1, d2, a2, b2, c2, 2, -0x173848aa_i32, 12);
+ round_f!(c1, d1, a1, b1, c2, d2, a2, b2, 3, 0x242070db_i32, 17);
+ round_f!(b1, c1, d1, a1, b2, c2, d2, a2, 4, -0x3e423112_i32, 22);
+
+ round_f!(a1, b1, c1, d1, a2, b2, c2, d2, 5, -0x0a83f051_i32, 7);
+ round_f!(d1, a1, b1, c1, d2, a2, b2, c2, 6, 0x4787c62a_i32, 12);
+ round_f!(c1, d1, a1, b1, c2, d2, a2, b2, 7, -0x57cfb9ed_i32, 17);
+ round_f!(b1, c1, d1, a1, b2, c2, d2, a2, 8, -0x02b96aff_i32, 22);
+
+ round_f!(a1, b1, c1, d1, a2, b2, c2, d2, 9, 0x698098d8_i32, 7);
+ round_f!(d1, a1, b1, c1, d2, a2, b2, c2, 10, -0x74bb0851_i32, 12);
+ round_f!(c1, d1, a1, b1, c2, d2, a2, b2, 11, -0x0000a44f_i32, 17);
+ round_f!(b1, c1, d1, a1, b2, c2, d2, a2, 12, -0x76a32842_i32, 22);
+
+ round_f!(a1, b1, c1, d1, a2, b2, c2, d2, 13, 0x6b901122_i32, 7);
+ round_f!(d1, a1, b1, c1, d2, a2, b2, c2, 14, -0x02678e6d_i32, 12);
+ round_f!(c1, d1, a1, b1, c2, d2, a2, b2, 15, -0x5986bc72_i32, 17);
+ round_f!(b1, c1, d1, a1, b2, c2, d2, a2, 1, 0x49b40821_i32, 22);
+
+ // Round 2 (G) — BMI1 form
+ round_g!(a1, b1, c1, d1, a2, b2, c2, d2, 6, -0x09e1da9e_i32, 5);
+ round_g!(d1, a1, b1, c1, d2, a2, b2, c2, 11, -0x3fbf4cc0_i32, 9);
+ round_g!(c1, d1, a1, b1, c2, d2, a2, b2, 0, 0x265e5a51_i32, 14);
+ round_g!(b1, c1, d1, a1, b2, c2, d2, a2, 5, -0x16493856_i32, 20);
+
+ round_g!(a1, b1, c1, d1, a2, b2, c2, d2, 10, -0x29d0efa3_i32, 5);
+ round_g!(d1, a1, b1, c1, d2, a2, b2, c2, 15, 0x02441453_i32, 9);
+ round_g!(c1, d1, a1, b1, c2, d2, a2, b2, 4, -0x275e197f_i32, 14);
+ round_g!(b1, c1, d1, a1, b2, c2, d2, a2, 9, -0x182c0438_i32, 20);
+
+ round_g!(a1, b1, c1, d1, a2, b2, c2, d2, 14, 0x21e1cde6_i32, 5);
+ round_g!(d1, a1, b1, c1, d2, a2, b2, c2, 3, -0x3cc8f82a_i32, 9);
+ round_g!(c1, d1, a1, b1, c2, d2, a2, b2, 8, -0x0b2af279_i32, 14);
+ round_g!(b1, c1, d1, a1, b2, c2, d2, a2, 13, 0x455a14ed_i32, 20);
+
+ round_g!(a1, b1, c1, d1, a2, b2, c2, d2, 2, -0x561c16fb_i32, 5);
+ round_g!(d1, a1, b1, c1, d2, a2, b2, c2, 7, -0x03105c08_i32, 9);
+ round_g!(c1, d1, a1, b1, c2, d2, a2, b2, 12, 0x676f02d9_i32, 14);
+ round_g!(b1, c1, d1, a1, b2, c2, d2, a2, 5, -0x72d5b376_i32, 20);
+
+ // Round 3 (H) — identical schedule to scalar
+ round_h!(a1, b1, c1, d1, a2, b2, c2, d2, 8, -0x0005c6be_i32, 4);
+ round_h!(d1, a1, b1, c1, d2, a2, b2, c2, 11, -0x788e097f_i32, 11);
+ round_h!(c1, d1, a1, b1, c2, d2, a2, b2, 14, 0x6d9d6122_i32, 16);
+ round_h!(b1, c1, d1, a1, b2, c2, d2, a2, 1, -0x021ac7f4_i32, 23);
+
+ round_h!(a1, b1, c1, d1, a2, b2, c2, d2, 4, -0x5b4115bc_i32, 4);
+ round_h!(d1, a1, b1, c1, d2, a2, b2, c2, 7, 0x4bdecfa9_i32, 11);
+ round_h!(c1, d1, a1, b1, c2, d2, a2, b2, 10, -0x0944b4a0_i32, 16);
+ round_h!(b1, c1, d1, a1, b2, c2, d2, a2, 13, -0x41404390_i32, 23);
+
+ round_h!(a1, b1, c1, d1, a2, b2, c2, d2, 0, 0x289b7ec6_i32, 4);
+ round_h!(d1, a1, b1, c1, d2, a2, b2, c2, 3, -0x155ed806_i32, 11);
+ round_h!(c1, d1, a1, b1, c2, d2, a2, b2, 6, -0x2b10cf7b_i32, 16);
+ round_h!(b1, c1, d1, a1, b2, c2, d2, a2, 9, 0x04881d05_i32, 23);
+
+ round_h!(a1, b1, c1, d1, a2, b2, c2, d2, 12, -0x262b2fc7_i32, 4);
+ round_h!(d1, a1, b1, c1, d2, a2, b2, c2, 15, -0x1924661b_i32, 11);
+ round_h!(c1, d1, a1, b1, c2, d2, a2, b2, 2, 0x1fa27cf8_i32, 16);
+ round_h!(b1, c1, d1, a1, b2, c2, d2, a2, 0, -0x3b53a99b_i32, 23);
+
+ // Round 4 (I) — BMI1 form
+ round_i!(a1, b1, c1, d1, a2, b2, c2, d2, 7, -0x0bd6ddbc_i32, 6);
+ round_i!(d1, a1, b1, c1, d2, a2, b2, c2, 14, 0x432aff97_i32, 10);
+ round_i!(c1, d1, a1, b1, c2, d2, a2, b2, 5, -0x546bdc59_i32, 15);
+ round_i!(b1, c1, d1, a1, b2, c2, d2, a2, 12, -0x036c5fc7_i32, 21);
+
+ round_i!(a1, b1, c1, d1, a2, b2, c2, d2, 3, 0x655b59c3_i32, 6);
+ round_i!(d1, a1, b1, c1, d2, a2, b2, c2, 10, -0x70f3336e_i32, 10);
+ round_i!(c1, d1, a1, b1, c2, d2, a2, b2, 1, -0x00100b83_i32, 15);
+ round_i!(b1, c1, d1, a1, b2, c2, d2, a2, 8, -0x7a7ba22f_i32, 21);
+
+ round_i!(a1, b1, c1, d1, a2, b2, c2, d2, 15, 0x6fa87e4f_i32, 6);
+ round_i!(d1, a1, b1, c1, d2, a2, b2, c2, 6, -0x01d31920_i32, 10);
+ round_i!(c1, d1, a1, b1, c2, d2, a2, b2, 13, -0x5cfebcec_i32, 15);
+ round_i!(b1, c1, d1, a1, b2, c2, d2, a2, 4, 0x4e0811a1_i32, 21);
+
+ round_i!(a1, b1, c1, d1, a2, b2, c2, d2, 11, -0x08ac817e_i32, 6);
+ round_i!(d1, a1, b1, c1, d2, a2, b2, c2, 2, -0x42c50dcb_i32, 10);
+ round_i!(c1, d1, a1, b1, c2, d2, a2, b2, 9, 0x2ad7d2bb_i32, 15);
+ round_i_last!(b1, c1, d1, a1, b2, c2, d2, a2, -0x14792c6f_i32, 21);
+
+ state[0] = state[0].wrapping_add(a1);
+ state[1] = state[1].wrapping_add(b1);
+ state[2] = state[2].wrapping_add(c1);
+ state[3] = state[3].wrapping_add(d1);
+ state[4] = state[4].wrapping_add(a2);
+ state[5] = state[5].wrapping_add(b2);
+ state[6] = state[6].wrapping_add(c2);
+ state[7] = state[7].wrapping_add(d2);
+}
+
+#[inline]
+unsafe fn read32(p: *const u8, word_idx: usize) -> u32 {
+ let off = word_idx * 4;
+ let bytes = core::ptr::read_unaligned(p.add(off) as *const [u8; 4]);
+ u32::from_le_bytes(bytes)
+}
+
+/// `Md5x2` backend wrapper for the BMI1 implementation. Identical state
+/// layout / IVs to the scalar backend; the dispatcher in `hasher_input`
+/// chooses this on CPUs that report BMI1 (e.g. Zen3, Skylake+).
+pub struct Bmi1;
+
+impl crate::parpar_hasher::md5x2::Md5x2 for Bmi1 {
+ type State = [u32; 8];
+
+ #[inline]
+ fn init_state() -> Self::State {
+ crate::parpar_hasher::md5x2_scalar::init_state()
+ }
+
+ #[inline]
+ fn init_lane(state: &mut Self::State, lane: usize) {
+ crate::parpar_hasher::md5x2_scalar::init_lane(state, lane);
+ }
+
+ #[inline]
+ fn extract_lane(state: &Self::State, lane: usize) -> [u8; 16] {
+ let off = lane * 4;
+ let mut out = [0u8; 16];
+ for i in 0..4 {
+ out[i * 4..i * 4 + 4].copy_from_slice(&state[off + i].to_le_bytes());
+ }
+ out
+ }
+
+ #[inline]
+ unsafe fn process_block(state: &mut Self::State, data1: *const u8, data2: *const u8) {
+ // Caller must verify BMI1 support before instantiating
+ // HasherInput; the runtime dispatcher in hasher_input does
+ // this via is_x86_feature_detected!. Direct construction of
+ // HasherInput on a non-BMI1 CPU is UB, mirroring the
+ // upstream contract for HasherInput_BMI1.
+ process_block_x2_bmi1(state, data1, data2)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::parpar_hasher::md5x2_scalar::process_block_x2_scalar;
+
+ /// Cross-check: compress the same two messages with scalar and BMI1;
+ /// every state word must match. This is the strongest test we have —
+ /// scalar already passes its own RFC 1321 oracle test, so equality
+ /// here proves BMI1 is correct.
+ #[test]
+ fn cross_check_against_scalar() {
+ if !std::is_x86_feature_detected!("bmi1") {
+ return; // skip on non-BMI1 hosts
+ }
+
+ // Deterministic varied inputs; 17 blocks per lane to exercise
+ // multi-block accumulation through the state array.
+ let mut buf1 = [0u8; 64 * 17];
+ let mut buf2 = [0u8; 64 * 17];
+ for (i, b) in buf1.iter_mut().enumerate() {
+ *b = (i.wrapping_mul(31) ^ 0xa5) as u8;
+ }
+ for (i, b) in buf2.iter_mut().enumerate() {
+ *b = (i.wrapping_mul(17) ^ 0x5a) as u8;
+ }
+
+ let mut s_scalar = crate::parpar_hasher::md5x2_scalar::init_state();
+ let mut s_bmi1 = crate::parpar_hasher::md5x2_scalar::init_state();
+ for blk in 0..17 {
+ unsafe {
+ let p1 = buf1.as_ptr().add(blk * 64);
+ let p2 = buf2.as_ptr().add(blk * 64);
+ process_block_x2_scalar(&mut s_scalar, p1, p2);
+ process_block_x2_bmi1(&mut s_bmi1, p1, p2);
+ }
+ assert_eq!(s_scalar, s_bmi1, "state divergence at block {blk}");
+ }
+ }
+
+ /// Single-block sanity check: known vector "abc"-padded.
+ #[test]
+ fn known_vector_abc() {
+ if !std::is_x86_feature_detected!("bmi1") {
+ return;
+ }
+ let mut block = [0u8; 64];
+ block[0] = b'a';
+ block[1] = b'b';
+ block[2] = b'c';
+ block[3] = 0x80; // padding bit
+ // 24-bit (3-byte) length in little-endian at the end
+ block[56] = 24;
+
+ let mut s_scalar = crate::parpar_hasher::md5x2_scalar::init_state();
+ let mut s_bmi1 = crate::parpar_hasher::md5x2_scalar::init_state();
+ unsafe {
+ process_block_x2_scalar(&mut s_scalar, block.as_ptr(), block.as_ptr());
+ process_block_x2_bmi1(&mut s_bmi1, block.as_ptr(), block.as_ptr());
+ }
+ assert_eq!(s_scalar, s_bmi1);
+ }
+}
diff --git a/src/parpar_hasher/md5x2_neon.rs b/src/parpar_hasher/md5x2_neon.rs
new file mode 100644
index 00000000..d0fb5a31
--- /dev/null
+++ b/src/parpar_hasher/md5x2_neon.rs
@@ -0,0 +1,315 @@
+//! MD5x2 NEON backend for ARM64 (AArch64).
+//!
+//! This module provides a two-lane MD5 block compressor using ARM NEON
+//! SIMD intrinsics. It ports the algorithm from par2cmdline-turbo/ParPar:
+//! - parpar/hasher/md5x2-neon.h (intrinsic macro definitions)
+//! - parpar/hasher/md5-base.h (64-round expansion)
+//!
+//! This implementation uses NEON intrinsics from `core::arch::aarch64`
+//! rather than inline asm, following the "intrinsics-first" design
+//! to prioritize code clarity and let the compiler handle scheduling.
+//!
+//! ## Layout
+//! Each NEON 64-bit register holds two 32-bit lanes (lane0 in bits [31:0],
+//! lane1 in bits [63:32]). The state is four uint32x2_t registers:
+//! - state[0] = [A_lane0, A_lane1]
+//! - state[1] = [B_lane0, B_lane1]
+//! - state[2] = [C_lane0, C_lane1]
+//! - state[3] = [D_lane0, D_lane1]
+//!
+//! ## Round macro
+//! `rx!(TYPE, a, b, c, d, msg_word, K_const, R, 32-R)` computes one MD5
+//! step: `a = rotate_left(a + msg + K + TYPE(b,c,d), R) + b`.
+//! The rotation is split into `vsli_n_u32::(vshr_n_u32::<32-R>(x), x)`
+//! using literal const generics so the compiler emits a single SLI/SHR pair.
+//!
+//! ## Message loading
+//! 16 words from `data1` and `data2` are interleaved into `msg[0..16]`
+//! via four `vzipq_u32` calls: msg[i] = uint32x2_t{data1[i], data2[i]}.
+//! On LE aarch64, `vld1q_u8 + vreinterpretq_u32_u8` gives native-endian
+//! u32 words, matching MD5's LE word convention.
+
+#![cfg(target_arch = "aarch64")]
+
+use super::md5x2::Md5x2;
+use core::arch::aarch64::{
+ uint32x2_t, vadd_u32, vbsl_u32, vdup_n_u32, veor_u32, vget_high_u32, vget_lane_u32,
+ vget_low_u32, vld1q_u8, vorn_u32, vreinterpretq_u32_u8, vset_lane_u32, vshr_n_u32, vsli_n_u32,
+ vzipq_u32,
+};
+
+// One MD5 round step. Each arm computes:
+// a = rotate_left(a + msg_word + K_const + ROUND_FUNC(b, c, d), R) + b
+// Round functions (Reference: par2cmdline-turbo/parpar/hasher/md5x2-neon.h):
+// F(b,c,d) = (b & c) | (~b & d) = vbsl_u32(b, c, d)
+// G(b,c,d) = (d & b) | (~d & c) = vbsl_u32(d, b, c)
+// H(b,c,d) = b ^ c ^ d = veor_u32(veor_u32(b,c),d)
+// I(b,c,d) = c ^ (b | ~d) = veor_u32(c, vorn_u32(b, d))
+//
+// The rotation `rotate_left(x, R)` expands to:
+// vsli_n_u32::(vshr_n_u32::<{32-R}>(x), x)
+// Both shift amounts must be compile-time literals; callers pass both
+// R and 32-R explicitly since generic_const_exprs is not stable.
+macro_rules! rx {
+ (F, $a:ident, $b:ident, $c:ident, $d:ident, $m:expr, $k:literal, $r:literal, $rr:literal) => {{
+ let mk = vadd_u32($m, vdup_n_u32($k));
+ let sum = vadd_u32(vadd_u32($a, mk), vbsl_u32($b, $c, $d));
+ $a = vadd_u32(vsli_n_u32::<$r>(vshr_n_u32::<$rr>(sum), sum), $b);
+ }};
+ (G, $a:ident, $b:ident, $c:ident, $d:ident, $m:expr, $k:literal, $r:literal, $rr:literal) => {{
+ let mk = vadd_u32($m, vdup_n_u32($k));
+ let sum = vadd_u32(vadd_u32($a, mk), vbsl_u32($d, $b, $c));
+ $a = vadd_u32(vsli_n_u32::<$r>(vshr_n_u32::<$rr>(sum), sum), $b);
+ }};
+ (H, $a:ident, $b:ident, $c:ident, $d:ident, $m:expr, $k:literal, $r:literal, $rr:literal) => {{
+ let mk = vadd_u32($m, vdup_n_u32($k));
+ let sum = vadd_u32(vadd_u32($a, mk), veor_u32(veor_u32($b, $c), $d));
+ $a = vadd_u32(vsli_n_u32::<$r>(vshr_n_u32::<$rr>(sum), sum), $b);
+ }};
+ (I, $a:ident, $b:ident, $c:ident, $d:ident, $m:expr, $k:literal, $r:literal, $rr:literal) => {{
+ let mk = vadd_u32($m, vdup_n_u32($k));
+ let sum = vadd_u32(vadd_u32($a, mk), veor_u32($c, vorn_u32($b, $d)));
+ $a = vadd_u32(vsli_n_u32::<$r>(vshr_n_u32::<$rr>(sum), sum), $b);
+ }};
+}
+
+/// NEON MD5x2 backend state: four uint32x2_t registers (64 bits each,
+/// holding two 32-bit lanes in the lower/upper halves of each 64-bit qword).
+#[derive(Clone, Copy)]
+pub struct State {
+ pub regs: [uint32x2_t; 4],
+}
+
+impl Md5x2 for State {
+ type State = Self;
+ const USE_AVX512_CRC: bool = false;
+
+ fn init_state() -> Self::State {
+ unsafe {
+ // All lanes start at the standard MD5 IV.
+ let mut s = State {
+ regs: [vdup_n_u32(0); 4],
+ };
+ s.regs[0] = vdup_n_u32(0x67452301);
+ s.regs[1] = vdup_n_u32(0xefcdab89);
+ s.regs[2] = vdup_n_u32(0x98badcfe);
+ s.regs[3] = vdup_n_u32(0x10325476);
+ s
+ }
+ }
+
+ fn init_lane(state: &mut Self::State, lane: usize) {
+ unsafe {
+ // Reset a single lane (0 or 1) to the MD5 IV while keeping the other lane unchanged.
+ // vset_lane_u32 requires a const index, so we must use a match.
+ match lane {
+ 0 => {
+ state.regs[0] = vset_lane_u32(0x67452301, state.regs[0], 0);
+ state.regs[1] = vset_lane_u32(0xefcdab89, state.regs[1], 0);
+ state.regs[2] = vset_lane_u32(0x98badcfe, state.regs[2], 0);
+ state.regs[3] = vset_lane_u32(0x10325476, state.regs[3], 0);
+ }
+ 1 => {
+ state.regs[0] = vset_lane_u32(0x67452301, state.regs[0], 1);
+ state.regs[1] = vset_lane_u32(0xefcdab89, state.regs[1], 1);
+ state.regs[2] = vset_lane_u32(0x98badcfe, state.regs[2], 1);
+ state.regs[3] = vset_lane_u32(0x10325476, state.regs[3], 1);
+ }
+ _ => panic!("invalid lane index: expected 0 or 1, got {}", lane),
+ }
+ }
+ }
+
+ fn extract_lane(state: &Self::State, lane: usize) -> [u8; 16] {
+ unsafe {
+ // Extract one lane's MD5 digest as a 16-byte little-endian value.
+ let (a, b, c, d) = match lane {
+ 0 => {
+ let a = vget_lane_u32(state.regs[0], 0);
+ let b = vget_lane_u32(state.regs[1], 0);
+ let c = vget_lane_u32(state.regs[2], 0);
+ let d = vget_lane_u32(state.regs[3], 0);
+ (a, b, c, d)
+ }
+ 1 => {
+ let a = vget_lane_u32(state.regs[0], 1);
+ let b = vget_lane_u32(state.regs[1], 1);
+ let c = vget_lane_u32(state.regs[2], 1);
+ let d = vget_lane_u32(state.regs[3], 1);
+ (a, b, c, d)
+ }
+ _ => panic!("invalid lane index: expected 0 or 1, got {}", lane),
+ };
+
+ // Convert to little-endian bytes. MD5 output is [A,B,C,D] as u32 LE.
+ let mut digest = [0u8; 16];
+ digest[0..4].copy_from_slice(&a.to_le_bytes());
+ digest[4..8].copy_from_slice(&b.to_le_bytes());
+ digest[8..12].copy_from_slice(&c.to_le_bytes());
+ digest[12..16].copy_from_slice(&d.to_le_bytes());
+ digest
+ }
+ }
+
+ unsafe fn process_block(state: &mut Self::State, data1: *const u8, data2: *const u8) {
+ // Load 16 message words from both buffers, interleaving lane0 (data1)
+ // and lane1 (data2) so that msg[i] = {data1_word_i, data2_word_i}.
+ //
+ // vzipq_u32([a0,a1,a2,a3], [b0,b1,b2,b3]) → ([a0,b0,a1,b1], [a2,b2,a3,b3])
+ // vget_low/high extract the two uint32x2_t halves.
+ // Reference: par2cmdline-turbo/parpar/hasher/md5x2-neon.h LOAD4 macro.
+ let mut msg = [vdup_n_u32(0u32); 16];
+ for i in 0..4usize {
+ let in0 = vreinterpretq_u32_u8(vld1q_u8(data1.add(i * 16)));
+ let in1 = vreinterpretq_u32_u8(vld1q_u8(data2.add(i * 16)));
+ let zipped = vzipq_u32(in0, in1);
+ msg[i * 4] = vget_low_u32(zipped.0); // [data1[4i+0], data2[4i+0]]
+ msg[i * 4 + 1] = vget_high_u32(zipped.0); // [data1[4i+1], data2[4i+1]]
+ msg[i * 4 + 2] = vget_low_u32(zipped.1); // [data1[4i+2], data2[4i+2]]
+ msg[i * 4 + 3] = vget_high_u32(zipped.1); // [data1[4i+3], data2[4i+3]]
+ }
+
+ // Save initial state for the Davies-Meyer feed-forward at the end.
+ let oa = state.regs[0];
+ let ob = state.regs[1];
+ let oc = state.regs[2];
+ let od = state.regs[3];
+ let mut a = oa;
+ let mut b = ob;
+ let mut c = oc;
+ let mut d = od;
+
+ // 64 MD5 rounds, fully unrolled.
+ // Reference: par2cmdline-turbo/parpar/hasher/md5-base.h (64 RX() calls).
+ // K constants from RFC 1321 §3.4; rotation amounts per RFC 1321 §3.4.
+ //
+ // rx!(TYPE, accumulator, b, c, d, msg_word, K, R, 32-R)
+
+ // Round 0-15 (F)
+ rx!(F, a, b, c, d, msg[0], 0xd76aa478u32, 7, 25);
+ rx!(F, d, a, b, c, msg[1], 0xe8c7b756u32, 12, 20);
+ rx!(F, c, d, a, b, msg[2], 0x242070dbu32, 17, 15);
+ rx!(F, b, c, d, a, msg[3], 0xc1bdceeeu32, 22, 10);
+ rx!(F, a, b, c, d, msg[4], 0xf57c0fafu32, 7, 25);
+ rx!(F, d, a, b, c, msg[5], 0x4787c62au32, 12, 20);
+ rx!(F, c, d, a, b, msg[6], 0xa8304613u32, 17, 15);
+ rx!(F, b, c, d, a, msg[7], 0xfd469501u32, 22, 10);
+ rx!(F, a, b, c, d, msg[8], 0x698098d8u32, 7, 25);
+ rx!(F, d, a, b, c, msg[9], 0x8b44f7afu32, 12, 20);
+ rx!(F, c, d, a, b, msg[10], 0xffff5bb1u32, 17, 15);
+ rx!(F, b, c, d, a, msg[11], 0x895cd7beu32, 22, 10);
+ rx!(F, a, b, c, d, msg[12], 0x6b901122u32, 7, 25);
+ rx!(F, d, a, b, c, msg[13], 0xfd987193u32, 12, 20);
+ rx!(F, c, d, a, b, msg[14], 0xa679438eu32, 17, 15);
+ rx!(F, b, c, d, a, msg[15], 0x49b40821u32, 22, 10);
+
+ // Round 16-31 (G)
+ rx!(G, a, b, c, d, msg[1], 0xf61e2562u32, 5, 27);
+ rx!(G, d, a, b, c, msg[6], 0xc040b340u32, 9, 23);
+ rx!(G, c, d, a, b, msg[11], 0x265e5a51u32, 14, 18);
+ rx!(G, b, c, d, a, msg[0], 0xe9b6c7aau32, 20, 12);
+ rx!(G, a, b, c, d, msg[5], 0xd62f105du32, 5, 27);
+ rx!(G, d, a, b, c, msg[10], 0x02441453u32, 9, 23);
+ rx!(G, c, d, a, b, msg[15], 0xd8a1e681u32, 14, 18);
+ rx!(G, b, c, d, a, msg[4], 0xe7d3fbc8u32, 20, 12);
+ rx!(G, a, b, c, d, msg[9], 0x21e1cde6u32, 5, 27);
+ rx!(G, d, a, b, c, msg[14], 0xc33707d6u32, 9, 23);
+ rx!(G, c, d, a, b, msg[3], 0xf4d50d87u32, 14, 18);
+ rx!(G, b, c, d, a, msg[8], 0x455a14edu32, 20, 12);
+ rx!(G, a, b, c, d, msg[13], 0xa9e3e905u32, 5, 27);
+ rx!(G, d, a, b, c, msg[2], 0xfcefa3f8u32, 9, 23);
+ rx!(G, c, d, a, b, msg[7], 0x676f02d9u32, 14, 18);
+ rx!(G, b, c, d, a, msg[12], 0x8d2a4c8au32, 20, 12);
+
+ // Round 32-47 (H)
+ rx!(H, a, b, c, d, msg[5], 0xfffa3942u32, 4, 28);
+ rx!(H, d, a, b, c, msg[8], 0x8771f681u32, 11, 21);
+ rx!(H, c, d, a, b, msg[11], 0x6d9d6122u32, 16, 16);
+ rx!(H, b, c, d, a, msg[14], 0xfde5380cu32, 23, 9);
+ rx!(H, a, b, c, d, msg[1], 0xa4beea44u32, 4, 28);
+ rx!(H, d, a, b, c, msg[4], 0x4bdecfa9u32, 11, 21);
+ rx!(H, c, d, a, b, msg[7], 0xf6bb4b60u32, 16, 16);
+ rx!(H, b, c, d, a, msg[10], 0xbebfbc70u32, 23, 9);
+ rx!(H, a, b, c, d, msg[13], 0x289b7ec6u32, 4, 28);
+ rx!(H, d, a, b, c, msg[0], 0xeaa127fau32, 11, 21);
+ rx!(H, c, d, a, b, msg[3], 0xd4ef3085u32, 16, 16);
+ rx!(H, b, c, d, a, msg[6], 0x04881d05u32, 23, 9);
+ rx!(H, a, b, c, d, msg[9], 0xd9d4d039u32, 4, 28);
+ rx!(H, d, a, b, c, msg[12], 0xe6db99e5u32, 11, 21);
+ rx!(H, c, d, a, b, msg[15], 0x1fa27cf8u32, 16, 16);
+ rx!(H, b, c, d, a, msg[2], 0xc4ac5665u32, 23, 9);
+
+ // Round 48-63 (I)
+ rx!(I, a, b, c, d, msg[0], 0xf4292244u32, 6, 26);
+ rx!(I, d, a, b, c, msg[7], 0x432aff97u32, 10, 22);
+ rx!(I, c, d, a, b, msg[14], 0xab9423a7u32, 15, 17);
+ rx!(I, b, c, d, a, msg[5], 0xfc93a039u32, 21, 11);
+ rx!(I, a, b, c, d, msg[12], 0x655b59c3u32, 6, 26);
+ rx!(I, d, a, b, c, msg[3], 0x8f0ccc92u32, 10, 22);
+ rx!(I, c, d, a, b, msg[10], 0xffeff47du32, 15, 17);
+ rx!(I, b, c, d, a, msg[1], 0x85845dd1u32, 21, 11);
+ rx!(I, a, b, c, d, msg[8], 0x6fa87e4fu32, 6, 26);
+ rx!(I, d, a, b, c, msg[15], 0xfe2ce6e0u32, 10, 22);
+ rx!(I, c, d, a, b, msg[6], 0xa3014314u32, 15, 17);
+ rx!(I, b, c, d, a, msg[13], 0x4e0811a1u32, 21, 11);
+ rx!(I, a, b, c, d, msg[4], 0xf7537e82u32, 6, 26);
+ rx!(I, d, a, b, c, msg[11], 0xbd3af235u32, 10, 22);
+ rx!(I, c, d, a, b, msg[2], 0x2ad7d2bbu32, 15, 17);
+ rx!(I, b, c, d, a, msg[9], 0xeb86d391u32, 21, 11);
+
+ // Davies-Meyer feed-forward: add initial state to compressed state.
+ state.regs[0] = vadd_u32(oa, a);
+ state.regs[1] = vadd_u32(ob, b);
+ state.regs[2] = vadd_u32(oc, c);
+ state.regs[3] = vadd_u32(od, d);
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::parpar_hasher::md5x2::Md5x2;
+ use crate::parpar_hasher::md5x2_scalar::Scalar;
+
+ /// Verify NEON process_block produces the same digest as the scalar oracle
+ /// for an arbitrary pair of distinct 64-byte blocks.
+ ///
+ /// This test can only run on aarch64 hardware (or QEMU aarch64).
+ #[test]
+ fn neon_matches_scalar_oracle() {
+ // Two distinct 64-byte blocks.
+ let block1: [u8; 64] = core::array::from_fn(|i| i as u8);
+ let block2: [u8; 64] = core::array::from_fn(|i| (i as u8).wrapping_mul(3).wrapping_add(7));
+
+ // Compute with NEON backend.
+ let neon_digest = unsafe {
+ let mut state = State::init_state();
+ State::init_lane(&mut state, 0);
+ State::init_lane(&mut state, 1);
+ State::process_block(&mut state, block1.as_ptr(), block2.as_ptr());
+ (
+ State::extract_lane(&state, 0),
+ State::extract_lane(&state, 1),
+ )
+ };
+
+ // Compute with scalar oracle (two independent MD5 states).
+ let scalar_lane0 = unsafe {
+ let mut state = Scalar::init_state();
+ Scalar::init_lane(&mut state, 0);
+ Scalar::init_lane(&mut state, 1);
+ Scalar::process_block(&mut state, block1.as_ptr(), block1.as_ptr());
+ Scalar::extract_lane(&state, 0)
+ };
+ let scalar_lane1 = unsafe {
+ let mut state = Scalar::init_state();
+ Scalar::init_lane(&mut state, 0);
+ Scalar::init_lane(&mut state, 1);
+ Scalar::process_block(&mut state, block2.as_ptr(), block2.as_ptr());
+ Scalar::extract_lane(&state, 0)
+ };
+
+ assert_eq!(neon_digest.0, scalar_lane0, "NEON lane0 mismatch vs scalar");
+ assert_eq!(neon_digest.1, scalar_lane1, "NEON lane1 mismatch vs scalar");
+ }
+}
diff --git a/src/parpar_hasher/md5x2_scalar.rs b/src/parpar_hasher/md5x2_scalar.rs
new file mode 100644
index 00000000..bf43fb08
--- /dev/null
+++ b/src/parpar_hasher/md5x2_scalar.rs
@@ -0,0 +1,664 @@
+// Copyright (C) Anomaly Industries Inc. and par2rs contributors.
+// SPDX-License-Identifier: GPL-2.0-or-later
+//
+// Two-lane scalar (GPR) MD5 block compression — portable across architectures.
+//
+// On x86_64, uses hand-written inline assembly matching par2cmdline-turbo's
+// md5x2-x86-asm.h (AT&T syntax, mirrors the upstream round bodies one-for-one).
+//
+// On aarch64 and all other targets, uses a pure-Rust implementation ported
+// from par2cmdline-turbo's md5-base.h (non-MD5_USE_ASM path with MD5X2 defined).
+// rotate_left() lowers to a single `ror` on aarch64 via LLVM.
+//
+// The core trick: each MD5 round operates on independent state, so two
+// independent MD5 streams ("lane 1" and "lane 2") can be interleaved on a
+// single core and the out-of-order engine schedules them in parallel for
+// almost no extra cost vs a single MD5. ParPar uses this so the per-block
+// MD5 (lane 1) and the per-file MD5 (lane 2) advance from a single walk
+// over the input bytes.
+//
+// State layout (matches upstream):
+// state[0..4] = lane 1 (A1, B1, C1, D1)
+// state[4..8] = lane 2 (A2, B2, C2, D2)
+//
+// Faster tiers (SSE2, AVX-512 on x86_64; NEON on aarch64) layer on top
+// via runtime dispatch.
+
+#[cfg(target_arch = "x86_64")]
+use core::arch::asm;
+
+// ============================================================================
+// x86_64 asm path (matches par2cmdline-turbo/parpar/hasher/md5x2-x86-asm.h)
+// ============================================================================
+
+/// Process one 64-byte block for each of two MD5 lanes.
+///
+/// `state[0..4]` is lane 1 (A1, B1, C1, D1); `state[4..8]` is lane 2.
+/// `data1` and `data2` must each point to at least 64 readable bytes.
+///
+/// # Safety
+/// `data1` and `data2` must be valid for 64-byte reads. They may overlap
+/// (the upstream API is happy to be called with `data1 == data2`, which is
+/// how the very first block of a file ends up hashed identically into
+/// both lanes via the staggered-offset bookkeeping in `HasherInput`).
+#[cfg(target_arch = "x86_64")]
+#[inline]
+pub unsafe fn process_block_x2_scalar(state: &mut [u32; 8], data1: *const u8, data2: *const u8) {
+ // Pull state into Rust locals — the asm! blocks operate on these.
+ let mut a1 = state[0];
+ let mut b1 = state[1];
+ let mut c1 = state[2];
+ let mut d1 = state[3];
+ let mut a2 = state[4];
+ let mut b2 = state[5];
+ let mut c2 = state[6];
+ let mut d2 = state[7];
+
+ // Save initial A for the final fold-in. Upstream does `A1 += read32(_data[0])`
+ // up-front before round 0, which folds in input word 0; this is equivalent
+ // to absorbing the first input word into A as part of the F round 0 add.
+ // We mirror upstream exactly: pre-add input word 0 to each lane's A.
+ a1 = a1.wrapping_add(read32(data1, 0));
+ a2 = a2.wrapping_add(read32(data2, 0));
+
+ // The macros below translate the upstream ROUND_F / ROUND_G / ROUND_H /
+ // ROUND_I_INIT / ROUND_I / ROUND_I_LAST sequences. Each `round_*!`
+ // emits one asm! block whose instruction sequence is byte-for-byte the
+ // upstream macro body, with %k[X] register operands replaced by Rust
+ // template arguments and %[i0]/%[i1] memory operands replaced by
+ // `{i_off:e}({base})`-style displacements off the input base pointers.
+
+ // F round: TMP = (B & (C^D)) ^ D ; A = rol(A + F + K + Mi, R) + B
+ macro_rules! round_f {
+ ($a1:ident, $b1:ident, $c1:ident, $d1:ident,
+ $a2:ident, $b2:ident, $c2:ident, $d2:ident,
+ $i_off:expr, $k:expr, $r:expr) => {
+ asm!(
+ "movl {d1:e}, {tmp1:e}",
+ "movl {d2:e}, {tmp2:e}",
+ "addl ${k}, {a1:e}",
+ "addl ${k}, {a2:e}",
+ "xorl {c1:e}, {tmp1:e}",
+ "xorl {c2:e}, {tmp2:e}",
+ "andl {b1:e}, {tmp1:e}",
+ "andl {b2:e}, {tmp2:e}",
+ "xorl {d1:e}, {tmp1:e}",
+ "xorl {d2:e}, {tmp2:e}",
+ "addl {i_off}({base1}), {d1:e}",
+ "addl {i_off}({base2}), {d2:e}",
+ "addl {tmp1:e}, {a1:e}",
+ "addl {tmp2:e}, {a2:e}",
+ "roll ${r}, {a1:e}",
+ "roll ${r}, {a2:e}",
+ "addl {b1:e}, {a1:e}",
+ "addl {b2:e}, {a2:e}",
+ k = const ($k as u32 as i32),
+ r = const $r,
+ i_off = const ($i_off * 4),
+ a1 = inout(reg) $a1,
+ b1 = in(reg) $b1,
+ c1 = in(reg) $c1,
+ d1 = inout(reg) $d1,
+ a2 = inout(reg) $a2,
+ b2 = in(reg) $b2,
+ c2 = in(reg) $c2,
+ d2 = inout(reg) $d2,
+ tmp1 = out(reg) _,
+ tmp2 = out(reg) _,
+ base1 = in(reg) data1,
+ base2 = in(reg) data2,
+ options(att_syntax, nostack, readonly),
+ );
+ };
+ }
+
+ // G round: TMP = (~D & C) + (B & D) added in two stages (non-BMI1 path).
+ // Mirrors upstream non-BMI1 ROUND_G.
+ macro_rules! round_g {
+ ($a1:ident, $b1:ident, $c1:ident, $d1:ident,
+ $a2:ident, $b2:ident, $c2:ident, $d2:ident,
+ $i_off:expr, $k:expr, $r:expr) => {
+ asm!(
+ "movl {d1:e}, {tmp1:e}",
+ "movl {d2:e}, {tmp2:e}",
+ "addl ${k}, {a1:e}",
+ "addl ${k}, {a2:e}",
+ "notl {tmp1:e}",
+ "notl {tmp2:e}",
+ "andl {c1:e}, {tmp1:e}",
+ "andl {c2:e}, {tmp2:e}",
+ "addl {tmp1:e}, {a1:e}",
+ "addl {tmp2:e}, {a2:e}",
+ "movl {d1:e}, {tmp1:e}",
+ "movl {d2:e}, {tmp2:e}",
+ "addl {i_off}({base1}), {d1:e}",
+ "addl {i_off}({base2}), {d2:e}",
+ "andl {b1:e}, {tmp1:e}",
+ "andl {b2:e}, {tmp2:e}",
+ "addl {tmp1:e}, {a1:e}",
+ "addl {tmp2:e}, {a2:e}",
+ "roll ${r}, {a1:e}",
+ "roll ${r}, {a2:e}",
+ "addl {b1:e}, {a1:e}",
+ "addl {b2:e}, {a2:e}",
+ k = const ($k as u32 as i32),
+ r = const $r,
+ i_off = const ($i_off * 4),
+ a1 = inout(reg) $a1,
+ b1 = in(reg) $b1,
+ c1 = in(reg) $c1,
+ d1 = inout(reg) $d1,
+ a2 = inout(reg) $a2,
+ b2 = in(reg) $b2,
+ c2 = in(reg) $c2,
+ d2 = inout(reg) $d2,
+ tmp1 = out(reg) _,
+ tmp2 = out(reg) _,
+ base1 = in(reg) data1,
+ base2 = in(reg) data2,
+ options(att_syntax, nostack, readonly),
+ );
+ };
+ }
+
+ // H round: TMP = D ^ C ^ B (associative XOR sequence, D updated with input
+ // before the final XOR). Upstream notes "can't use H shortcut because D
+ // input is updated early".
+ macro_rules! round_h {
+ ($a1:ident, $b1:ident, $c1:ident, $d1:ident,
+ $a2:ident, $b2:ident, $c2:ident, $d2:ident,
+ $i_off:expr, $k:expr, $r:expr) => {
+ asm!(
+ "movl {d1:e}, {tmp1:e}",
+ "movl {d2:e}, {tmp2:e}",
+ "addl ${k}, {a1:e}",
+ "addl ${k}, {a2:e}",
+ "xorl {c1:e}, {tmp1:e}",
+ "xorl {c2:e}, {tmp2:e}",
+ "addl {i_off}({base1}), {d1:e}",
+ "addl {i_off}({base2}), {d2:e}",
+ "xorl {b1:e}, {tmp1:e}",
+ "xorl {b2:e}, {tmp2:e}",
+ "addl {tmp1:e}, {a1:e}",
+ "addl {tmp2:e}, {a2:e}",
+ "roll ${r}, {a1:e}",
+ "roll ${r}, {a2:e}",
+ "addl {b1:e}, {a1:e}",
+ "addl {b2:e}, {a2:e}",
+ k = const ($k as u32 as i32),
+ r = const $r,
+ i_off = const ($i_off * 4),
+ a1 = inout(reg) $a1,
+ b1 = in(reg) $b1,
+ c1 = in(reg) $c1,
+ d1 = inout(reg) $d1,
+ a2 = inout(reg) $a2,
+ b2 = in(reg) $b2,
+ c2 = in(reg) $c2,
+ d2 = inout(reg) $d2,
+ tmp1 = out(reg) _,
+ tmp2 = out(reg) _,
+ base1 = in(reg) data1,
+ base2 = in(reg) data2,
+ options(att_syntax, nostack, readonly),
+ );
+ };
+ }
+
+ // I round: TMP_INIT = (~D | B); then TMP ^= C ; A += TMP ; D += Mi ; rol; +B.
+ // (non-BMI1 path)
+ macro_rules! round_i {
+ ($a1:ident, $b1:ident, $c1:ident, $d1:ident,
+ $a2:ident, $b2:ident, $c2:ident, $d2:ident,
+ $i_off:expr, $k:expr, $r:expr) => {
+ asm!(
+ "movl {d1:e}, {tmp1:e}",
+ "movl {d2:e}, {tmp2:e}",
+ "addl ${k}, {a1:e}",
+ "addl ${k}, {a2:e}",
+ "notl {tmp1:e}",
+ "notl {tmp2:e}",
+ "orl {b1:e}, {tmp1:e}",
+ "orl {b2:e}, {tmp2:e}",
+ "xorl {c1:e}, {tmp1:e}",
+ "xorl {c2:e}, {tmp2:e}",
+ "addl {i_off}({base1}), {d1:e}",
+ "addl {i_off}({base2}), {d2:e}",
+ "addl {tmp1:e}, {a1:e}",
+ "addl {tmp2:e}, {a2:e}",
+ "roll ${r}, {a1:e}",
+ "roll ${r}, {a2:e}",
+ "addl {b1:e}, {a1:e}",
+ "addl {b2:e}, {a2:e}",
+ k = const ($k as u32 as i32),
+ r = const $r,
+ i_off = const ($i_off * 4),
+ a1 = inout(reg) $a1,
+ b1 = in(reg) $b1,
+ c1 = in(reg) $c1,
+ d1 = inout(reg) $d1,
+ a2 = inout(reg) $a2,
+ b2 = in(reg) $b2,
+ c2 = in(reg) $c2,
+ d2 = inout(reg) $d2,
+ tmp1 = out(reg) _,
+ tmp2 = out(reg) _,
+ base1 = in(reg) data1,
+ base2 = in(reg) data2,
+ options(att_syntax, nostack, readonly),
+ );
+ };
+ }
+
+ // ROUND_I_LAST: same as ROUND_I but without the input add into D
+ // (it's the final round, no next input word to fold).
+ macro_rules! round_i_last {
+ ($a1:ident, $b1:ident, $c1:ident, $d1:ident,
+ $a2:ident, $b2:ident, $c2:ident, $d2:ident,
+ $k:expr, $r:expr) => {
+ asm!(
+ "movl {d1:e}, {tmp1:e}",
+ "movl {d2:e}, {tmp2:e}",
+ "addl ${k}, {a1:e}",
+ "addl ${k}, {a2:e}",
+ "notl {tmp1:e}",
+ "notl {tmp2:e}",
+ "orl {b1:e}, {tmp1:e}",
+ "orl {b2:e}, {tmp2:e}",
+ "xorl {c1:e}, {tmp1:e}",
+ "xorl {c2:e}, {tmp2:e}",
+ "addl {tmp1:e}, {a1:e}",
+ "addl {tmp2:e}, {a2:e}",
+ "roll ${r}, {a1:e}",
+ "roll ${r}, {a2:e}",
+ "addl {b1:e}, {a1:e}",
+ "addl {b2:e}, {a2:e}",
+ k = const ($k as u32 as i32),
+ r = const $r,
+ a1 = inout(reg) $a1,
+ b1 = in(reg) $b1,
+ c1 = in(reg) $c1,
+ d1 = in(reg) $d1,
+ a2 = inout(reg) $a2,
+ b2 = in(reg) $b2,
+ c2 = in(reg) $c2,
+ d2 = in(reg) $d2,
+ tmp1 = out(reg) _,
+ tmp2 = out(reg) _,
+ options(att_syntax, nostack, readonly),
+ );
+ };
+ }
+
+ // RF4 / RG4 / RH4 / RI4 expand to four rounds with rotated register roles.
+
+ // Round 1 (F)
+ round_f!(a1, b1, c1, d1, a2, b2, c2, d2, 1, -0x28955b88_i32, 7);
+ round_f!(d1, a1, b1, c1, d2, a2, b2, c2, 2, -0x173848aa_i32, 12);
+ round_f!(c1, d1, a1, b1, c2, d2, a2, b2, 3, 0x242070db_i32, 17);
+ round_f!(b1, c1, d1, a1, b2, c2, d2, a2, 4, -0x3e423112_i32, 22);
+
+ round_f!(a1, b1, c1, d1, a2, b2, c2, d2, 5, -0x0a83f051_i32, 7);
+ round_f!(d1, a1, b1, c1, d2, a2, b2, c2, 6, 0x4787c62a_i32, 12);
+ round_f!(c1, d1, a1, b1, c2, d2, a2, b2, 7, -0x57cfb9ed_i32, 17);
+ round_f!(b1, c1, d1, a1, b2, c2, d2, a2, 8, -0x02b96aff_i32, 22);
+
+ round_f!(a1, b1, c1, d1, a2, b2, c2, d2, 9, 0x698098d8_i32, 7);
+ round_f!(d1, a1, b1, c1, d2, a2, b2, c2, 10, -0x74bb0851_i32, 12);
+ round_f!(c1, d1, a1, b1, c2, d2, a2, b2, 11, -0x0000a44f_i32, 17);
+ round_f!(b1, c1, d1, a1, b2, c2, d2, a2, 12, -0x76a32842_i32, 22);
+
+ round_f!(a1, b1, c1, d1, a2, b2, c2, d2, 13, 0x6b901122_i32, 7);
+ round_f!(d1, a1, b1, c1, d2, a2, b2, c2, 14, -0x02678e6d_i32, 12);
+ round_f!(c1, d1, a1, b1, c2, d2, a2, b2, 15, -0x5986bc72_i32, 17);
+ round_f!(b1, c1, d1, a1, b2, c2, d2, a2, 1, 0x49b40821_i32, 22);
+
+ // Round 2 (G)
+ round_g!(a1, b1, c1, d1, a2, b2, c2, d2, 6, -0x09e1da9e_i32, 5);
+ round_g!(d1, a1, b1, c1, d2, a2, b2, c2, 11, -0x3fbf4cc0_i32, 9);
+ round_g!(c1, d1, a1, b1, c2, d2, a2, b2, 0, 0x265e5a51_i32, 14);
+ round_g!(b1, c1, d1, a1, b2, c2, d2, a2, 5, -0x16493856_i32, 20);
+
+ round_g!(a1, b1, c1, d1, a2, b2, c2, d2, 10, -0x29d0efa3_i32, 5);
+ round_g!(d1, a1, b1, c1, d2, a2, b2, c2, 15, 0x02441453_i32, 9);
+ round_g!(c1, d1, a1, b1, c2, d2, a2, b2, 4, -0x275e197f_i32, 14);
+ round_g!(b1, c1, d1, a1, b2, c2, d2, a2, 9, -0x182c0438_i32, 20);
+
+ round_g!(a1, b1, c1, d1, a2, b2, c2, d2, 14, 0x21e1cde6_i32, 5);
+ round_g!(d1, a1, b1, c1, d2, a2, b2, c2, 3, -0x3cc8f82a_i32, 9);
+ round_g!(c1, d1, a1, b1, c2, d2, a2, b2, 8, -0x0b2af279_i32, 14);
+ round_g!(b1, c1, d1, a1, b2, c2, d2, a2, 13, 0x455a14ed_i32, 20);
+
+ round_g!(a1, b1, c1, d1, a2, b2, c2, d2, 2, -0x561c16fb_i32, 5);
+ round_g!(d1, a1, b1, c1, d2, a2, b2, c2, 7, -0x03105c08_i32, 9);
+ round_g!(c1, d1, a1, b1, c2, d2, a2, b2, 12, 0x676f02d9_i32, 14);
+ round_g!(b1, c1, d1, a1, b2, c2, d2, a2, 5, -0x72d5b376_i32, 20);
+
+ // Round 3 (H)
+ round_h!(a1, b1, c1, d1, a2, b2, c2, d2, 8, -0x0005c6be_i32, 4);
+ round_h!(d1, a1, b1, c1, d2, a2, b2, c2, 11, -0x788e097f_i32, 11);
+ round_h!(c1, d1, a1, b1, c2, d2, a2, b2, 14, 0x6d9d6122_i32, 16);
+ round_h!(b1, c1, d1, a1, b2, c2, d2, a2, 1, -0x021ac7f4_i32, 23);
+
+ round_h!(a1, b1, c1, d1, a2, b2, c2, d2, 4, -0x5b4115bc_i32, 4);
+ round_h!(d1, a1, b1, c1, d2, a2, b2, c2, 7, 0x4bdecfa9_i32, 11);
+ round_h!(c1, d1, a1, b1, c2, d2, a2, b2, 10, -0x0944b4a0_i32, 16);
+ round_h!(b1, c1, d1, a1, b2, c2, d2, a2, 13, -0x41404390_i32, 23);
+
+ round_h!(a1, b1, c1, d1, a2, b2, c2, d2, 0, 0x289b7ec6_i32, 4);
+ round_h!(d1, a1, b1, c1, d2, a2, b2, c2, 3, -0x155ed806_i32, 11);
+ round_h!(c1, d1, a1, b1, c2, d2, a2, b2, 6, -0x2b10cf7b_i32, 16);
+ round_h!(b1, c1, d1, a1, b2, c2, d2, a2, 9, 0x04881d05_i32, 23);
+
+ round_h!(a1, b1, c1, d1, a2, b2, c2, d2, 12, -0x262b2fc7_i32, 4);
+ round_h!(d1, a1, b1, c1, d2, a2, b2, c2, 15, -0x1924661b_i32, 11);
+ round_h!(c1, d1, a1, b1, c2, d2, a2, b2, 2, 0x1fa27cf8_i32, 16);
+ round_h!(b1, c1, d1, a1, b2, c2, d2, a2, 0, -0x3b53a99b_i32, 23);
+
+ // Round 4 (I)
+ round_i!(a1, b1, c1, d1, a2, b2, c2, d2, 7, -0x0bd6ddbc_i32, 6);
+ round_i!(d1, a1, b1, c1, d2, a2, b2, c2, 14, 0x432aff97_i32, 10);
+ round_i!(c1, d1, a1, b1, c2, d2, a2, b2, 5, -0x546bdc59_i32, 15);
+ round_i!(b1, c1, d1, a1, b2, c2, d2, a2, 12, -0x036c5fc7_i32, 21);
+
+ round_i!(a1, b1, c1, d1, a2, b2, c2, d2, 3, 0x655b59c3_i32, 6);
+ round_i!(d1, a1, b1, c1, d2, a2, b2, c2, 10, -0x70f3336e_i32, 10);
+ round_i!(c1, d1, a1, b1, c2, d2, a2, b2, 1, -0x00100b83_i32, 15);
+ round_i!(b1, c1, d1, a1, b2, c2, d2, a2, 8, -0x7a7ba22f_i32, 21);
+
+ round_i!(a1, b1, c1, d1, a2, b2, c2, d2, 15, 0x6fa87e4f_i32, 6);
+ round_i!(d1, a1, b1, c1, d2, a2, b2, c2, 6, -0x01d31920_i32, 10);
+ round_i!(c1, d1, a1, b1, c2, d2, a2, b2, 13, -0x5cfebcec_i32, 15);
+ round_i!(b1, c1, d1, a1, b2, c2, d2, a2, 4, 0x4e0811a1_i32, 21);
+
+ round_i!(a1, b1, c1, d1, a2, b2, c2, d2, 11, -0x08ac817e_i32, 6);
+ round_i!(d1, a1, b1, c1, d2, a2, b2, c2, 2, -0x42c50dcb_i32, 10);
+ round_i!(c1, d1, a1, b1, c2, d2, a2, b2, 9, 0x2ad7d2bb_i32, 15);
+ round_i_last!(b1, c1, d1, a1, b2, c2, d2, a2, -0x14792c6f_i32, 21);
+
+ state[0] = state[0].wrapping_add(a1);
+ state[1] = state[1].wrapping_add(b1);
+ state[2] = state[2].wrapping_add(c1);
+ state[3] = state[3].wrapping_add(d1);
+ state[4] = state[4].wrapping_add(a2);
+ state[5] = state[5].wrapping_add(b2);
+ state[6] = state[6].wrapping_add(c2);
+ state[7] = state[7].wrapping_add(d2);
+}
+
+// ============================================================================
+// Portable pure-Rust path (aarch64 and other non-x86_64 targets)
+// Reference: par2cmdline-turbo/parpar/hasher/md5-base.h (non-MD5_USE_ASM path)
+// with MD5X2 defined — two independent lanes run in lock-step.
+// Compiler will lower rotate_left() to a single `ror` on aarch64.
+// ============================================================================
+
+#[cfg(not(target_arch = "x86_64"))]
+#[inline]
+pub unsafe fn process_block_x2_scalar(state: &mut [u32; 8], data1: *const u8, data2: *const u8) {
+ // MD5 per-round constants (from RFC 1321 Table T).
+ #[rustfmt::skip]
+ const K: [u32; 64] = [
+ // Round 0 – F
+ 0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee,
+ 0xf57c0faf, 0x4787c62a, 0xa8304613, 0xfd469501,
+ 0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be,
+ 0x6b901122, 0xfd987193, 0xa679438e, 0x49b40821,
+ // Round 1 – G
+ 0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa,
+ 0xd62f105d, 0x02441453, 0xd8a1e681, 0xe7d3fbc8,
+ 0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed,
+ 0xa9e3e905, 0xfcefa3f8, 0x676f02d9, 0x8d2a4c8a,
+ // Round 2 – H
+ 0xfffa3942, 0x8771f681, 0x6d9d6122, 0xfde5380c,
+ 0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70,
+ 0x289b7ec6, 0xeaa127fa, 0xd4ef3085, 0x04881d05,
+ 0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665,
+ // Round 3 – I
+ 0xf4292244, 0x432aff97, 0xab9423a7, 0xfc93a039,
+ 0x655b59c3, 0x8f0ccc92, 0xffeff47d, 0x85845dd1,
+ 0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1,
+ 0xf7537e82, 0xbd3af235, 0x2ad7d2bb, 0xeb86d391,
+ ];
+ // Per-round left-rotation amounts (RFC 1321 §3.4).
+ #[rustfmt::skip]
+ const R: [u32; 64] = [
+ 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22,
+ 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20,
+ 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23,
+ 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21,
+ ];
+
+ // Load 16 little-endian words from each input buffer.
+ let mut m1 = [0u32; 16];
+ let mut m2 = [0u32; 16];
+ for i in 0..16 {
+ m1[i] = read32(data1, i);
+ m2[i] = read32(data2, i);
+ }
+
+ let (mut a1, mut b1, mut c1, mut d1) = (state[0], state[1], state[2], state[3]);
+ let (mut a2, mut b2, mut c2, mut d2) = (state[4], state[5], state[6], state[7]);
+
+ // 64 rounds — identical logic for each lane, run in parallel.
+ // The compiler interleaves the two instruction streams for free OoO throughput.
+ for i in 0..64usize {
+ // Message-word index for this round (RFC 1321 §3.4).
+ let g = match i {
+ 0..=15 => i,
+ 16..=31 => (5 * i + 1) % 16,
+ 32..=47 => (3 * i + 5) % 16,
+ _ => (7 * i) % 16,
+ };
+
+ // Mix function and rotate-add-B for each lane.
+ // F = ((c ^ d) & b) ^ d (equivalent to (b & c) | (!b & d))
+ // G = (d & b) | (!d & c)
+ // H = d ^ c ^ b
+ // I = c ^ (b | !d)
+ let (f1, f2) = match i {
+ 0..=15 => (((c1 ^ d1) & b1) ^ d1, ((c2 ^ d2) & b2) ^ d2),
+ 16..=31 => ((d1 & b1) | (!d1 & c1), (d2 & b2) | (!d2 & c2)),
+ 32..=47 => (d1 ^ c1 ^ b1, d2 ^ c2 ^ b2),
+ _ => (c1 ^ (b1 | !d1), c2 ^ (b2 | !d2)),
+ };
+
+ // _RX: a = (a + f + K[i] + M[g]).rotate_left(R[i]) + b
+ // then rotate state: (a,b,c,d) → (d, a+…, b, c)
+ let na1 = a1
+ .wrapping_add(f1)
+ .wrapping_add(K[i])
+ .wrapping_add(m1[g])
+ .rotate_left(R[i])
+ .wrapping_add(b1);
+ let na2 = a2
+ .wrapping_add(f2)
+ .wrapping_add(K[i])
+ .wrapping_add(m2[g])
+ .rotate_left(R[i])
+ .wrapping_add(b2);
+ a1 = d1;
+ d1 = c1;
+ c1 = b1;
+ b1 = na1;
+ a2 = d2;
+ d2 = c2;
+ c2 = b2;
+ b2 = na2;
+ }
+
+ state[0] = state[0].wrapping_add(a1);
+ state[1] = state[1].wrapping_add(b1);
+ state[2] = state[2].wrapping_add(c1);
+ state[3] = state[3].wrapping_add(d1);
+ state[4] = state[4].wrapping_add(a2);
+ state[5] = state[5].wrapping_add(b2);
+ state[6] = state[6].wrapping_add(c2);
+ state[7] = state[7].wrapping_add(d2);
+}
+
+#[inline]
+unsafe fn read32(p: *const u8, word_idx: usize) -> u32 {
+ let off = word_idx * 4;
+ let bytes = core::ptr::read_unaligned(p.add(off) as *const [u8; 4]);
+ u32::from_le_bytes(bytes)
+}
+
+/// Initialise both lanes of an MD5x2 state to the standard MD5 IV.
+#[inline]
+pub fn init_state() -> [u32; 8] {
+ [
+ 0x67452301, 0xefcdab89, 0x98badcfe, 0x10325476, // lane 1
+ 0x67452301, 0xefcdab89, 0x98badcfe, 0x10325476, // lane 2
+ ]
+}
+
+/// Reset only one lane of an existing MD5x2 state. Used between blocks
+/// for the per-block lane while the per-file lane keeps accumulating.
+#[inline]
+pub fn init_lane(state: &mut [u32; 8], lane: usize) {
+ debug_assert!(lane < 2);
+ let off = lane * 4;
+ state[off] = 0x67452301;
+ state[off + 1] = 0xefcdab89;
+ state[off + 2] = 0x98badcfe;
+ state[off + 3] = 0x10325476;
+}
+
+/// `Md5x2` backend wrapper for the scalar implementation. Used as the
+/// portable fallback and as a correctness oracle for the SSE2 backend.
+pub struct Scalar;
+
+impl crate::parpar_hasher::md5x2::Md5x2 for Scalar {
+ type State = [u32; 8];
+
+ #[inline]
+ fn init_state() -> Self::State {
+ init_state()
+ }
+
+ #[inline]
+ fn init_lane(state: &mut Self::State, lane: usize) {
+ init_lane(state, lane);
+ }
+
+ #[inline]
+ fn extract_lane(state: &Self::State, lane: usize) -> [u8; 16] {
+ let off = lane * 4;
+ let mut out = [0u8; 16];
+ for i in 0..4 {
+ out[i * 4..i * 4 + 4].copy_from_slice(&state[off + i].to_le_bytes());
+ }
+ out
+ }
+
+ #[inline]
+ unsafe fn process_block(state: &mut Self::State, data1: *const u8, data2: *const u8) {
+ process_block_x2_scalar(state, data1, data2)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// Run our two-lane block compressor over a single 64-byte block of
+ /// each of two messages and compare both lanes against a portable
+ /// reference MD5 block compressor (transcribed from RFC 1321 below).
+ fn check_two_messages(msg1: &[u8; 64], msg2: &[u8; 64]) {
+ let mut state = init_state();
+ unsafe {
+ process_block_x2_scalar(&mut state, msg1.as_ptr(), msg2.as_ptr());
+ }
+
+ for (lane, msg) in [msg1, msg2].iter().enumerate() {
+ let mut ref_state = [0x67452301u32, 0xefcdab89, 0x98badcfe, 0x10325476];
+ md5_compress_one_block_reference(&mut ref_state, msg);
+
+ let our_lane: [u32; 4] = [
+ state[lane * 4],
+ state[lane * 4 + 1],
+ state[lane * 4 + 2],
+ state[lane * 4 + 3],
+ ];
+ assert_eq!(our_lane, ref_state, "lane {lane} mismatch");
+ }
+ }
+
+ /// Plain portable MD5 block compressor used as the test oracle. This
+ /// is RFC 1321 transcribed straight to Rust and is intentionally
+ /// independent of our asm! port so a bug in the asm! port can't hide
+ /// behind the same bug in the oracle.
+ fn md5_compress_one_block_reference(state: &mut [u32; 4], block: &[u8; 64]) {
+ const K: [u32; 64] = [
+ 0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee, 0xf57c0faf, 0x4787c62a, 0xa8304613,
+ 0xfd469501, 0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be, 0x6b901122, 0xfd987193,
+ 0xa679438e, 0x49b40821, 0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa, 0xd62f105d,
+ 0x02441453, 0xd8a1e681, 0xe7d3fbc8, 0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed,
+ 0xa9e3e905, 0xfcefa3f8, 0x676f02d9, 0x8d2a4c8a, 0xfffa3942, 0x8771f681, 0x6d9d6122,
+ 0xfde5380c, 0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70, 0x289b7ec6, 0xeaa127fa,
+ 0xd4ef3085, 0x04881d05, 0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665, 0xf4292244,
+ 0x432aff97, 0xab9423a7, 0xfc93a039, 0x655b59c3, 0x8f0ccc92, 0xffeff47d, 0x85845dd1,
+ 0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1, 0xf7537e82, 0xbd3af235, 0x2ad7d2bb,
+ 0xeb86d391,
+ ];
+ const R: [u32; 64] = [
+ 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 5, 9, 14, 20, 5, 9, 14, 20,
+ 5, 9, 14, 20, 5, 9, 14, 20, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23,
+ 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21,
+ ];
+ let mut m = [0u32; 16];
+ for i in 0..16 {
+ m[i] = u32::from_le_bytes(block[i * 4..i * 4 + 4].try_into().unwrap());
+ }
+ let mut a = state[0];
+ let mut b = state[1];
+ let mut c = state[2];
+ let mut d = state[3];
+ for i in 0..64 {
+ let (f, g) = match i {
+ 0..=15 => ((b & c) | (!b & d), i),
+ 16..=31 => ((d & b) | (!d & c), (5 * i + 1) % 16),
+ 32..=47 => (b ^ c ^ d, (3 * i + 5) % 16),
+ _ => (c ^ (b | !d), (7 * i) % 16),
+ };
+ let t = d;
+ d = c;
+ c = b;
+ b = b.wrapping_add(
+ a.wrapping_add(f)
+ .wrapping_add(K[i])
+ .wrapping_add(m[g])
+ .rotate_left(R[i]),
+ );
+ a = t;
+ }
+ state[0] = state[0].wrapping_add(a);
+ state[1] = state[1].wrapping_add(b);
+ state[2] = state[2].wrapping_add(c);
+ state[3] = state[3].wrapping_add(d);
+ }
+
+ #[test]
+ fn zero_blocks() {
+ check_two_messages(&[0u8; 64], &[0u8; 64]);
+ }
+
+ #[test]
+ fn ascending_blocks() {
+ let mut a = [0u8; 64];
+ let mut b = [0u8; 64];
+ for i in 0..64 {
+ a[i] = i as u8;
+ b[i] = (255 - i) as u8;
+ }
+ check_two_messages(&a, &b);
+ }
+
+ #[test]
+ fn distinct_messages() {
+ let a = *b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@";
+ let b = *b"The quick brown fox jumps over the lazy dog. Pack my box with fi";
+ check_two_messages(&a, &b);
+ }
+}
diff --git a/src/parpar_hasher/md5x2_sse2.rs b/src/parpar_hasher/md5x2_sse2.rs
new file mode 100644
index 00000000..d1b49383
--- /dev/null
+++ b/src/parpar_hasher/md5x2_sse2.rs
@@ -0,0 +1,522 @@
+// Copyright (C) Anomaly Industries Inc. and par2rs contributors.
+// SPDX-License-Identifier: GPL-2.0-or-later
+//
+// SSE2 two-lane MD5 block compression for x86_64.
+//
+// 1:1 Rust port of the SSE2 specialisation produced by:
+// parpar/hasher/md5x2-sse.h (provides ROTATE, ADDF, init_lane, extract macros)
+// parpar/hasher/md5x2-base.h (provides the round expansion machinery)
+// parpar/hasher/md5-base.h (provides the actual 64-round body)
+// from par2cmdline-turbo (https://github.com/animetosho/par2cmdline-turbo)
+// and upstream ParPar (https://github.com/animetosho/ParPar), both
+// GPL-2.0-or-later.
+//
+// State layout (matches upstream `state_[i]` where `state_` is `__m128i*`):
+//
+// state[0] xmm holding A: [A_lane0 (32b), GARBAGE (32b), A_lane1 (32b), GARBAGE (32b)]
+// state[1] xmm holding B: [B_lane0, GARBAGE, B_lane1, GARBAGE]
+// state[2] xmm holding C: [C_lane0, GARBAGE, C_lane1, GARBAGE]
+// state[3] xmm holding D: [D_lane0, GARBAGE, D_lane1, GARBAGE]
+//
+// Each MD5 word lives in 32 bits; SSE2 holds two of them per xmm in
+// 32-bit lanes 0 and 2 (the qword-low halves of each qword), with the
+// other two 32-bit lanes containing don't-care bits. The "rotate by r"
+// trick from upstream:
+//
+// ROTATE(a, r) = _mm_srli_epi64(_mm_shuffle_epi32(a, _MM_SHUFFLE(2,2,0,0)), 32-r)
+//
+// The shuffle duplicates each valid 32-bit word into both 32-bit halves
+// of the 64-bit qword that holds it ([X, ?, Y, ?] -> [X, X, Y, Y]).
+// `srli_epi64(.., 32-r)` then logically shifts each 64-bit qword right
+// by `32-r`, producing `[ROL(X, r), 0, ROL(Y, r), 0]` (the low 32 bits
+// of each qword are the desired rotated result; the high 32 bits become
+// "GARBAGE" for the next round, which is fine — only 32-bit lanes 0
+// and 2 ever carry signal).
+//
+// The four MD5 bool functions (F, G, H, I) are spelled exactly as
+// upstream's `ADDF` macro:
+//
+// F(b, c, d) = ((c ^ d) & b) ^ d
+// G(b, c, d) = (~d & c) + (d & b) /* note: ADD, not OR */
+// H(b, c, d) = d ^ c ^ b
+// I(b, c, d) = (~d | b) ^ c
+//
+// Per-round update body (`_RX` from md5-base.h, with `ADDF` from
+// md5x2-sse.h's SSE2 block):
+//
+// a = a + ik /* ik = INPUT(k, ...) = m[i] + K[i] (constant + data word) */
+// a = a + f(b,c,d) /* via ADDF, which folds in the f directly */
+// a = ROTATE(a, r)
+// a = a + b
+//
+// Round 0 (F) uses pre-loaded data words from `LOAD4`; rounds 1-3 use
+// the same `XX0..XX15` state, since MD5 reuses the same 16 message
+// words across rounds in shuffled orders.
+
+#![cfg(target_arch = "x86_64")]
+#![allow(non_snake_case)] // mirror upstream variable names A, B, C, D, XX0..XX15
+
+use core::arch::x86_64::{
+ __m128i, _mm_add_epi32, _mm_and_si128, _mm_andnot_si128, _mm_loadu_si128, _mm_set1_epi32,
+ _mm_set_epi32, _mm_shuffle_epi32, _mm_srli_epi64, _mm_storeu_si128, _mm_unpackhi_epi64,
+ _mm_unpacklo_epi64, _mm_xor_si128,
+};
+
+/// Internal SSE2 state: four 128-bit registers, each carrying both
+/// lanes' matching MD5 word in 32-bit positions 0 and 2.
+#[derive(Clone, Copy)]
+#[repr(C, align(16))]
+pub struct State(pub [__m128i; 4]);
+
+impl State {
+ /// Build the SSE2 state from a flat `[u32; 8]` (`[a0,b0,c0,d0,a1,b1,c1,d1]`).
+ /// Used at backend boundaries: init / lane reset / extract.
+ ///
+ /// Made `pub(crate)` so the AVX-512VL backend can share the exact
+ /// same XMM lane layout (`md5_extract_x2_avx512 = md5_extract_x2_sse`
+ /// in upstream `md5x2-sse.h`).
+ #[inline(always)]
+ pub(crate) unsafe fn from_flat(flat: &[u32; 8]) -> Self {
+ // Each xmm holds [word_lane0, 0, word_lane1, 0].
+ // _mm_set_epi32 takes args in (e3, e2, e1, e0) order — i.e.
+ // bits 96..128, 64..96, 32..64, 0..32 — so to get
+ // [lane0, 0, lane1, 0] in positions (e0, e1, e2, e3) we pass
+ // _mm_set_epi32(0, lane1, 0, lane0).
+ let a = _mm_set_epi32(0, flat[4] as i32, 0, flat[0] as i32);
+ let b = _mm_set_epi32(0, flat[5] as i32, 0, flat[1] as i32);
+ let c = _mm_set_epi32(0, flat[6] as i32, 0, flat[2] as i32);
+ let d = _mm_set_epi32(0, flat[7] as i32, 0, flat[3] as i32);
+ State([a, b, c, d])
+ }
+
+ /// Convert back to flat `[u32; 8]` form by reading 32-bit lanes 0
+ /// and 2 of each register. The "GARBAGE" lanes 1 and 3 are discarded.
+ ///
+ /// `pub(crate)` for the same reason as `from_flat` — the AVX-512VL
+ /// backend reuses this layout verbatim.
+ #[inline(always)]
+ pub(crate) unsafe fn to_flat(&self) -> [u32; 8] {
+ let mut tmp = [0u32; 4];
+ let mut out = [0u32; 8];
+ for (idx, reg) in self.0.iter().enumerate() {
+ _mm_storeu_si128(tmp.as_mut_ptr() as *mut __m128i, *reg);
+ out[idx] = tmp[0]; // lane 0 -> [a0,b0,c0,d0]
+ out[idx + 4] = tmp[2]; // lane 1 -> [a1,b1,c1,d1]
+ }
+ out
+ }
+}
+
+/// 32-bit constant broadcast into both valid (and both garbage) slots.
+/// Direct port of upstream `VAL(k) = _mm_set1_epi32(k)`.
+#[inline(always)]
+#[allow(unused_unsafe)]
+unsafe fn val(k: u32) -> __m128i {
+ _mm_set1_epi32(k as i32)
+}
+
+/// Upstream `ROTATE(a, r) = _mm_srli_epi64(_mm_shuffle_epi32(a, 2200), 32-r)`.
+///
+/// `_mm_srli_epi64` requires its shift count as an immediate, and stable
+/// Rust's const-generic arithmetic (`{ 32 - R }`) isn't allowed in const
+/// position, so the macro callers pass the *already-subtracted* value
+/// `S = 32 - r` directly.
+#[inline(always)]
+#[allow(unused_unsafe)]
+unsafe fn rotate(a: __m128i) -> __m128i {
+ // _MM_SHUFFLE(2,2,0,0) = (2<<6)|(2<<4)|(0<<2)|0 = 0xa0
+ let dup = _mm_shuffle_epi32(a, 0xa0);
+ _mm_srli_epi64::(dup)
+}
+
+/// LOAD4 for one source pointer: load 16 contiguous bytes (= 4 input words),
+/// returning four xmm regs each shaped `[word_X, GARBAGE, word_X', GARBAGE]`
+/// where `_X` and `_X'` are the corresponding words from data1 and data2.
+///
+/// Direct port of `LOAD4` macro (md5x2-sse.h:13-20). Loads 4 input words
+/// from each of `ptr0` and `ptr1` at byte offset `idx*4`, interleaves
+/// them into MD5x2 layout, and returns (XX_idx, XX_idx+1, XX_idx+2, XX_idx+3).
+// `pub(crate)` so the AVX-512VL backend can reuse the same load+shuffle
+// path: upstream's `LOAD4` is shared between SSE and AVX-512 codepaths
+// (`md5x2-sse.h` defines `LOAD4` once at file scope and the AVX-512VL
+// block doesn't redefine it).
+#[inline(always)]
+#[allow(unused_unsafe)]
+pub(crate) unsafe fn load4(
+ ptr0: *const u8,
+ ptr1: *const u8,
+ idx: usize,
+) -> (__m128i, __m128i, __m128i, __m128i) {
+ let in0 = _mm_loadu_si128(ptr0.add(idx * 4) as *const __m128i);
+ let in1 = _mm_loadu_si128(ptr1.add(idx * 4) as *const __m128i);
+ // var0 = unpacklo(in0, in1) = [in0_w0, in1_w0, in0_w1, in1_w1]
+ // -> 32-bit lanes (0,1,2,3) where (0,2) carry the two lanes' word0/word1 alternately.
+ // Wait — that's not the upstream layout. Re-read md5x2-sse.h:14-19:
+ //
+ // in0 = loadu(ptr0+idx*4) [in0_w0, in0_w1, in0_w2, in0_w3]
+ // in1 = loadu(ptr1+idx*4) [in1_w0, in1_w1, in1_w2, in1_w3]
+ // var0 = unpacklo_epi64(in0, in1) [in0_w0, in0_w1, in1_w0, in1_w1]
+ // var1 = shuffle_epi32(var0, 2,3,0,1) [in0_w1, in0_w0, in1_w1, in1_w0]
+ // var2 = unpackhi_epi64(in0, in1) [in0_w2, in0_w3, in1_w2, in1_w3]
+ // var3 = shuffle_epi32(var2, 2,3,0,1) [in0_w3, in0_w2, in1_w3, in1_w2]
+ //
+ // So var0 has shape [w0_lane0, GARBAGE_lane0, w0_lane1, GARBAGE_lane1]?
+ // Position 0 = in0_w0 (lane0 word0). Position 1 = in0_w1 (this is the
+ // "garbage" slot for word0 — it holds the *next* word, but at round
+ // time it's masked out by ROTATE / state updates only writing the
+ // qword-low halves). Position 2 = in1_w0 (lane1 word0). Position 3 =
+ // in1_w1 (lane1's garbage for word0).
+ //
+ // So var0 = XX_idx (word i for both lanes, with their word i+1 in
+ // the garbage slots). Similarly:
+ //
+ // var1 = XX_idx+1: position 0 = in0_w1 (lane0 word i+1), position 2 = in1_w1 (lane1 word i+1).
+ // var2 = XX_idx+2: position 0 = in0_w2, position 2 = in1_w2.
+ // var3 = XX_idx+3: position 0 = in0_w3, position 2 = in1_w3.
+ //
+ // The "garbage" 32-bit slots are *deterministic but unused* — only
+ // the qword-low halves (32-bit lanes 0 and 2) carry the active state.
+ let var0 = _mm_unpacklo_epi64(in0, in1);
+ // _MM_SHUFFLE(2,3,0,1) = (2<<6)|(3<<4)|(0<<2)|1 = 0xb1
+ let var1 = _mm_shuffle_epi32(var0, 0xb1);
+ let var2 = _mm_unpackhi_epi64(in0, in1);
+ let var3 = _mm_shuffle_epi32(var2, 0xb1);
+ (var0, var1, var2, var3)
+}
+
+// ----- Bool functions ADDF, expanded per-round from md5x2-sse.h:43-50 -----
+//
+// Each `addf_*` returns ADD(a, f(b,c,d)) so the round body can be:
+//
+// a = a + ik
+// a = addf_X(a, b, c, d)
+// a = rotate(a, r)
+// a = a + b
+
+#[inline(always)]
+#[allow(unused_unsafe)]
+unsafe fn addf_f(a: __m128i, b: __m128i, c: __m128i, d: __m128i) -> __m128i {
+ // F: ((c ^ d) & b) ^ d
+ let cd = _mm_xor_si128(c, d);
+ let cdb = _mm_and_si128(cd, b);
+ let f = _mm_xor_si128(cdb, d);
+ _mm_add_epi32(a, f)
+}
+
+#[inline(always)]
+#[allow(unused_unsafe)]
+unsafe fn addf_g(a: __m128i, b: __m128i, c: __m128i, d: __m128i) -> __m128i {
+ // G special form (upstream md5x2-sse.h:44):
+ // ADD(ADD(andnot(d, c), a), and(d, b))
+ // i.e. a + (~d & c) + (d & b)
+ let andn = _mm_andnot_si128(d, c);
+ let andd = _mm_and_si128(d, b);
+ _mm_add_epi32(_mm_add_epi32(andn, a), andd)
+}
+
+#[inline(always)]
+#[allow(unused_unsafe)]
+unsafe fn addf_h(a: __m128i, b: __m128i, c: __m128i, d: __m128i) -> __m128i {
+ // H: d ^ c ^ b
+ let h = _mm_xor_si128(_mm_xor_si128(d, c), b);
+ _mm_add_epi32(a, h)
+}
+
+#[inline(always)]
+#[allow(unused_unsafe)]
+unsafe fn addf_i(a: __m128i, b: __m128i, c: __m128i, d: __m128i) -> __m128i {
+ // I (SSE2 form, md5x2-sse.h:47):
+ // xor(or(xor(d, set1_epi8(-1)), b), c)
+ // i.e. (~d | b) ^ c
+ let nd = _mm_xor_si128(d, _mm_set1_epi32(-1));
+ let or_ = core::arch::x86_64::_mm_or_si128(nd, b);
+ let xored = _mm_xor_si128(or_, c);
+ _mm_add_epi32(a, xored)
+}
+
+// ----------------------------------------------------------------------
+// Round body: one `_RX(f, a, b, c, d, ik, r)` step.
+//
+// Each round takes the current xmm A (called `a`), the current B/C/D,
+// the pre-added input word `ik = m[i] + K_i` (or for round 0,
+// loaded-from-data + K), and the rotation `r`. It updates `a` in place.
+// ----------------------------------------------------------------------
+
+macro_rules! rx {
+ // Round-step. `$r` is the MD5 rotation amount (left rotate by r);
+ // `$s` must be `32 - r` (precomputed by caller because stable Rust
+ // forbids `{ 32 - R }` in const-generic position). The pair is kept
+ // explicit so a typo desyncs the test, not the runtime.
+ (F, $a:ident, $b:ident, $c:ident, $d:ident, $xx:expr, $r:literal, $s:literal, $k:expr) => {{
+ $a = _mm_add_epi32($a, _mm_add_epi32($xx, val($k)));
+ $a = addf_f($a, $b, $c, $d);
+ $a = rotate::<$s>($a);
+ $a = _mm_add_epi32($a, $b);
+ let _ = $r; // keep r in source for grep/audit against md5-base.h
+ }};
+ (G, $a:ident, $b:ident, $c:ident, $d:ident, $xx:expr, $r:literal, $s:literal, $k:expr) => {{
+ $a = _mm_add_epi32($a, _mm_add_epi32($xx, val($k)));
+ $a = addf_g($a, $b, $c, $d);
+ $a = rotate::<$s>($a);
+ $a = _mm_add_epi32($a, $b);
+ let _ = $r;
+ }};
+ (H, $a:ident, $b:ident, $c:ident, $d:ident, $xx:expr, $r:literal, $s:literal, $k:expr) => {{
+ $a = _mm_add_epi32($a, _mm_add_epi32($xx, val($k)));
+ $a = addf_h($a, $b, $c, $d);
+ $a = rotate::<$s>($a);
+ $a = _mm_add_epi32($a, $b);
+ let _ = $r;
+ }};
+ (I, $a:ident, $b:ident, $c:ident, $d:ident, $xx:expr, $r:literal, $s:literal, $k:expr) => {{
+ $a = _mm_add_epi32($a, _mm_add_epi32($xx, val($k)));
+ $a = addf_i($a, $b, $c, $d);
+ $a = rotate::<$s>($a);
+ $a = _mm_add_epi32($a, $b);
+ let _ = $r;
+ }};
+}
+
+/// Process one 64-byte block per lane, MD5x2 SSE2.
+///
+/// 1:1 expansion of `md5_process_block_x2_sse` (md5x2-base.h ->
+/// md5-base.h with the SSE2 macros from md5x2-sse.h substituted).
+///
+/// # Safety
+/// `data1` and `data2` must each be valid for 64-byte reads. They may
+/// alias.
+#[target_feature(enable = "sse2")]
+pub unsafe fn process_block_x2_sse2(state: &mut State, data1: *const u8, data2: *const u8) {
+ // Save initial state — added back at the end.
+ let oA = state.0[0];
+ let oB = state.0[1];
+ let oC = state.0[2];
+ let oD = state.0[3];
+
+ // Working state.
+ let mut A = oA;
+ let mut B = oB;
+ let mut C = oC;
+ let mut D = oD;
+
+ // Load all 16 message words at once via four LOAD4 calls.
+ // XX[i] holds message word i for both lanes in 32-bit positions 0 and 2.
+ let (XX0, XX1, XX2, XX3) = load4(data1, data2, 0);
+ let (XX4, XX5, XX6, XX7) = load4(data1, data2, 4);
+ let (XX8, XX9, XX10, XX11) = load4(data1, data2, 8);
+ let (XX12, XX13, XX14, XX15) = load4(data1, data2, 12);
+
+ // ----- Round 0 (F) — uses L (load + add k) -----
+ // Sequence and constants from md5-base.h:148-174.
+ rx!(F, A, B, C, D, XX0, 7, 25, 0xd76aa478);
+ rx!(F, D, A, B, C, XX1, 12, 20, 0xe8c7b756);
+ rx!(F, C, D, A, B, XX2, 17, 15, 0x242070db);
+ rx!(F, B, C, D, A, XX3, 22, 10, 0xc1bdceee);
+ rx!(F, A, B, C, D, XX4, 7, 25, 0xf57c0faf);
+ rx!(F, D, A, B, C, XX5, 12, 20, 0x4787c62a);
+ rx!(F, C, D, A, B, XX6, 17, 15, 0xa8304613);
+ rx!(F, B, C, D, A, XX7, 22, 10, 0xfd469501);
+ rx!(F, A, B, C, D, XX8, 7, 25, 0x698098d8);
+ rx!(F, D, A, B, C, XX9, 12, 20, 0x8b44f7af);
+ rx!(F, C, D, A, B, XX10, 17, 15, 0xffff5bb1);
+ rx!(F, B, C, D, A, XX11, 22, 10, 0x895cd7be);
+ rx!(F, A, B, C, D, XX12, 7, 25, 0x6b901122);
+ rx!(F, D, A, B, C, XX13, 12, 20, 0xfd987193);
+ rx!(F, C, D, A, B, XX14, 17, 15, 0xa679438e);
+ rx!(F, B, C, D, A, XX15, 22, 10, 0x49b40821);
+
+ // ----- Round 1 (G) — md5-base.h:192-207 -----
+ rx!(G, A, B, C, D, XX1, 5, 27, 0xf61e2562);
+ rx!(G, D, A, B, C, XX6, 9, 23, 0xc040b340);
+ rx!(G, C, D, A, B, XX11, 14, 18, 0x265e5a51);
+ rx!(G, B, C, D, A, XX0, 20, 12, 0xe9b6c7aa);
+ rx!(G, A, B, C, D, XX5, 5, 27, 0xd62f105d);
+ rx!(G, D, A, B, C, XX10, 9, 23, 0x02441453);
+ rx!(G, C, D, A, B, XX15, 14, 18, 0xd8a1e681);
+ rx!(G, B, C, D, A, XX4, 20, 12, 0xe7d3fbc8);
+ rx!(G, A, B, C, D, XX9, 5, 27, 0x21e1cde6);
+ rx!(G, D, A, B, C, XX14, 9, 23, 0xc33707d6);
+ rx!(G, C, D, A, B, XX3, 14, 18, 0xf4d50d87);
+ rx!(G, B, C, D, A, XX8, 20, 12, 0x455a14ed);
+ rx!(G, A, B, C, D, XX13, 5, 27, 0xa9e3e905);
+ rx!(G, D, A, B, C, XX2, 9, 23, 0xfcefa3f8);
+ rx!(G, C, D, A, B, XX7, 14, 18, 0x676f02d9);
+ rx!(G, B, C, D, A, XX12, 20, 12, 0x8d2a4c8a);
+
+ // ----- Round 2 (H) — md5-base.h:225-240 -----
+ rx!(H, A, B, C, D, XX5, 4, 28, 0xfffa3942);
+ rx!(H, D, A, B, C, XX8, 11, 21, 0x8771f681);
+ rx!(H, C, D, A, B, XX11, 16, 16, 0x6d9d6122);
+ rx!(H, B, C, D, A, XX14, 23, 9, 0xfde5380c);
+ rx!(H, A, B, C, D, XX1, 4, 28, 0xa4beea44);
+ rx!(H, D, A, B, C, XX4, 11, 21, 0x4bdecfa9);
+ rx!(H, C, D, A, B, XX7, 16, 16, 0xf6bb4b60);
+ rx!(H, B, C, D, A, XX10, 23, 9, 0xbebfbc70);
+ rx!(H, A, B, C, D, XX13, 4, 28, 0x289b7ec6);
+ rx!(H, D, A, B, C, XX0, 11, 21, 0xeaa127fa);
+ rx!(H, C, D, A, B, XX3, 16, 16, 0xd4ef3085);
+ rx!(H, B, C, D, A, XX6, 23, 9, 0x04881d05);
+ rx!(H, A, B, C, D, XX9, 4, 28, 0xd9d4d039);
+ rx!(H, D, A, B, C, XX12, 11, 21, 0xe6db99e5);
+ rx!(H, C, D, A, B, XX15, 16, 16, 0x1fa27cf8);
+ rx!(H, B, C, D, A, XX2, 23, 9, 0xc4ac5665);
+
+ // ----- Round 3 (I) — md5-base.h:263-278 -----
+ // SSE2 path uses IOFFSET = 0 (only AVX/AVX512 set IOFFSET = -1).
+ rx!(I, A, B, C, D, XX0, 6, 26, 0xf4292244);
+ rx!(I, D, A, B, C, XX7, 10, 22, 0x432aff97);
+ rx!(I, C, D, A, B, XX14, 15, 17, 0xab9423a7);
+ rx!(I, B, C, D, A, XX5, 21, 11, 0xfc93a039);
+ rx!(I, A, B, C, D, XX12, 6, 26, 0x655b59c3);
+ rx!(I, D, A, B, C, XX3, 10, 22, 0x8f0ccc92);
+ rx!(I, C, D, A, B, XX10, 15, 17, 0xffeff47d);
+ rx!(I, B, C, D, A, XX1, 21, 11, 0x85845dd1);
+ rx!(I, A, B, C, D, XX8, 6, 26, 0x6fa87e4f);
+ rx!(I, D, A, B, C, XX15, 10, 22, 0xfe2ce6e0);
+ rx!(I, C, D, A, B, XX6, 15, 17, 0xa3014314);
+ rx!(I, B, C, D, A, XX13, 21, 11, 0x4e0811a1);
+ rx!(I, A, B, C, D, XX4, 6, 26, 0xf7537e82);
+ rx!(I, D, A, B, C, XX11, 10, 22, 0xbd3af235);
+ rx!(I, C, D, A, B, XX2, 15, 17, 0x2ad7d2bb);
+ rx!(I, B, C, D, A, XX9, 21, 11, 0xeb86d391);
+
+ // Fold the round-final state back onto the saved initial state.
+ state.0[0] = _mm_add_epi32(oA, A);
+ state.0[1] = _mm_add_epi32(oB, B);
+ state.0[2] = _mm_add_epi32(oC, C);
+ state.0[3] = _mm_add_epi32(oD, D);
+}
+
+/// `Md5x2` backend wrapper for the SSE2 implementation.
+pub struct Sse2;
+
+impl crate::parpar_hasher::md5x2::Md5x2 for Sse2 {
+ type State = State;
+
+ #[inline(always)]
+ fn init_state() -> Self::State {
+ // Both lanes initialised to MD5 IV.
+ let flat: [u32; 8] = [
+ 0x67452301, 0xefcdab89, 0x98badcfe, 0x10325476, 0x67452301, 0xefcdab89, 0x98badcfe,
+ 0x10325476,
+ ];
+ unsafe { State::from_flat(&flat) }
+ }
+
+ #[inline(always)]
+ fn init_lane(state: &mut Self::State, lane: usize) {
+ // Round-trip through flat form to reset just one lane. Cheap —
+ // happens once per ~block_size bytes.
+ debug_assert!(lane < 2);
+ let mut flat = unsafe { state.to_flat() };
+ let off = lane * 4;
+ flat[off] = 0x67452301;
+ flat[off + 1] = 0xefcdab89;
+ flat[off + 2] = 0x98badcfe;
+ flat[off + 3] = 0x10325476;
+ *state = unsafe { State::from_flat(&flat) };
+ }
+
+ #[inline(always)]
+ fn extract_lane(state: &Self::State, lane: usize) -> [u8; 16] {
+ debug_assert!(lane < 2);
+ let flat = unsafe { state.to_flat() };
+ let off = lane * 4;
+ let mut out = [0u8; 16];
+ for i in 0..4 {
+ out[i * 4..i * 4 + 4].copy_from_slice(&flat[off + i].to_le_bytes());
+ }
+ out
+ }
+
+ #[inline(always)]
+ unsafe fn process_block(state: &mut Self::State, data1: *const u8, data2: *const u8) {
+ process_block_x2_sse2(state, data1, data2)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::parpar_hasher::md5x2::Md5x2;
+ use crate::parpar_hasher::md5x2_scalar::Scalar;
+
+ /// Hash an N-block message via both backends and assert the resulting
+ /// (lane0, lane1) digests match exactly.
+ fn cross_check(blocks: &[u8]) {
+ assert_eq!(blocks.len() % 64, 0);
+ let n = blocks.len() / 64;
+
+ let mut s_state = Scalar::init_state();
+ let mut v_state = Sse2::init_state();
+
+ // Feed lane 0 from a forward walk, lane 1 from a reverse walk
+ // — using the same 64-byte block on both is too easy a test
+ // because the two lanes carry identical state and bugs that
+ // mix lanes 0/1 are hidden. Use distinct inputs per lane.
+ for i in 0..n {
+ let d1 = &blocks[i * 64..i * 64 + 64];
+ let d2 = &blocks[(n - 1 - i) * 64..(n - 1 - i) * 64 + 64];
+ unsafe {
+ Scalar::process_block(&mut s_state, d1.as_ptr(), d2.as_ptr());
+ Sse2::process_block(&mut v_state, d1.as_ptr(), d2.as_ptr());
+ }
+ }
+
+ let s0 = Scalar::extract_lane(&s_state, 0);
+ let s1 = Scalar::extract_lane(&s_state, 1);
+ let v0 = Sse2::extract_lane(&v_state, 0);
+ let v1 = Sse2::extract_lane(&v_state, 1);
+
+ assert_eq!(v0, s0, "lane 0 mismatch after {n} blocks");
+ assert_eq!(v1, s1, "lane 1 mismatch after {n} blocks");
+ }
+
+ fn synth(len: usize, seed: u64) -> Vec