diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index bf11ba642..b4e086b2c 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -16,7 +16,7 @@ env:
   RUST_BACKTRACE: 1
   RUSTFLAGS: -Dwarnings
   RUSTDOCFLAGS: -Dwarnings
-  MSRV: "1.81"
+  MSRV: "1.85"
   SCCACHE_CACHE_SIZE: "50G"
   IROH_FORCE_STAGING_RELAYS: "1"
 
diff --git a/Cargo.lock b/Cargo.lock
index f87f022e2..498a13f2e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1,20 +1,6 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-version = 3
-
-[[package]]
-name = "acto"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "31c372578ce4215ccf94ec3f3585fbb6a902e47d07b064ff8a55d850ffb5025e"
-dependencies = [
- "parking_lot",
- "pin-project-lite",
- "rustc_version",
- "smol_str",
- "tokio",
- "tracing",
-]
+version = 4
 
 [[package]]
 name = "addr2line"
@@ -27,9 +13,9 @@ dependencies = [
 
 [[package]]
 name = "adler2"
-version = "2.0.0"
+version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
+checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
 
 [[package]]
 name = "aead"
@@ -42,19 +28,6 @@ dependencies = [
  "generic-array",
 ]
 
-[[package]]
-name = "ahash"
-version = "0.8.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
-dependencies = [
- "cfg-if",
- "getrandom 0.2.15",
- "once_cell",
- "version_check",
- "zerocopy 0.7.35",
-]
-
 [[package]]
 name = "aho-corasick"
 version = "1.1.3"
@@ -87,9 +60,9 @@ dependencies = [
 
 [[package]]
 name = "anstream"
-version = "0.6.18"
+version = "0.6.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b"
+checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933"
 dependencies = [
  "anstyle",
  "anstyle-parse",
@@ -102,50 +75,47 @@ dependencies = [
 
 [[package]]
 name = "anstyle"
-version = "1.0.10"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"
+checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd"
 
 [[package]]
 name = "anstyle-parse"
-version = "0.2.6"
+version = "0.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9"
+checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
 dependencies = [
  "utf8parse",
 ]
 
 [[package]]
 name = "anstyle-query"
-version = "1.1.2"
+version = "1.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c"
+checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9"
 dependencies = [
  "windows-sys 0.59.0",
 ]
 
 [[package]]
 name = "anstyle-wincon"
-version = "3.0.7"
+version = "3.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e"
+checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882"
 dependencies = [
  "anstyle",
- "once_cell",
+ "once_cell_polyfill",
  "windows-sys 0.59.0",
 ]
 
 [[package]]
 name = "anyhow"
-version = "1.0.97"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f"
-
-[[package]]
-name = "arc-swap"
-version = "1.7.1"
+version = "1.0.98"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457"
+checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"
+dependencies = [
+ "backtrace",
+]
 
 [[package]]
 name = "arrayref"
@@ -159,57 +129,6 @@ version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
 
-[[package]]
-name = "asn1-rs"
-version = "0.6.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048"
-dependencies = [
- "asn1-rs-derive",
- "asn1-rs-impl",
- "displaydoc",
- "nom",
- "num-traits",
- "rusticata-macros",
- "thiserror 1.0.69",
- "time",
-]
-
-[[package]]
-name = "asn1-rs-derive"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.100",
- "synstructure",
-]
-
-[[package]]
-name = "asn1-rs-impl"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.100",
-]
-
-[[package]]
-name = "async-channel"
-version = "2.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a"
-dependencies = [
- "concurrent-queue",
- "event-listener-strategy",
- "futures-core",
- "pin-project-lite",
-]
-
 [[package]]
 name = "async-compat"
 version = "0.2.4"
@@ -225,13 +144,13 @@ dependencies = [
 
 [[package]]
 name = "async-trait"
-version = "0.1.87"
+version = "0.1.88"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d556ec1359574147ec0c4fc5eb525f3f23263a592b1a9c07e0a75b427de55c97"
+checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
@@ -273,69 +192,15 @@ dependencies = [
 
 [[package]]
 name = "autocfg"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
-
-[[package]]
-name = "axum"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5"
-dependencies = [
- "axum-core",
- "bytes",
- "form_urlencoded",
- "futures-util",
- "http 1.2.0",
- "http-body",
- "http-body-util",
- "hyper",
- "hyper-util",
- "itoa",
- "matchit",
- "memchr",
- "mime",
- "percent-encoding",
- "pin-project-lite",
- "rustversion",
- "serde",
- "serde_json",
- "serde_path_to_error",
- "serde_urlencoded",
- "sync_wrapper",
- "tokio",
- "tower",
- "tower-layer",
- "tower-service",
- "tracing",
-]
-
-[[package]]
-name = "axum-core"
-version = "0.5.2"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6"
-dependencies = [
- "bytes",
- "futures-core",
- "http 1.2.0",
- "http-body",
- "http-body-util",
- "mime",
- "pin-project-lite",
- "rustversion",
- "sync_wrapper",
- "tower-layer",
- "tower-service",
- "tracing",
-]
+checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
 
 [[package]]
 name = "backon"
-version = "1.4.0"
+version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49fef586913a57ff189f25c9b3d034356a5bf6b3fa9a7f067588fe1698ba1f5d"
+checksum = "302eaff5357a264a2c42f127ecb8bac761cf99749fc3dc95677e2743991f99e7"
 dependencies = [
  "fastrand",
  "gloo-timers",
@@ -344,9 +209,9 @@ dependencies = [
 
 [[package]]
 name = "backtrace"
-version = "0.3.74"
+version = "0.3.75"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a"
+checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002"
 dependencies = [
  "addr2line",
  "cfg-if",
@@ -371,7 +236,9 @@ dependencies = [
  "positioned-io",
  "range-collections",
  "self_cell",
+ "serde",
  "smallvec",
+ "tokio",
 ]
 
 [[package]]
@@ -394,9 +261,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
 [[package]]
 name = "base64ct"
-version = "1.6.0"
+version = "1.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b"
+checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3"
 
 [[package]]
 name = "binary-merge"
@@ -427,15 +294,15 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "2.9.0"
+version = "2.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd"
+checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967"
 
 [[package]]
 name = "blake3"
-version = "1.8.0"
+version = "1.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34a796731680be7931955498a16a10b2270c7762963d5d570fdbfe02dcbf314f"
+checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0"
 dependencies = [
  "arrayref",
  "arrayvec",
@@ -459,11 +326,17 @@ version = "0.5.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "102dbef1187b1893e6dfe05a774e79fd52265f49f214f6879c8ff49f52c8188b"
 
+[[package]]
+name = "btparse"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "387e80962b798815a2b5c4bcfdb6bf626fa922ffe9f74e373103b858738e9f31"
+
 [[package]]
 name = "bumpalo"
-version = "3.17.0"
+version = "3.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
+checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
 
 [[package]]
 name = "byteorder"
@@ -480,42 +353,11 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "camino"
-version = "1.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3"
-dependencies = [
- "serde",
-]
-
-[[package]]
-name = "cargo-platform"
-version = "0.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24b1f0365a6c6bb4020cd05806fd0d33c44d38046b8bd7f0e40814b9763cabfc"
-dependencies = [
- "serde",
-]
-
-[[package]]
-name = "cargo_metadata"
-version = "0.14.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa"
-dependencies = [
- "camino",
- "cargo-platform",
- "semver",
- "serde",
- "serde_json",
-]
-
 [[package]]
 name = "cc"
-version = "1.2.16"
+version = "1.2.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c"
+checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc"
 dependencies = [
  "shlex",
 ]
@@ -528,9 +370,9 @@ checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c"
 
 [[package]]
 name = "cfg-if"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268"
 
 [[package]]
 name = "cfg_aliases"
@@ -551,9 +393,9 @@ dependencies = [
 
 [[package]]
 name = "chrono"
-version = "0.4.40"
+version = "0.4.41"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c"
+checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d"
 dependencies = [
  "android-tzdata",
  "iana-time-zone",
@@ -577,9 +419,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.5.31"
+version = "4.5.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "027bb0d98429ae334a8698531da7077bdf906419543a35a55c2cb1b66437d767"
+checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -587,9 +429,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.5.31"
+version = "4.5.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5589e0cba072e0f3d23791efac0fd8627b49c829c196a492e88168e6a669d863"
+checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e"
 dependencies = [
  "anstream",
  "anstyle",
@@ -599,33 +441,47 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "4.5.28"
+version = "4.5.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed"
+checksum = "d2c7947ae4cc3d851207c1adb5b5e260ff0cca11446b1d6d1423788e442257ce"
 dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
 name = "clap_lex"
-version = "0.7.4"
+version = "0.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
+checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675"
 
 [[package]]
 name = "cobs"
-version = "0.2.3"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1"
+dependencies = [
+ "thiserror 2.0.12",
+]
+
+[[package]]
+name = "color-backtrace"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15"
+checksum = "2123a5984bd52ca861c66f66a9ab9883b27115c607f801f86c1bc2a84eb69f0f"
+dependencies = [
+ "backtrace",
+ "btparse",
+ "termcolor",
+]
 
 [[package]]
 name = "colorchoice"
-version = "1.0.3"
+version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
+checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
 
 [[package]]
 name = "combine"
@@ -637,28 +493,6 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "concurrent-queue"
-version = "2.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973"
-dependencies = [
- "crossbeam-utils",
-]
-
-[[package]]
-name = "console"
-version = "0.15.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"
-dependencies = [
- "encode_unicode",
- "libc",
- "once_cell",
- "unicode-width",
- "windows-sys 0.59.0",
-]
-
 [[package]]
 name = "const-oid"
 version = "0.9.6"
@@ -673,11 +507,11 @@ checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
 
 [[package]]
 name = "cordyceps"
-version = "0.3.2"
+version = "0.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec10f0a762d93c4498d2e97a333805cb6250d60bead623f71d8034f9a4152ba3"
+checksum = "688d7fbb8092b8de775ef2536f36c8c31f2bc4006ece2e8d8ad2d17d00ce0a2a"
 dependencies = [
- "loom 0.5.6",
+ "loom",
  "tracing",
 ]
 
@@ -693,9 +527,9 @@ dependencies = [
 
 [[package]]
 name = "core-foundation"
-version = "0.10.0"
+version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63"
+checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"
 dependencies = [
  "core-foundation-sys",
  "libc",
@@ -718,9 +552,9 @@ dependencies = [
 
 [[package]]
 name = "crc"
-version = "3.2.1"
+version = "3.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636"
+checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675"
 dependencies = [
  "crc-catalog",
 ]
@@ -761,18 +595,6 @@ version = "0.8.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
 
-[[package]]
-name = "crypto-bigint"
-version = "0.5.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
-dependencies = [
- "generic-array",
- "rand_core 0.6.4",
- "subtle",
- "zeroize",
-]
-
 [[package]]
 name = "crypto-common"
 version = "0.1.6"
@@ -842,34 +664,20 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
-]
-
-[[package]]
-name = "dashmap"
-version = "6.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
-dependencies = [
- "cfg-if",
- "crossbeam-utils",
- "hashbrown 0.14.5",
- "lock_api",
- "once_cell",
- "parking_lot_core",
+ "syn 2.0.104",
 ]
 
 [[package]]
 name = "data-encoding"
-version = "2.8.0"
+version = "2.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010"
+checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476"
 
 [[package]]
 name = "der"
-version = "0.7.9"
+version = "0.7.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0"
+checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb"
 dependencies = [
  "const-oid",
  "der_derive",
@@ -877,20 +685,6 @@ dependencies = [
  "zeroize",
 ]
 
-[[package]]
-name = "der-parser"
-version = "9.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553"
-dependencies = [
- "asn1-rs",
- "displaydoc",
- "nom",
- "num-bigint",
- "num-traits",
- "rusticata-macros",
-]
-
 [[package]]
 name = "der_derive"
 version = "0.7.3"
@@ -899,14 +693,14 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
 name = "deranged"
-version = "0.3.11"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4"
+checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e"
 dependencies = [
  "powerfmt",
 ]
@@ -917,7 +711,16 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05"
 dependencies = [
- "derive_more-impl",
+ "derive_more-impl 1.0.0",
+]
+
+[[package]]
+name = "derive_more"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678"
+dependencies = [
+ "derive_more-impl 2.0.1",
 ]
 
 [[package]]
@@ -928,7 +731,19 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
+ "unicode-xid",
+]
+
+[[package]]
+name = "derive_more-impl"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.104",
  "unicode-xid",
 ]
 
@@ -945,7 +760,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
 dependencies = [
  "block-buffer",
- "const-oid",
  "crypto-common",
  "subtle",
 ]
@@ -958,7 +772,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
@@ -987,20 +801,6 @@ version = "1.0.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005"
 
-[[package]]
-name = "ecdsa"
-version = "0.16.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca"
-dependencies = [
- "der",
- "digest",
- "elliptic-curve",
- "rfc6979",
- "signature",
- "spki",
-]
-
 [[package]]
 name = "ed25519"
 version = "2.2.3"
@@ -1027,25 +827,6 @@ dependencies = [
  "zeroize",
 ]
 
-[[package]]
-name = "elliptic-curve"
-version = "0.13.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47"
-dependencies = [
- "base16ct",
- "crypto-bigint",
- "digest",
- "ff",
- "generic-array",
- "group",
- "pkcs8",
- "rand_core 0.6.4",
- "sec1",
- "subtle",
- "zeroize",
-]
-
 [[package]]
 name = "embedded-io"
 version = "0.4.0"
@@ -1058,12 +839,6 @@ version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d"
 
-[[package]]
-name = "encode_unicode"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
-
 [[package]]
 name = "enum-as-inner"
 version = "0.6.1"
@@ -1073,27 +848,27 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
 name = "enumflags2"
-version = "0.7.11"
+version = "0.7.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba2f4b465f5318854c6f8dd686ede6c0a9dc67d4b1ac241cf0eb51521a309147"
+checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef"
 dependencies = [
  "enumflags2_derive",
 ]
 
 [[package]]
 name = "enumflags2_derive"
-version = "0.7.11"
+version = "0.7.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc4caf64a58d7a6d65ab00639b046ff54399a39f5f2554728895ace4b297cd79"
+checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
@@ -1104,33 +879,12 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
 
 [[package]]
 name = "errno"
-version = "0.3.10"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
+checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad"
 dependencies = [
  "libc",
- "windows-sys 0.59.0",
-]
-
-[[package]]
-name = "event-listener"
-version = "5.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae"
-dependencies = [
- "concurrent-queue",
- "parking",
- "pin-project-lite",
-]
-
-[[package]]
-name = "event-listener-strategy"
-version = "0.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2"
-dependencies = [
- "event-listener",
- "pin-project-lite",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
@@ -1145,34 +899,12 @@ version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
 
-[[package]]
-name = "ff"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393"
-dependencies = [
- "rand_core 0.6.4",
- "subtle",
-]
-
 [[package]]
 name = "fiat-crypto"
 version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d"
 
-[[package]]
-name = "flume"
-version = "0.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095"
-dependencies = [
- "futures-core",
- "futures-sink",
- "nanorand",
- "spin",
-]
-
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -1181,9 +913,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
 [[package]]
 name = "foldhash"
-version = "0.1.4"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f"
+checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
 
 [[package]]
 name = "form_urlencoded"
@@ -1276,7 +1008,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
@@ -1291,12 +1023,6 @@ version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988"
 
-[[package]]
-name = "futures-timer"
-version = "3.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24"
-
 [[package]]
 name = "futures-util"
 version = "0.3.31"
@@ -1348,28 +1074,16 @@ dependencies = [
 
 [[package]]
 name = "generator"
-version = "0.7.5"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cc16584ff22b460a382b7feec54b23d2908d858152e5739a120b949293bd74e"
+checksum = "d18470a76cb7f8ff746cf1f7470914f900252ec36bbc40b569d74b1258446827"
 dependencies = [
  "cc",
- "libc",
- "log",
- "rustversion",
- "windows 0.48.0",
-]
-
-[[package]]
-name = "generator"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc6bd114ceda131d3b1d665eba35788690ad37f5916457286b32ab6fd3c438dd"
-dependencies = [
  "cfg-if",
  "libc",
  "log",
  "rustversion",
- "windows 0.58.0",
+ "windows 0.61.3",
 ]
 
 [[package]]
@@ -1385,22 +1099,22 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.15"
+version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
+checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592"
 dependencies = [
  "cfg-if",
  "js-sys",
  "libc",
- "wasi 0.11.0+wasi-snapshot-preview1",
+ "wasi 0.11.1+wasi-snapshot-preview1",
  "wasm-bindgen",
 ]
 
 [[package]]
 name = "getrandom"
-version = "0.3.2"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0"
+checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
 dependencies = [
  "cfg-if",
  "js-sys",
@@ -1434,52 +1148,18 @@ dependencies = [
  "wasm-bindgen",
 ]
 
-[[package]]
-name = "governor"
-version = "0.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be93b4ec2e4710b04d9264c0c7350cdd62a8c20e5e4ac732552ebb8f0debe8eb"
-dependencies = [
- "cfg-if",
- "dashmap",
- "futures-sink",
- "futures-timer",
- "futures-util",
- "getrandom 0.3.2",
- "no-std-compat",
- "nonzero_ext",
- "parking_lot",
- "portable-atomic",
- "quanta",
- "rand 0.9.0",
- "smallvec",
- "spinning_top",
- "web-time",
-]
-
-[[package]]
-name = "group"
-version = "0.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63"
-dependencies = [
- "ff",
- "rand_core 0.6.4",
- "subtle",
-]
-
 [[package]]
 name = "h2"
-version = "0.4.8"
+version = "0.4.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2"
+checksum = "a9421a676d1b147b16b82c9225157dc629087ef8ec4d5e2960f9437a90dac0a5"
 dependencies = [
  "atomic-waker",
  "bytes",
  "fnv",
  "futures-core",
  "futures-sink",
- "http 1.2.0",
+ "http 1.3.1",
  "indexmap",
  "slab",
  "tokio",
@@ -1498,18 +1178,9 @@ dependencies = [
 
 [[package]]
 name = "hashbrown"
-version = "0.14.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
-dependencies = [
- "ahash",
-]
-
-[[package]]
-name = "hashbrown"
-version = "0.15.2"
+version = "0.15.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
+checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5"
 dependencies = [
  "allocator-api2",
  "equivalent",
@@ -1518,11 +1189,11 @@ dependencies = [
 
 [[package]]
 name = "hashlink"
-version = "0.9.1"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af"
+checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1"
 dependencies = [
- "hashbrown 0.14.5",
+ "hashbrown",
 ]
 
 [[package]]
@@ -1545,12 +1216,6 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
-[[package]]
-name = "hermit-abi"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
-
 [[package]]
 name = "hex"
 version = "0.4.3"
@@ -1573,7 +1238,7 @@ dependencies = [
  "idna",
  "ipnet",
  "once_cell",
- "rand 0.9.0",
+ "rand 0.9.1",
  "ring",
  "thiserror 2.0.12",
  "tinyvec",
@@ -1595,7 +1260,7 @@ dependencies = [
  "moka",
  "once_cell",
  "parking_lot",
- "rand 0.9.0",
+ "rand 0.9.1",
  "resolv-conf",
  "smallvec",
  "thiserror 2.0.12",
@@ -1624,20 +1289,9 @@ dependencies = [
 
 [[package]]
 name = "hmac-sha256"
-version = "1.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a8575493d277c9092b988c780c94737fb9fd8651a1001e16bee3eccfc1baedb"
-
-[[package]]
-name = "hostname"
-version = "0.3.1"
+version = "1.1.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867"
-dependencies = [
- "libc",
- "match_cfg",
- "winapi",
-]
+checksum = "ad6880c8d4a9ebf39c6e8b77007ce223f646a4d21ce29d99f70cb16420545425"
 
 [[package]]
 name = "hostname-validator"
@@ -1658,9 +1312,9 @@ dependencies = [
 
 [[package]]
 name = "http"
-version = "1.2.0"
+version = "1.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea"
+checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565"
 dependencies = [
  "bytes",
  "fnv",
@@ -1674,18 +1328,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
 dependencies = [
  "bytes",
- "http 1.2.0",
+ "http 1.3.1",
 ]
 
 [[package]]
 name = "http-body-util"
-version = "0.1.2"
+version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f"
+checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
 dependencies = [
  "bytes",
- "futures-util",
- "http 1.2.0",
+ "futures-core",
+ "http 1.3.1",
  "http-body",
  "pin-project-lite",
 ]
@@ -1712,7 +1366,7 @@ dependencies = [
  "futures-channel",
  "futures-util",
  "h2",
- "http 1.2.0",
+ "http 1.3.1",
  "http-body",
  "httparse",
  "httpdate",
@@ -1725,12 +1379,11 @@ dependencies = [
 
 [[package]]
 name = "hyper-rustls"
-version = "0.27.5"
+version = "0.27.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2"
+checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58"
 dependencies = [
- "futures-util",
- "http 1.2.0",
+ "http 1.3.1",
  "hyper",
  "hyper-util",
  "rustls",
@@ -1738,22 +1391,26 @@ dependencies = [
  "tokio",
  "tokio-rustls",
  "tower-service",
- "webpki-roots",
+ "webpki-roots 1.0.1",
 ]
 
 [[package]]
 name = "hyper-util"
-version = "0.1.11"
+version = "0.1.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2"
+checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb"
 dependencies = [
+ "base64",
  "bytes",
  "futures-channel",
+ "futures-core",
  "futures-util",
- "http 1.2.0",
+ "http 1.3.1",
  "http-body",
  "hyper",
+ "ipnet",
  "libc",
+ "percent-encoding",
  "pin-project-lite",
  "socket2",
  "tokio",
@@ -1763,16 +1420,17 @@ dependencies = [
 
 [[package]]
 name = "iana-time-zone"
-version = "0.1.61"
+version = "0.1.63"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220"
+checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8"
 dependencies = [
  "android_system_properties",
  "core-foundation-sys",
  "iana-time-zone-haiku",
  "js-sys",
+ "log",
  "wasm-bindgen",
- "windows-core 0.52.0",
+ "windows-core 0.61.2",
 ]
 
 [[package]]
@@ -1786,21 +1444,22 @@ dependencies = [
 
 [[package]]
 name = "icu_collections"
-version = "1.5.0"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526"
+checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47"
 dependencies = [
  "displaydoc",
+ "potential_utf",
  "yoke",
  "zerofrom",
  "zerovec",
 ]
 
 [[package]]
-name = "icu_locid"
-version = "1.5.0"
+name = "icu_locale_core"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637"
+checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a"
 dependencies = [
  "displaydoc",
  "litemap",
@@ -1809,31 +1468,11 @@ dependencies = [
  "zerovec",
 ]
 
-[[package]]
-name = "icu_locid_transform"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e"
-dependencies = [
- "displaydoc",
- "icu_locid",
- "icu_locid_transform_data",
- "icu_provider",
- "tinystr",
- "zerovec",
-]
-
-[[package]]
-name = "icu_locid_transform_data"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e"
-
 [[package]]
 name = "icu_normalizer"
-version = "1.5.0"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f"
+checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979"
 dependencies = [
  "displaydoc",
  "icu_collections",
@@ -1841,67 +1480,54 @@ dependencies = [
  "icu_properties",
  "icu_provider",
  "smallvec",
- "utf16_iter",
- "utf8_iter",
- "write16",
  "zerovec",
 ]
 
 [[package]]
 name = "icu_normalizer_data"
-version = "1.5.0"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516"
+checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3"
 
 [[package]]
 name = "icu_properties"
-version = "1.5.1"
+version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5"
+checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b"
 dependencies = [
  "displaydoc",
  "icu_collections",
- "icu_locid_transform",
+ "icu_locale_core",
  "icu_properties_data",
  "icu_provider",
- "tinystr",
+ "potential_utf",
+ "zerotrie",
  "zerovec",
 ]
 
 [[package]]
 name = "icu_properties_data"
-version = "1.5.0"
+version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569"
+checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632"
 
 [[package]]
 name = "icu_provider"
-version = "1.5.0"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9"
+checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af"
 dependencies = [
  "displaydoc",
- "icu_locid",
- "icu_provider_macros",
+ "icu_locale_core",
  "stable_deref_trait",
  "tinystr",
  "writeable",
  "yoke",
  "zerofrom",
+ "zerotrie",
  "zerovec",
 ]
 
-[[package]]
-name = "icu_provider_macros"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.100",
-]
-
 [[package]]
 name = "idna"
 version = "1.0.3"
@@ -1915,9 +1541,9 @@ dependencies = [
 
 [[package]]
 name = "idna_adapter"
-version = "1.2.0"
+version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71"
+checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344"
 dependencies = [
  "icu_normalizer",
  "icu_properties",
@@ -1933,12 +1559,12 @@ dependencies = [
  "attohttpc",
  "bytes",
  "futures",
- "http 1.2.0",
+ "http 1.3.1",
  "http-body-util",
  "hyper",
  "hyper-util",
  "log",
- "rand 0.9.0",
+ "rand 0.9.1",
  "tokio",
  "url",
  "xmltree",
@@ -1946,25 +1572,12 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "2.7.1"
+version = "2.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652"
+checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e"
 dependencies = [
  "equivalent",
- "hashbrown 0.15.2",
-]
-
-[[package]]
-name = "indicatif"
-version = "0.17.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235"
-dependencies = [
- "console",
- "number_prefix",
- "portable-atomic",
- "unicode-width",
- "web-time",
+ "hashbrown",
 ]
 
 [[package]]
@@ -2015,30 +1628,36 @@ version = "2.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130"
 
+[[package]]
+name = "iri-string"
+version = "0.7.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
 [[package]]
 name = "iroh"
-version = "0.35.0"
+version = "0.90.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ca758f4ce39ae3f07de922be6c73de6a48a07f39554e78b5745585652ce38f5"
+checksum = "9436f319c2d24bca1b28a2fab4477c8d2ac795ab2d3aeda142d207b38ec068f4"
 dependencies = [
  "aead",
- "anyhow",
- "atomic-waker",
- "axum",
  "backon",
  "bytes",
  "cfg_aliases",
- "concurrent-queue",
  "crypto_box",
  "data-encoding",
  "der",
- "derive_more",
+ "derive_more 1.0.0",
  "ed25519-dalek",
  "futures-buffered",
  "futures-util",
- "getrandom 0.3.2",
+ "getrandom 0.3.3",
  "hickory-resolver",
- "http 1.2.0",
+ "http 1.3.1",
  "igd-next",
  "instant",
  "iroh-base",
@@ -2048,25 +1667,27 @@ dependencies = [
  "iroh-quinn-udp",
  "iroh-relay",
  "n0-future",
+ "n0-snafu",
+ "n0-watcher",
+ "nested_enum_utils",
  "netdev",
  "netwatch",
  "pin-project",
  "pkarr",
  "portmapper",
  "rand 0.8.5",
- "rcgen",
  "reqwest",
  "ring",
  "rustls",
+ "rustls-pki-types",
  "rustls-webpki",
  "serde",
  "smallvec",
+ "snafu",
  "spki",
  "strum",
  "stun-rs",
  "surge-ping",
- "swarm-discovery",
- "thiserror 2.0.12",
  "time",
  "tokio",
  "tokio-stream",
@@ -2074,86 +1695,75 @@ dependencies = [
  "tracing",
  "url",
  "wasm-bindgen-futures",
- "webpki-roots",
- "x509-parser",
+ "webpki-roots 0.26.11",
  "z32",
 ]
 
 [[package]]
 name = "iroh-base"
-version = "0.35.0"
+version = "0.90.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f91ac4aaab68153d726c4e6b39c30f9f9253743f0e25664e52f4caeb46f48d11"
+checksum = "8e0090050c4055b21e61cbcb856f043a2b24ad22c65d76bab91f121b4c7bece3"
 dependencies = [
  "curve25519-dalek",
  "data-encoding",
- "derive_more",
+ "derive_more 1.0.0",
  "ed25519-dalek",
+ "n0-snafu",
+ "nested_enum_utils",
  "postcard",
  "rand_core 0.6.4",
  "serde",
- "thiserror 2.0.12",
+ "snafu",
  "url",
 ]
 
 [[package]]
 name = "iroh-blobs"
-version = "0.35.0"
+version = "0.90.0"
 dependencies = [
  "anyhow",
- "async-channel",
+ "arrayvec",
  "bao-tree",
- "blake3",
  "bytes",
  "chrono",
  "clap",
- "console",
  "data-encoding",
- "derive_more",
+ "derive_more 2.0.1",
  "futures-buffered",
  "futures-lite",
- "futures-util",
  "genawaiter",
  "hashlink",
  "hex",
- "http-body",
- "indicatif",
  "iroh",
  "iroh-base",
  "iroh-io",
  "iroh-metrics",
  "iroh-quinn",
- "nested_enum_utils 0.1.0",
- "num_cpus",
- "oneshot",
- "parking_lot",
- "portable-atomic",
+ "iroh-test",
+ "irpc",
+ "n0-future",
+ "n0-snafu",
+ "nested_enum_utils",
  "postcard",
  "proptest",
- "quic-rpc",
- "quic-rpc-derive",
  "rand 0.8.5",
  "range-collections",
- "rcgen",
  "redb",
+ "ref-cast",
  "reflink-copy",
- "rustls",
  "self_cell",
  "serde",
- "serde-error",
  "serde_json",
  "serde_test",
  "smallvec",
- "ssh-key",
- "strum",
+ "snafu",
  "tempfile",
- "testdir",
+ "test-strategy",
  "testresult",
- "thiserror 2.0.12",
  "tokio",
  "tokio-util",
  "tracing",
- "tracing-futures",
  "tracing-subscriber",
  "tracing-test",
  "walkdir",
@@ -2174,19 +1784,15 @@ dependencies = [
 
 [[package]]
 name = "iroh-metrics"
-version = "0.34.0"
+version = "0.35.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f70466f14caff7420a14373676947e25e2917af6a5b1bec45825beb2bf1eb6a7"
+checksum = "c8922c169f1b84d39d325c02ef1bbe1419d4de6e35f0403462b3c7e60cc19634"
 dependencies = [
- "http-body-util",
- "hyper",
- "hyper-util",
  "iroh-metrics-derive",
  "itoa",
- "reqwest",
+ "postcard",
  "serde",
  "snafu",
- "tokio",
  "tracing",
 ]
 
@@ -2199,14 +1805,14 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
 name = "iroh-quinn"
-version = "0.13.0"
+version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76c6245c9ed906506ab9185e8d7f64857129aee4f935e899f398a3bd3b70338d"
+checksum = "0cde160ebee7aabede6ae887460cd303c8b809054224815addf1469d54a6fcf7"
 dependencies = [
  "bytes",
  "cfg_aliases",
@@ -2229,7 +1835,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "929d5d8fa77d5c304d3ee7cae9aede31f13908bd049f9de8c7c0094ad6f7c535"
 dependencies = [
  "bytes",
- "getrandom 0.2.15",
+ "getrandom 0.2.16",
  "rand 0.8.5",
  "ring",
  "rustc-hash",
@@ -2259,23 +1865,17 @@ dependencies = [
 
 [[package]]
 name = "iroh-relay"
-version = "0.35.0"
+version = "0.90.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c63f122cdfaa4b4e0e7d6d3921d2b878f42a0c6d3ee5a29456dc3f5ab5ec931f"
+checksum = "c3f3cdbdaebc92835452e4e1d0d4b36118206b0950089b7bc3654f13e843475b"
 dependencies = [
- "ahash",
- "anyhow",
  "bytes",
  "cfg_aliases",
- "clap",
- "dashmap",
  "data-encoding",
- "derive_more",
- "getrandom 0.3.2",
- "governor",
- "hickory-proto",
+ "derive_more 1.0.0",
+ "getrandom 0.3.3",
  "hickory-resolver",
- "http 1.2.0",
+ "http 1.3.1",
  "http-body-util",
  "hyper",
  "hyper-util",
@@ -2283,43 +1883,80 @@ dependencies = [
  "iroh-metrics",
  "iroh-quinn",
  "iroh-quinn-proto",
- "lru 0.12.5",
+ "lru",
  "n0-future",
+ "n0-snafu",
+ "nested_enum_utils",
  "num_enum",
  "pin-project",
  "pkarr",
  "postcard",
  "rand 0.8.5",
- "rcgen",
- "regex",
- "reloadable-state",
  "reqwest",
  "rustls",
- "rustls-cert-file-reader",
- "rustls-cert-reloadable-resolver",
- "rustls-pemfile",
+ "rustls-pki-types",
  "rustls-webpki",
  "serde",
  "sha1",
- "simdutf8",
+ "snafu",
  "strum",
- "stun-rs",
- "thiserror 2.0.12",
- "time",
  "tokio",
  "tokio-rustls",
- "tokio-rustls-acme",
  "tokio-util",
  "tokio-websockets",
- "toml",
  "tracing",
- "tracing-subscriber",
  "url",
- "webpki-roots",
+ "webpki-roots 0.26.11",
  "ws_stream_wasm",
  "z32",
 ]
 
+[[package]]
+name = "iroh-test"
+version = "0.31.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aca0e7c59e447ab8ced8cd4841d95fe13ed44dce203c6ece1a963cf1be2aed25"
+dependencies = [
+ "anyhow",
+ "tokio",
+ "tracing",
+ "tracing-subscriber",
+]
+
+[[package]]
+name = "irpc"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b355fe12226ee885e1c1056a867c2cf37be2b22032a16f5ab7091069e98a966f"
+dependencies = [
+ "anyhow",
+ "futures-buffered",
+ "futures-util",
+ "iroh-quinn",
+ "irpc-derive",
+ "n0-future",
+ "postcard",
+ "rcgen",
+ "rustls",
+ "serde",
+ "smallvec",
+ "thiserror 2.0.12",
+ "tokio",
+ "tokio-util",
+ "tracing",
+]
+
+[[package]]
+name = "irpc-derive"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "efeabe1ee5615ea0416340b1a6d71a16f971495859c87fad48633b6497ee7a77"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "is_terminal_polyfill"
 version = "1.70.1"
@@ -2369,33 +2006,24 @@ name = "lazy_static"
 version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
-dependencies = [
- "spin",
-]
 
 [[package]]
 name = "libc"
-version = "0.2.172"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"
-
-[[package]]
-name = "libm"
-version = "0.2.11"
+version = "0.2.174"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa"
+checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.9.2"
+version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6db9c683daf087dc577b7506e9695b3d556a9f3849903fa28186283afd6809e9"
+checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"
 
 [[package]]
 name = "litemap"
-version = "0.7.5"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856"
+checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956"
 
 [[package]]
 name = "litrs"
@@ -2405,9 +2033,9 @@ checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5"
 
 [[package]]
 name = "lock_api"
-version = "0.4.12"
+version = "0.4.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
+checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765"
 dependencies = [
  "autocfg",
  "scopeguard",
@@ -2415,22 +2043,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.26"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e"
-
-[[package]]
-name = "loom"
-version = "0.5.6"
+version = "0.4.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff50ecb28bb86013e935fb6683ab1f6d3a20016f123c76fd4c27470076ac30f5"
-dependencies = [
- "cfg-if",
- "generator 0.7.5",
- "scoped-tls",
- "tracing",
- "tracing-subscriber",
-]
+checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
 
 [[package]]
 name = "loom"
@@ -2439,32 +2054,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca"
 dependencies = [
  "cfg-if",
- "generator 0.8.4",
+ "generator",
  "scoped-tls",
  "tracing",
  "tracing-subscriber",
 ]
 
-[[package]]
-name = "lru"
-version = "0.12.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38"
-dependencies = [
- "hashbrown 0.15.2",
-]
-
 [[package]]
 name = "lru"
 version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "227748d55f2f0ab4735d87fd623798cb6b664512fe979705f829c9f81c934465"
+dependencies = [
+ "hashbrown",
+]
 
 [[package]]
-name = "match_cfg"
-version = "0.1.0"
+name = "lru-slab"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4"
+checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
 
 [[package]]
 name = "matchers"
@@ -2475,12 +2084,6 @@ dependencies = [
  "regex-automata 0.1.10",
 ]
 
-[[package]]
-name = "matchit"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
-
 [[package]]
 name = "md5"
 version = "0.7.0"
@@ -2489,40 +2092,28 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
 
 [[package]]
 name = "memchr"
-version = "2.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
-
-[[package]]
-name = "mime"
-version = "0.3.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
-
-[[package]]
-name = "minimal-lexical"
-version = "0.2.1"
+version = "2.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"
 
 [[package]]
 name = "miniz_oxide"
-version = "0.8.5"
+version = "0.8.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5"
+checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
 dependencies = [
  "adler2",
 ]
 
 [[package]]
 name = "mio"
-version = "1.0.3"
+version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd"
+checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c"
 dependencies = [
  "libc",
- "wasi 0.11.0+wasi-snapshot-preview1",
- "windows-sys 0.52.0",
+ "wasi 0.11.1+wasi-snapshot-preview1",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -2534,7 +2125,7 @@ dependencies = [
  "crossbeam-channel",
  "crossbeam-epoch",
  "crossbeam-utils",
- "loom 0.7.2",
+ "loom",
  "parking_lot",
  "portable-atomic",
  "rustc_version",
@@ -2551,7 +2142,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7bb0e5d99e681ab3c938842b96fcb41bf8a7bb4bfdb11ccbd653a7e83e06c794"
 dependencies = [
  "cfg_aliases",
- "derive_more",
+ "derive_more 1.0.0",
  "futures-buffered",
  "futures-lite",
  "futures-util",
@@ -2566,24 +2157,27 @@ dependencies = [
 ]
 
 [[package]]
-name = "nanorand"
-version = "0.7.0"
+name = "n0-snafu"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3"
+checksum = "c4fed465ff57041f29db78a9adc8864296ef93c6c16029f9e192dc303404ebd0"
 dependencies = [
- "getrandom 0.2.15",
+ "anyhow",
+ "btparse",
+ "color-backtrace",
+ "snafu",
+ "tracing-error",
 ]
 
 [[package]]
-name = "nested_enum_utils"
-version = "0.1.0"
+name = "n0-watcher"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f256ef99e7ac37428ef98c89bef9d84b590172de4bbfbe81b68a4cd3abadb32"
+checksum = "f216d4ebc5fcf9548244803cbb93f488a2ae160feba3706cd17040d69cf7a368"
 dependencies = [
- "proc-macro-crate",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
+ "derive_more 1.0.0",
+ "n0-future",
+ "snafu",
 ]
 
 [[package]]
@@ -2647,7 +2241,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0800eae8638a299eaa67476e1c6b6692922273e0f7939fd188fc861c837b9cd2"
 dependencies = [
  "anyhow",
- "bitflags 2.9.0",
+ "bitflags 2.9.1",
  "byteorder",
  "libc",
  "log",
@@ -2696,24 +2290,26 @@ dependencies = [
 
 [[package]]
 name = "netwatch"
-version = "0.5.0"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67eeaa5f7505c93c5a9b35ba84fd21fb8aa3f24678c76acfe8716af7862fb07a"
+checksum = "2a829a830199b14989f9bccce6136ab928ab48336ab1f8b9002495dbbbb2edbe"
 dependencies = [
  "atomic-waker",
  "bytes",
  "cfg_aliases",
- "derive_more",
+ "derive_more 1.0.0",
  "iroh-quinn-udp",
  "js-sys",
  "libc",
  "n0-future",
- "nested_enum_utils 0.2.2",
+ "n0-watcher",
+ "nested_enum_utils",
  "netdev",
  "netlink-packet-core",
  "netlink-packet-route 0.23.0",
  "netlink-proto",
  "netlink-sys",
+ "pin-project-lite",
  "serde",
  "snafu",
  "socket2",
@@ -2723,47 +2319,16 @@ dependencies = [
  "tracing",
  "web-sys",
  "windows 0.59.0",
- "windows-result 0.3.1",
+ "windows-result",
  "wmi",
 ]
 
-[[package]]
-name = "no-std-compat"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c"
-
 [[package]]
 name = "no-std-net"
 version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "43794a0ace135be66a25d3ae77d41b91615fb68ae937f904090203e81f755b65"
 
-[[package]]
-name = "nom"
-version = "7.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
-dependencies = [
- "memchr",
- "minimal-lexical",
-]
-
-[[package]]
-name = "nonzero_ext"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21"
-
-[[package]]
-name = "ntapi"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
-dependencies = [
- "winapi",
-]
-
 [[package]]
 name = "ntimestamp"
 version = "1.0.0"
@@ -2772,7 +2337,7 @@ checksum = "c50f94c405726d3e0095e89e72f75ce7f6587b94a8bd8dc8054b73f65c0fd68c"
 dependencies = [
  "base32",
  "document-features",
- "getrandom 0.2.15",
+ "getrandom 0.2.16",
  "httpdate",
  "js-sys",
  "once_cell",
@@ -2789,59 +2354,12 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "num-bigint"
-version = "0.4.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
-dependencies = [
- "num-integer",
- "num-traits",
-]
-
-[[package]]
-name = "num-bigint-dig"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151"
-dependencies = [
- "byteorder",
- "lazy_static",
- "libm",
- "num-integer",
- "num-iter",
- "num-traits",
- "rand 0.8.5",
- "smallvec",
- "zeroize",
-]
-
 [[package]]
 name = "num-conv"
 version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
 
-[[package]]
-name = "num-integer"
-version = "0.1.46"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
-dependencies = [
- "num-traits",
-]
-
-[[package]]
-name = "num-iter"
-version = "0.1.45"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf"
-dependencies = [
- "autocfg",
- "num-integer",
- "num-traits",
-]
-
 [[package]]
 name = "num-traits"
 version = "0.2.19"
@@ -2849,46 +2367,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
 dependencies = [
  "autocfg",
- "libm",
-]
-
-[[package]]
-name = "num_cpus"
-version = "1.16.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
-dependencies = [
- "hermit-abi",
- "libc",
 ]
 
 [[package]]
 name = "num_enum"
-version = "0.7.3"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179"
+checksum = "a973b4e44ce6cad84ce69d797acf9a044532e4184c4f267913d1b546a0727b7a"
 dependencies = [
  "num_enum_derive",
+ "rustversion",
 ]
 
 [[package]]
 name = "num_enum_derive"
-version = "0.7.3"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56"
+checksum = "77e878c846a8abae00dd069496dbe8751b16ac1c3d6bd2a7283a938e8228f90d"
 dependencies = [
  "proc-macro-crate",
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
-[[package]]
-name = "number_prefix"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
-
 [[package]]
 name = "object"
 version = "0.36.7"
@@ -2898,30 +2400,21 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "oid-registry"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9"
-dependencies = [
- "asn1-rs",
-]
-
 [[package]]
 name = "once_cell"
-version = "1.20.3"
+version = "1.21.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e"
+checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
 dependencies = [
  "critical-section",
  "portable-atomic",
 ]
 
 [[package]]
-name = "oneshot"
-version = "0.1.11"
+name = "once_cell_polyfill"
+version = "1.70.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4ce411919553d3f9fa53a0880544cda985a112117a0444d5ff1e870a893d6ea"
+checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad"
 
 [[package]]
 name = "opaque-debug"
@@ -2941,44 +2434,6 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
 
-[[package]]
-name = "p256"
-version = "0.13.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b"
-dependencies = [
- "ecdsa",
- "elliptic-curve",
- "primeorder",
- "sha2",
-]
-
-[[package]]
-name = "p384"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6"
-dependencies = [
- "ecdsa",
- "elliptic-curve",
- "primeorder",
- "sha2",
-]
-
-[[package]]
-name = "p521"
-version = "0.13.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fc9e2161f1f215afdfce23677034ae137bbd45016a880c2eb3ba8eb95f085b2"
-dependencies = [
- "base16ct",
- "ecdsa",
- "elliptic-curve",
- "primeorder",
- "rand_core 0.6.4",
- "sha2",
-]
-
 [[package]]
 name = "parking"
 version = "2.2.1"
@@ -2987,9 +2442,9 @@ checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba"
 
 [[package]]
 name = "parking_lot"
-version = "0.12.3"
+version = "0.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
+checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13"
 dependencies = [
  "lock_api",
  "parking_lot_core",
@@ -2997,9 +2452,9 @@ dependencies = [
 
 [[package]]
 name = "parking_lot_core"
-version = "0.9.10"
+version = "0.9.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
+checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5"
 dependencies = [
  "cfg-if",
  "libc",
@@ -3041,9 +2496,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
 
 [[package]]
 name = "pest"
-version = "2.7.15"
+version = "2.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc"
+checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323"
 dependencies = [
  "memchr",
  "thiserror 2.0.12",
@@ -3052,9 +2507,9 @@ dependencies = [
 
 [[package]]
 name = "pest_derive"
-version = "2.7.15"
+version = "2.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "816518421cfc6887a0d62bf441b6ffb4536fcc926395a69e1a85852d4363f57e"
+checksum = "bb056d9e8ea77922845ec74a1c4e8fb17e7c218cc4fc11a15c5d25e189aa40bc"
 dependencies = [
  "pest",
  "pest_generator",
@@ -3062,24 +2517,23 @@ dependencies = [
 
 [[package]]
 name = "pest_generator"
-version = "2.7.15"
+version = "2.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d1396fd3a870fc7838768d171b4616d5c91f6cc25e377b673d714567d99377b"
+checksum = "87e404e638f781eb3202dc82db6760c8ae8a1eeef7fb3fa8264b2ef280504966"
 dependencies = [
  "pest",
  "pest_meta",
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
 name = "pest_meta"
-version = "2.7.15"
+version = "2.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1e58089ea25d717bfd31fb534e4f3afcc2cc569c70de3e239778991ea3b7dea"
+checksum = "edd1101f170f5903fde0914f899bb503d9ff5271d7ba76bbb70bea63690cc0d5"
 dependencies = [
- "once_cell",
  "pest",
  "sha2",
 ]
@@ -3111,7 +2565,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
@@ -3128,9 +2582,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
 name = "pkarr"
-version = "3.7.1"
+version = "3.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e32222ae3d617bf92414db29085f8a959a4515effce916e038e9399a335a0d6d"
+checksum = "41a50f65a2b97031863fbdff2f085ba832360b4bef3106d1fcff9ab5bf4063fe"
 dependencies = [
  "async-compat",
  "base32",
@@ -3141,9 +2595,9 @@ dependencies = [
  "ed25519-dalek",
  "futures-buffered",
  "futures-lite",
- "getrandom 0.2.15",
+ "getrandom 0.2.16",
  "log",
- "lru 0.13.0",
+ "lru",
  "ntimestamp",
  "reqwest",
  "self_cell",
@@ -3157,17 +2611,6 @@ dependencies = [
  "wasm-bindgen-futures",
 ]
 
-[[package]]
-name = "pkcs1"
-version = "0.7.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f"
-dependencies = [
- "der",
- "pkcs8",
- "spki",
-]
-
 [[package]]
 name = "pkcs8"
 version = "0.10.2"
@@ -3196,7 +2639,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "regex",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
@@ -3233,26 +2676,26 @@ dependencies = [
 
 [[package]]
 name = "portable-atomic"
-version = "1.11.0"
+version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"
+checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
 
 [[package]]
 name = "portmapper"
-version = "0.5.0"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d6db66007eac4a0ec8331d0d20c734bd64f6445d64bbaf0d0a27fea7a054e36"
+checksum = "2d82975dc029c00d566f4e0f61f567d31f0297a290cb5416b5580dd8b4b54ade"
 dependencies = [
  "base64",
  "bytes",
- "derive_more",
+ "derive_more 1.0.0",
  "futures-lite",
  "futures-util",
  "hyper-util",
  "igd-next",
  "iroh-metrics",
  "libc",
- "nested_enum_utils 0.2.2",
+ "nested_enum_utils",
  "netwatch",
  "num_enum",
  "rand 0.8.5",
@@ -3270,9 +2713,9 @@ dependencies = [
 
 [[package]]
 name = "positioned-io"
-version = "0.3.3"
+version = "0.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ccabfeeb89c73adf4081f0dca7f8e28dbda90981a222ceea37f619e93ea6afe9"
+checksum = "e8078ce4d22da5e8f57324d985cc9befe40c49ab0507a192d6be9e59584495c9"
 dependencies = [
  "libc",
  "winapi",
@@ -3280,9 +2723,9 @@ dependencies = [
 
 [[package]]
 name = "postcard"
-version = "1.1.1"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "170a2601f67cc9dba8edd8c4870b15f71a6a2dc196daec8c83f72b59dff628a8"
+checksum = "6c1de96e20f51df24ca73cafcc4690e044854d803259db27a00a461cb3b9d17a"
 dependencies = [
  "cobs",
  "embedded-io 0.4.0",
@@ -3294,13 +2737,22 @@ dependencies = [
 
 [[package]]
 name = "postcard-derive"
-version = "0.1.2"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0239fa9c1d225d4b7eb69925c25c5e082307a141e470573fbbe3a817ce6a7a37"
+checksum = "68f049d94cb6dda6938cc8a531d2898e7c08d71c6de63d8e67123cca6cdde2cc"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 1.0.109",
+ "syn 2.0.104",
+]
+
+[[package]]
+name = "potential_utf"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585"
+dependencies = [
+ "zerovec",
 ]
 
 [[package]]
@@ -3315,7 +2767,7 @@ version = "0.2.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
 dependencies = [
- "zerocopy 0.8.23",
+ "zerocopy",
 ]
 
 [[package]]
@@ -3352,15 +2804,6 @@ dependencies = [
  "ucd-parse",
 ]
 
-[[package]]
-name = "primeorder"
-version = "0.13.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6"
-dependencies = [
- "elliptic-curve",
-]
-
 [[package]]
 name = "proc-macro-crate"
 version = "3.3.0"
@@ -3398,92 +2841,37 @@ dependencies = [
 
 [[package]]
 name = "proc-macro-hack"
-version = "0.5.20+deprecated"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
-
-[[package]]
-name = "proc-macro2"
-version = "1.0.94"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84"
-dependencies = [
- "unicode-ident",
-]
-
-[[package]]
-name = "proptest"
-version = "1.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50"
-dependencies = [
- "bit-set",
- "bit-vec",
- "bitflags 2.9.0",
- "lazy_static",
- "num-traits",
- "rand 0.8.5",
- "rand_chacha 0.3.1",
- "rand_xorshift",
- "regex-syntax 0.8.5",
- "rusty-fork",
- "tempfile",
- "unarray",
-]
-
-[[package]]
-name = "quanta"
-version = "0.12.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3bd1fe6824cea6538803de3ff1bc0cf3949024db3d43c9643024bfb33a807c0e"
-dependencies = [
- "crossbeam-utils",
- "libc",
- "once_cell",
- "raw-cpuid",
- "wasi 0.11.0+wasi-snapshot-preview1",
- "web-sys",
- "winapi",
-]
-
-[[package]]
-name = "quic-rpc"
-version = "0.20.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18bad98bd048264ceb1361ff9d77a031535d8c1e3fe8f12c6966ec825bf68eb7"
-dependencies = [
- "anyhow",
- "bytes",
- "document-features",
- "flume",
- "futures-lite",
- "futures-sink",
- "futures-util",
- "iroh-quinn",
- "pin-project",
- "postcard",
- "rcgen",
- "rustls",
- "serde",
- "slab",
- "smallvec",
- "time",
- "tokio",
- "tokio-serde",
- "tokio-util",
- "tracing",
+version = "0.5.20+deprecated"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.95"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
+dependencies = [
+ "unicode-ident",
 ]
 
 [[package]]
-name = "quic-rpc-derive"
-version = "0.20.0"
+name = "proptest"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abf13f1bced5f2f2642d9d89a29d75f2d81ab34c4acfcb434c209d6094b9b2b7"
+checksum = "6fcdab19deb5195a31cf7726a210015ff1496ba1464fd42cb4f537b8b01b471f"
 dependencies = [
- "proc-macro2",
- "quic-rpc",
- "quote",
- "syn 1.0.109",
+ "bit-set",
+ "bit-vec",
+ "bitflags 2.9.1",
+ "lazy_static",
+ "num-traits",
+ "rand 0.9.1",
+ "rand_chacha 0.9.0",
+ "rand_xorshift",
+ "regex-syntax 0.8.5",
+ "rusty-fork",
+ "tempfile",
+ "unarray",
 ]
 
 [[package]]
@@ -3494,11 +2882,12 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
 
 [[package]]
 name = "quinn"
-version = "0.11.6"
+version = "0.11.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef"
+checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8"
 dependencies = [
  "bytes",
+ "cfg_aliases",
  "pin-project-lite",
  "quinn-proto",
  "quinn-udp",
@@ -3508,17 +2897,19 @@ dependencies = [
  "thiserror 2.0.12",
  "tokio",
  "tracing",
+ "web-time",
 ]
 
 [[package]]
 name = "quinn-proto"
-version = "0.11.9"
+version = "0.11.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d"
+checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e"
 dependencies = [
  "bytes",
- "getrandom 0.2.15",
- "rand 0.8.5",
+ "getrandom 0.3.3",
+ "lru-slab",
+ "rand 0.9.1",
  "ring",
  "rustc-hash",
  "rustls",
@@ -3532,9 +2923,9 @@ dependencies = [
 
 [[package]]
 name = "quinn-udp"
-version = "0.5.10"
+version = "0.5.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e46f3055866785f6b92bc6164b76be02ca8f2eb4b002c0354b28cf4c119e5944"
+checksum = "fcebb1209ee276352ef14ff8732e24cc2b02bbac986cd74a4c81bcb2f9881970"
 dependencies = [
  "cfg_aliases",
  "libc",
@@ -3546,9 +2937,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.39"
+version = "1.0.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1f1914ce909e1658d9907913b4b91947430c7d9be598b15a1912935b8c04801"
+checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
 dependencies = [
  "proc-macro2",
 ]
@@ -3565,9 +2956,9 @@ dependencies = [
 
 [[package]]
 name = "r-efi"
-version = "5.2.0"
+version = "5.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"
+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
 
 [[package]]
 name = "rand"
@@ -3582,13 +2973,12 @@ dependencies = [
 
 [[package]]
 name = "rand"
-version = "0.9.0"
+version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94"
+checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97"
 dependencies = [
  "rand_chacha 0.9.0",
  "rand_core 0.9.3",
- "zerocopy 0.8.23",
 ]
 
 [[package]]
@@ -3617,7 +3007,7 @@ version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
 dependencies = [
- "getrandom 0.2.15",
+ "getrandom 0.2.16",
 ]
 
 [[package]]
@@ -3626,16 +3016,16 @@ version = "0.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
 dependencies = [
- "getrandom 0.3.2",
+ "getrandom 0.3.3",
 ]
 
 [[package]]
 name = "rand_xorshift"
-version = "0.3.0"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f"
+checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a"
 dependencies = [
- "rand_core 0.6.4",
+ "rand_core 0.9.3",
 ]
 
 [[package]]
@@ -3647,18 +3037,10 @@ dependencies = [
  "binary-merge",
  "inplace-vec-builder",
  "ref-cast",
+ "serde",
  "smallvec",
 ]
 
-[[package]]
-name = "raw-cpuid"
-version = "11.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6df7ab838ed27997ba19a4664507e6f82b41fe6e20be42929332156e5e85146"
-dependencies = [
- "bitflags 2.9.0",
-]
-
 [[package]]
 name = "rcgen"
 version = "0.13.2"
@@ -3683,11 +3065,11 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.5.10"
+version = "0.5.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1"
+checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6"
 dependencies = [
- "bitflags 2.9.0",
+ "bitflags 2.9.1",
 ]
 
 [[package]]
@@ -3707,19 +3089,19 @@ checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
 name = "reflink-copy"
-version = "0.1.25"
+version = "0.1.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b86038e146b9a61557e1a2e58cdf2eddc0b46ce141b55541b1c1b9f3189d618"
+checksum = "78c81d000a2c524133cc00d2f92f019d399e57906c3b7119271a2495354fe895"
 dependencies = [
  "cfg-if",
  "libc",
  "rustix",
- "windows 0.60.0",
+ "windows 0.61.3",
 ]
 
 [[package]]
@@ -3772,49 +3154,28 @@ version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
 
-[[package]]
-name = "reloadable-core"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1dc20ac1418988b60072d783c9f68e28a173fb63493c127952f6face3b40c6e0"
-
-[[package]]
-name = "reloadable-state"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3853ef78d45b50f8b989896304a85239539d39b7f866a000e8846b9b72d74ce8"
-dependencies = [
- "arc-swap",
- "reloadable-core",
- "tokio",
-]
-
 [[package]]
 name = "reqwest"
-version = "0.12.15"
+version = "0.12.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb"
+checksum = "eabf4c97d9130e2bf606614eb937e86edac8292eaa6f422f995d7e8de1eb1813"
 dependencies = [
  "base64",
  "bytes",
  "futures-core",
  "futures-util",
- "http 1.2.0",
+ "http 1.3.1",
  "http-body",
  "http-body-util",
  "hyper",
  "hyper-rustls",
  "hyper-util",
- "ipnet",
  "js-sys",
  "log",
- "mime",
- "once_cell",
  "percent-encoding",
  "pin-project-lite",
  "quinn",
  "rustls",
- "rustls-pemfile",
  "rustls-pki-types",
  "serde",
  "serde_json",
@@ -3824,76 +3185,41 @@ dependencies = [
  "tokio-rustls",
  "tokio-util",
  "tower",
+ "tower-http",
  "tower-service",
  "url",
  "wasm-bindgen",
  "wasm-bindgen-futures",
  "wasm-streams",
  "web-sys",
- "webpki-roots",
- "windows-registry",
+ "webpki-roots 1.0.1",
 ]
 
 [[package]]
 name = "resolv-conf"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00"
-dependencies = [
- "hostname",
- "quick-error",
-]
-
-[[package]]
-name = "rfc6979"
-version = "0.4.0"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2"
-dependencies = [
- "hmac",
- "subtle",
-]
+checksum = "95325155c684b1c89f7765e30bc1c42e4a6da51ca513615660cb8a62ef9a88e3"
 
 [[package]]
 name = "ring"
-version = "0.17.13"
+version = "0.17.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee"
+checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
 dependencies = [
  "cc",
  "cfg-if",
- "getrandom 0.2.15",
+ "getrandom 0.2.16",
  "libc",
  "untrusted",
  "windows-sys 0.52.0",
 ]
 
-[[package]]
-name = "rsa"
-version = "0.9.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47c75d7c5c6b673e58bf54d8544a9f432e3a925b0e80f7cd3602ab5c50c55519"
-dependencies = [
- "const-oid",
- "digest",
- "num-bigint-dig",
- "num-integer",
- "num-traits",
- "pkcs1",
- "pkcs8",
- "rand_core 0.6.4",
- "sha2",
- "signature",
- "spki",
- "subtle",
- "zeroize",
-]
-
 [[package]]
 name = "rustc-demangle"
-version = "0.1.24"
+version = "0.1.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
+checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f"
 
 [[package]]
 name = "rustc-hash"
@@ -3910,22 +3236,13 @@ dependencies = [
  "semver",
 ]
 
-[[package]]
-name = "rusticata-macros"
-version = "4.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632"
-dependencies = [
- "nom",
-]
-
 [[package]]
 name = "rustix"
-version = "1.0.1"
+version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dade4812df5c384711475be5fcd8c162555352945401aed22a35bffeab61f657"
+checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266"
 dependencies = [
- "bitflags 2.9.0",
+ "bitflags 2.9.1",
  "errno",
  "libc",
  "linux-raw-sys",
@@ -3934,9 +3251,9 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.23.23"
+version = "0.23.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395"
+checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643"
 dependencies = [
  "log",
  "once_cell",
@@ -3947,41 +3264,6 @@ dependencies = [
  "zeroize",
 ]
 
-[[package]]
-name = "rustls-cert-file-reader"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f351eaf1dd003022222d2b1399caac198fefeab45c46b0f98bb03fc7cda9bb27"
-dependencies = [
- "rustls-cert-read",
- "rustls-pemfile",
- "rustls-pki-types",
- "thiserror 2.0.12",
- "tokio",
-]
-
-[[package]]
-name = "rustls-cert-read"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd46e8c5ae4de3345c4786a83f99ec7aff287209b9e26fa883c473aeb28f19d5"
-dependencies = [
- "rustls-pki-types",
-]
-
-[[package]]
-name = "rustls-cert-reloadable-resolver"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe1baa8a3a1f05eaa9fc55aed4342867f70e5c170ea3bfed1b38c51a4857c0c8"
-dependencies = [
- "futures-util",
- "reloadable-state",
- "rustls",
- "rustls-cert-read",
- "thiserror 2.0.12",
-]
-
 [[package]]
 name = "rustls-native-certs"
 version = "0.8.1"
@@ -3994,31 +3276,23 @@ dependencies = [
  "security-framework",
 ]
 
-[[package]]
-name = "rustls-pemfile"
-version = "2.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50"
-dependencies = [
- "rustls-pki-types",
-]
-
 [[package]]
 name = "rustls-pki-types"
-version = "1.11.0"
+version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c"
+checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79"
 dependencies = [
  "web-time",
+ "zeroize",
 ]
 
 [[package]]
 name = "rustls-platform-verifier"
-version = "0.5.0"
+version = "0.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e012c45844a1790332c9386ed4ca3a06def221092eda277e6f079728f8ea99da"
+checksum = "19787cda76408ec5404443dc8b31795c87cd8fec49762dc75fa727740d34acc1"
 dependencies = [
- "core-foundation 0.10.0",
+ "core-foundation 0.10.1",
  "core-foundation-sys",
  "jni",
  "log",
@@ -4029,8 +3303,8 @@ dependencies = [
  "rustls-webpki",
  "security-framework",
  "security-framework-sys",
- "webpki-root-certs",
- "windows-sys 0.52.0",
+ "webpki-root-certs 0.26.11",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -4041,9 +3315,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
 
 [[package]]
 name = "rustls-webpki"
-version = "0.102.8"
+version = "0.103.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9"
+checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435"
 dependencies = [
  "ring",
  "rustls-pki-types",
@@ -4052,9 +3326,9 @@ dependencies = [
 
 [[package]]
 name = "rustversion"
-version = "1.0.20"
+version = "1.0.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2"
+checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d"
 
 [[package]]
 name = "rusty-fork"
@@ -4113,28 +3387,14 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
-[[package]]
-name = "sec1"
-version = "0.7.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
-dependencies = [
- "base16ct",
- "der",
- "generic-array",
- "pkcs8",
- "subtle",
- "zeroize",
-]
-
 [[package]]
 name = "security-framework"
 version = "3.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316"
 dependencies = [
- "bitflags 2.9.0",
- "core-foundation 0.10.0",
+ "bitflags 2.9.1",
+ "core-foundation 0.10.1",
  "core-foundation-sys",
  "libc",
  "security-framework-sys",
@@ -4152,18 +3412,15 @@ dependencies = [
 
 [[package]]
 name = "self_cell"
-version = "1.1.0"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2fdfc24bc566f839a2da4c4295b82db7d25a24253867d5c64355abb5799bdbe"
+checksum = "0f7d95a54511e0c7be3f51e8867aa8cf35148d7b9445d44de2f943e2b206e749"
 
 [[package]]
 name = "semver"
 version = "1.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0"
-dependencies = [
- "serde",
-]
 
 [[package]]
 name = "send_wrapper"
@@ -4180,15 +3437,6 @@ dependencies = [
  "serde_derive",
 ]
 
-[[package]]
-name = "serde-error"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "342110fb7a5d801060c885da03bf91bfa7c7ca936deafcc64bb6706375605d47"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "serde_derive"
 version = "1.0.219"
@@ -4197,7 +3445,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
@@ -4212,25 +3460,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "serde_path_to_error"
-version = "0.1.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59fab13f937fa393d08645bf3a84bdfe86e296747b506ada67bb15f10f218b2a"
-dependencies = [
- "itoa",
- "serde",
-]
-
-[[package]]
-name = "serde_spanned"
-version = "0.6.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "serde_test"
 version = "1.0.177"
@@ -4281,9 +3510,9 @@ checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d"
 
 [[package]]
 name = "sha2"
-version = "0.10.8"
+version = "0.10.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
+checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
 dependencies = [
  "cfg-if",
  "cpufeatures",
@@ -4307,9 +3536,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
 
 [[package]]
 name = "signal-hook-registry"
-version = "1.4.2"
+version = "1.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1"
+checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410"
 dependencies = [
  "libc",
 ]
@@ -4320,7 +3549,6 @@ version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de"
 dependencies = [
- "digest",
  "rand_core 0.6.4",
 ]
 
@@ -4336,59 +3564,51 @@ version = "0.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dee851d0e5e7af3721faea1843e8015e820a234f81fda3dea9247e15bac9a86a"
 dependencies = [
- "bitflags 2.9.0",
+ "bitflags 2.9.1",
 ]
 
 [[package]]
 name = "slab"
-version = "0.4.9"
+version = "0.4.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
-dependencies = [
- "autocfg",
-]
+checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d"
 
 [[package]]
 name = "smallvec"
-version = "1.14.0"
+version = "1.15.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd"
+checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
 dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "smol_str"
-version = "0.1.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fad6c857cbab2627dcf01ec85a623ca4e7dcb5691cbaa3d7fb7653671f0d09c9"
-
 [[package]]
 name = "snafu"
-version = "0.8.5"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "223891c85e2a29c3fe8fb900c1fae5e69c2e42415e3177752e8718475efa5019"
+checksum = "320b01e011bf8d5d7a4a4a4be966d9160968935849c83b918827f6a435e7f627"
 dependencies = [
+ "backtrace",
  "snafu-derive",
 ]
 
 [[package]]
 name = "snafu-derive"
-version = "0.8.5"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917"
+checksum = "1961e2ef424c1424204d3a5d6975f934f56b6d50ff5732382d84ebf460e147f7"
 dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
 name = "socket2"
-version = "0.5.9"
+version = "0.5.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef"
+checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
 dependencies = [
  "libc",
  "windows-sys 0.52.0",
@@ -4403,15 +3623,6 @@ dependencies = [
  "lock_api",
 ]
 
-[[package]]
-name = "spinning_top"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d96d2d1d716fb500937168cc09353ffdc7a012be8475ac7308e1bdf0e3923300"
-dependencies = [
- "lock_api",
-]
-
 [[package]]
 name = "spki"
 version = "0.7.3"
@@ -4423,58 +3634,39 @@ dependencies = [
 ]
 
 [[package]]
-name = "ssh-cipher"
-version = "0.2.0"
+name = "stable_deref_trait"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "caac132742f0d33c3af65bfcde7f6aa8f62f0e991d80db99149eb9d44708784f"
-dependencies = [
- "cipher",
- "ssh-encoding",
-]
+checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
 
 [[package]]
-name = "ssh-encoding"
-version = "0.2.0"
+name = "strsim"
+version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb9242b9ef4108a78e8cd1a2c98e193ef372437f8c22be363075233321dd4a15"
-dependencies = [
- "base64ct",
- "pem-rfc7468",
- "sha2",
-]
+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
 
 [[package]]
-name = "ssh-key"
-version = "0.6.7"
+name = "structmeta"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b86f5297f0f04d08cabaa0f6bff7cb6aec4d9c3b49d87990d63da9d9156a8c3"
+checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329"
 dependencies = [
- "ed25519-dalek",
- "p256",
- "p384",
- "p521",
- "rand_core 0.6.4",
- "rsa",
- "sec1",
- "sha2",
- "signature",
- "ssh-cipher",
- "ssh-encoding",
- "subtle",
- "zeroize",
+ "proc-macro2",
+ "quote",
+ "structmeta-derive",
+ "syn 2.0.104",
 ]
 
 [[package]]
-name = "stable_deref_trait"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
-
-[[package]]
-name = "strsim"
-version = "0.11.1"
+name = "structmeta-derive"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
+checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.104",
+]
 
 [[package]]
 name = "strum"
@@ -4495,7 +3687,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
@@ -4519,7 +3711,7 @@ dependencies = [
  "precis-core",
  "precis-profiles",
  "quoted-string-parser",
- "rand 0.9.0",
+ "rand 0.9.1",
 ]
 
 [[package]]
@@ -4537,28 +3729,13 @@ dependencies = [
  "hex",
  "parking_lot",
  "pnet_packet",
- "rand 0.9.0",
+ "rand 0.9.1",
  "socket2",
  "thiserror 1.0.69",
  "tokio",
  "tracing",
 ]
 
-[[package]]
-name = "swarm-discovery"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3a95032b94c1dc318f55e0b130e3d2176cda022310a65c3df0092764ea69562"
-dependencies = [
- "acto",
- "anyhow",
- "hickory-proto",
- "rand 0.8.5",
- "socket2",
- "tokio",
- "tracing",
-]
-
 [[package]]
 name = "syn"
 version = "1.0.109"
@@ -4572,9 +3749,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.100"
+version = "2.0.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0"
+checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4603,27 +3780,13 @@ dependencies = [
 
 [[package]]
 name = "synstructure"
-version = "0.13.1"
+version = "0.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
+checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
-]
-
-[[package]]
-name = "sysinfo"
-version = "0.26.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c18a6156d1f27a9592ee18c1a846ca8dd5c258b7179fc193ae87c74ebb666f5"
-dependencies = [
- "cfg-if",
- "core-foundation-sys",
- "libc",
- "ntapi",
- "once_cell",
- "winapi",
+ "syn 2.0.104",
 ]
 
 [[package]]
@@ -4632,7 +3795,7 @@ version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
 dependencies = [
- "bitflags 2.9.0",
+ "bitflags 2.9.1",
  "core-foundation 0.9.4",
  "system-configuration-sys",
 ]
@@ -4655,31 +3818,36 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
 
 [[package]]
 name = "tempfile"
-version = "3.18.0"
+version = "3.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c317e0a526ee6120d8dabad239c8dadca62b24b6f168914bbbc8e2fb1f0e567"
+checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1"
 dependencies = [
- "cfg-if",
  "fastrand",
- "getrandom 0.3.2",
+ "getrandom 0.3.3",
  "once_cell",
  "rustix",
  "windows-sys 0.59.0",
 ]
 
 [[package]]
-name = "testdir"
-version = "0.9.3"
+name = "termcolor"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "test-strategy"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9ffa013be124f7e8e648876190de818e3a87088ed97ccd414a398b403aec8c8"
+checksum = "95eb2d223f5cd3ec8dd7874cf4ada95c9cf2b5ed84ecfb1046d9aefee0c28b12"
 dependencies = [
- "anyhow",
- "backtrace",
- "cargo-platform",
- "cargo_metadata",
- "once_cell",
- "sysinfo",
- "whoami",
+ "proc-macro2",
+ "quote",
+ "structmeta",
+ "syn 2.0.104",
 ]
 
 [[package]]
@@ -4714,7 +3882,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
@@ -4725,56 +3893,43 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
 name = "thread_local"
-version = "1.1.8"
+version = "1.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c"
+checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185"
 dependencies = [
  "cfg-if",
- "once_cell",
 ]
 
 [[package]]
 name = "time"
-version = "0.3.39"
+version = "0.3.41"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dad298b01a40a23aac4580b67e3dbedb7cc8402f3592d7f49469de2ea4aecdd8"
+checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40"
 dependencies = [
  "deranged",
- "itoa",
  "js-sys",
  "num-conv",
  "powerfmt",
  "serde",
  "time-core",
- "time-macros",
 ]
 
 [[package]]
 name = "time-core"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "765c97a5b985b7c11d7bc27fa927dc4fe6af3a6dfb021d28deb60d3bf51e76ef"
-
-[[package]]
-name = "time-macros"
-version = "0.2.20"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8093bc3e81c3bc5f7879de09619d06c9a5a5e45ca44dfeeb7225bae38005c5c"
-dependencies = [
- "num-conv",
- "time-core",
-]
+checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c"
 
 [[package]]
 name = "tinystr"
-version = "0.7.6"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f"
+checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b"
 dependencies = [
  "displaydoc",
  "zerovec",
@@ -4797,14 +3952,15 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.44.2"
+version = "1.45.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48"
+checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779"
 dependencies = [
  "backtrace",
  "bytes",
  "libc",
  "mio",
+ "parking_lot",
  "pin-project-lite",
  "signal-hook-registry",
  "socket2",
@@ -4820,7 +3976,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
@@ -4833,46 +3989,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "tokio-rustls-acme"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f296d48ff72e0df96e2d7ef064ad5904d016a130869e542f00b08c8e05cc18cf"
-dependencies = [
- "async-trait",
- "base64",
- "chrono",
- "futures",
- "log",
- "num-bigint",
- "pem",
- "proc-macro2",
- "rcgen",
- "reqwest",
- "ring",
- "rustls",
- "serde",
- "serde_json",
- "thiserror 2.0.12",
- "time",
- "tokio",
- "tokio-rustls",
- "webpki-roots",
- "x509-parser",
-]
-
-[[package]]
-name = "tokio-serde"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "caf600e7036b17782571dd44fa0a5cea3c82f60db5137f774a325a76a0d6852b"
-dependencies = [
- "bytes",
- "futures-core",
- "futures-sink",
- "pin-project",
-]
-
 [[package]]
 name = "tokio-stream"
 version = "0.1.17"
@@ -4893,9 +4009,10 @@ checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df"
 dependencies = [
  "bytes",
  "futures-core",
+ "futures-io",
  "futures-sink",
  "futures-util",
- "hashbrown 0.15.2",
+ "hashbrown",
  "pin-project-lite",
  "slab",
  "tokio",
@@ -4911,10 +4028,10 @@ dependencies = [
  "bytes",
  "futures-core",
  "futures-sink",
- "getrandom 0.3.2",
- "http 1.2.0",
+ "getrandom 0.3.3",
+ "http 1.3.1",
  "httparse",
- "rand 0.9.0",
+ "rand 0.9.1",
  "ring",
  "rustls-pki-types",
  "simdutf8",
@@ -4923,36 +4040,19 @@ dependencies = [
  "tokio-util",
 ]
 
-[[package]]
-name = "toml"
-version = "0.8.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148"
-dependencies = [
- "serde",
- "serde_spanned",
- "toml_datetime",
- "toml_edit",
-]
-
 [[package]]
 name = "toml_datetime"
-version = "0.6.8"
+version = "0.6.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
-dependencies = [
- "serde",
-]
+checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
 
 [[package]]
 name = "toml_edit"
-version = "0.22.24"
+version = "0.22.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474"
+checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
 dependencies = [
  "indexmap",
- "serde",
- "serde_spanned",
  "toml_datetime",
  "winnow",
 ]
@@ -4970,7 +4070,24 @@ dependencies = [
  "tokio",
  "tower-layer",
  "tower-service",
- "tracing",
+]
+
+[[package]]
+name = "tower-http"
+version = "0.6.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2"
+dependencies = [
+ "bitflags 2.9.1",
+ "bytes",
+ "futures-util",
+ "http 1.3.1",
+ "http-body",
+ "iri-string",
+ "pin-project-lite",
+ "tower",
+ "tower-layer",
+ "tower-service",
 ]
 
 [[package]]
@@ -4999,33 +4116,33 @@ dependencies = [
 
 [[package]]
 name = "tracing-attributes"
-version = "0.1.28"
+version = "0.1.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d"
+checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
 name = "tracing-core"
-version = "0.1.33"
+version = "0.1.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c"
+checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678"
 dependencies = [
  "once_cell",
  "valuable",
 ]
 
 [[package]]
-name = "tracing-futures"
-version = "0.2.5"
+name = "tracing-error"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"
+checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db"
 dependencies = [
- "pin-project",
  "tracing",
+ "tracing-subscriber",
 ]
 
 [[package]]
@@ -5075,7 +4192,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "04659ddb06c87d233c566112c1c9c5b9e98256d9af50ec3bc9c8327f873a7568"
 dependencies = [
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
@@ -5126,12 +4243,6 @@ dependencies = [
  "tinyvec",
 ]
 
-[[package]]
-name = "unicode-width"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"
-
 [[package]]
 name = "unicode-xid"
 version = "0.2.6"
@@ -5166,12 +4277,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "utf16_iter"
-version = "1.0.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
-
 [[package]]
 name = "utf8_iter"
 version = "1.0.4"
@@ -5186,11 +4291,13 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
 name = "uuid"
-version = "1.15.1"
+version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0f540e3240398cce6128b64ba83fdbdd86129c16a3aa1a3a252efd66eb3d587"
+checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d"
 dependencies = [
- "getrandom 0.3.2",
+ "getrandom 0.3.3",
+ "js-sys",
+ "wasm-bindgen",
 ]
 
 [[package]]
@@ -5235,9 +4342,9 @@ dependencies = [
 
 [[package]]
 name = "wasi"
-version = "0.11.0+wasi-snapshot-preview1"
+version = "0.11.1+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
 
 [[package]]
 name = "wasi"
@@ -5248,12 +4355,6 @@ dependencies = [
  "wit-bindgen-rt",
 ]
 
-[[package]]
-name = "wasite"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b"
-
 [[package]]
 name = "wasm-bindgen"
 version = "0.2.100"
@@ -5276,7 +4377,7 @@ dependencies = [
  "log",
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
  "wasm-bindgen-shared",
 ]
 
@@ -5311,7 +4412,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -5360,38 +4461,45 @@ dependencies = [
 
 [[package]]
 name = "webpki-root-certs"
-version = "0.26.8"
+version = "0.26.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75c7f0ef91146ebfb530314f5f1d24528d7f0767efbfd31dce919275413e393e"
+dependencies = [
+ "webpki-root-certs 1.0.1",
+]
+
+[[package]]
+name = "webpki-root-certs"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09aed61f5e8d2c18344b3faa33a4c837855fe56642757754775548fee21386c4"
+checksum = "86138b15b2b7d561bc4469e77027b8dd005a43dc502e9031d1f5afc8ce1f280e"
 dependencies = [
  "rustls-pki-types",
 ]
 
 [[package]]
 name = "webpki-roots"
-version = "0.26.8"
+version = "0.26.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2210b291f7ea53617fbafcc4939f10914214ec15aace5ba62293a668f322c5c9"
+checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9"
 dependencies = [
- "rustls-pki-types",
+ "webpki-roots 1.0.1",
 ]
 
 [[package]]
-name = "whoami"
-version = "1.5.2"
+name = "webpki-roots"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d"
+checksum = "8782dd5a41a24eed3a4f40b606249b3e236ca61adf1f25ea4d45c73de122b502"
 dependencies = [
- "redox_syscall",
- "wasite",
- "web-sys",
+ "rustls-pki-types",
 ]
 
 [[package]]
 name = "widestring"
-version = "1.1.0"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7219d36b6eac893fa81e84ebe06485e7dcbb616177469b142df14f1f4deb1311"
+checksum = "dd7cf3379ca1aac9eea11fba24fd7e315d621f8dfe35c8d7d2be8b793726e07d"
 
 [[package]]
 name = "winapi"
@@ -5424,25 +4532,6 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
-[[package]]
-name = "windows"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f"
-dependencies = [
- "windows-targets 0.48.5",
-]
-
-[[package]]
-name = "windows"
-version = "0.58.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6"
-dependencies = [
- "windows-core 0.58.0",
- "windows-targets 0.52.6",
-]
-
 [[package]]
 name = "windows"
 version = "0.59.0"
@@ -5450,17 +4539,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f919aee0a93304be7f62e8e5027811bbba96bcb1de84d6618be56e43f8a32a1"
 dependencies = [
  "windows-core 0.59.0",
- "windows-targets 0.53.0",
+ "windows-targets 0.53.2",
 ]
 
 [[package]]
 name = "windows"
-version = "0.60.0"
+version = "0.61.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddf874e74c7a99773e62b1c671427abf01a425e77c3d3fb9fb1e4883ea934529"
+checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893"
 dependencies = [
  "windows-collections",
- "windows-core 0.60.1",
+ "windows-core 0.61.2",
  "windows-future",
  "windows-link",
  "windows-numerics",
@@ -5468,33 +4557,11 @@ dependencies = [
 
 [[package]]
 name = "windows-collections"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5467f79cc1ba3f52ebb2ed41dbb459b8e7db636cc3429458d9a852e15bc24dec"
-dependencies = [
- "windows-core 0.60.1",
-]
-
-[[package]]
-name = "windows-core"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
-dependencies = [
- "windows-targets 0.52.6",
-]
-
-[[package]]
-name = "windows-core"
-version = "0.58.0"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99"
+checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8"
 dependencies = [
- "windows-implement 0.58.0",
- "windows-interface 0.58.0",
- "windows-result 0.2.0",
- "windows-strings 0.1.0",
- "windows-targets 0.52.6",
+ "windows-core 0.61.2",
 ]
 
 [[package]]
@@ -5504,44 +4571,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "810ce18ed2112484b0d4e15d022e5f598113e220c53e373fb31e67e21670c1ce"
 dependencies = [
  "windows-implement 0.59.0",
- "windows-interface 0.59.0",
- "windows-result 0.3.1",
+ "windows-interface",
+ "windows-result",
  "windows-strings 0.3.1",
- "windows-targets 0.53.0",
+ "windows-targets 0.53.2",
 ]
 
 [[package]]
 name = "windows-core"
-version = "0.60.1"
+version = "0.61.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca21a92a9cae9bf4ccae5cf8368dce0837100ddf6e6d57936749e85f152f6247"
+checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"
 dependencies = [
- "windows-implement 0.59.0",
- "windows-interface 0.59.0",
+ "windows-implement 0.60.0",
+ "windows-interface",
  "windows-link",
- "windows-result 0.3.1",
- "windows-strings 0.3.1",
+ "windows-result",
+ "windows-strings 0.4.2",
 ]
 
 [[package]]
 name = "windows-future"
-version = "0.1.1"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a787db4595e7eb80239b74ce8babfb1363d8e343ab072f2ffe901400c03349f0"
+checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e"
 dependencies = [
- "windows-core 0.60.1",
+ "windows-core 0.61.2",
  "windows-link",
-]
-
-[[package]]
-name = "windows-implement"
-version = "0.58.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.100",
+ "windows-threading",
 ]
 
 [[package]]
@@ -5552,91 +4609,70 @@ checksum = "83577b051e2f49a058c308f17f273b570a6a758386fc291b5f6a934dd84e48c1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
-name = "windows-interface"
-version = "0.58.0"
+name = "windows-implement"
+version = "0.60.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515"
+checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
 name = "windows-interface"
-version = "0.59.0"
+version = "0.59.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb26fd936d991781ea39e87c3a27285081e3c0da5ca0fcbc02d368cc6f52ff01"
+checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
 name = "windows-link"
-version = "0.1.0"
+version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6dccfd733ce2b1753b03b6d3c65edf020262ea35e20ccdf3e288043e6dd620e3"
+checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a"
 
 [[package]]
 name = "windows-numerics"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "005dea54e2f6499f2cee279b8f703b3cf3b5734a2d8d21867c8f44003182eeed"
-dependencies = [
- "windows-core 0.60.1",
- "windows-link",
-]
-
-[[package]]
-name = "windows-registry"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3"
-dependencies = [
- "windows-result 0.3.1",
- "windows-strings 0.3.1",
- "windows-targets 0.53.0",
-]
-
-[[package]]
-name = "windows-result"
 version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"
+checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1"
 dependencies = [
- "windows-targets 0.52.6",
+ "windows-core 0.61.2",
+ "windows-link",
 ]
 
 [[package]]
 name = "windows-result"
-version = "0.3.1"
+version = "0.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06374efe858fab7e4f881500e6e86ec8bc28f9462c47e5a9941a0142ad86b189"
+checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6"
 dependencies = [
  "windows-link",
 ]
 
 [[package]]
 name = "windows-strings"
-version = "0.1.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
+checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319"
 dependencies = [
- "windows-result 0.2.0",
- "windows-targets 0.52.6",
+ "windows-link",
 ]
 
 [[package]]
 name = "windows-strings"
-version = "0.3.1"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319"
+checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57"
 dependencies = [
  "windows-link",
 ]
@@ -5677,6 +4713,15 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
+[[package]]
+name = "windows-sys"
+version = "0.60.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
+dependencies = [
+ "windows-targets 0.53.2",
+]
+
 [[package]]
 name = "windows-targets"
 version = "0.42.2"
@@ -5725,9 +4770,9 @@ dependencies = [
 
 [[package]]
 name = "windows-targets"
-version = "0.53.0"
+version = "0.53.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b"
+checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef"
 dependencies = [
  "windows_aarch64_gnullvm 0.53.0",
  "windows_aarch64_msvc 0.53.0",
@@ -5739,6 +4784,15 @@ dependencies = [
  "windows_x86_64_msvc 0.53.0",
 ]
 
+[[package]]
+name = "windows-threading"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6"
+dependencies = [
+ "windows-link",
+]
+
 [[package]]
 name = "windows_aarch64_gnullvm"
 version = "0.42.2"
@@ -5921,9 +4975,9 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"
 
 [[package]]
 name = "winnow"
-version = "0.7.3"
+version = "0.7.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e7f4ea97f6f78012141bcdb6a216b2609f0979ada50b20ca5b52dde2eac2bb1"
+checksum = "74c7b26e3480b707944fc872477815d29a8e429d2f93a1ce000f5fa84a15cbcd"
 dependencies = [
  "memchr",
 ]
@@ -5944,7 +4998,7 @@ version = "0.39.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
 dependencies = [
- "bitflags 2.9.0",
+ "bitflags 2.9.1",
 ]
 
 [[package]]
@@ -5962,23 +5016,17 @@ dependencies = [
  "windows-core 0.59.0",
 ]
 
-[[package]]
-name = "write16"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"
-
 [[package]]
 name = "writeable"
-version = "0.5.5"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51"
+checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb"
 
 [[package]]
 name = "ws_stream_wasm"
-version = "0.7.4"
+version = "0.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7999f5f4217fe3818726b66257a4475f71e74ffd190776ad053fa159e50737f5"
+checksum = "6c173014acad22e83f16403ee360115b38846fe754e735c5d9d3803fe70c6abc"
 dependencies = [
  "async_io_stream",
  "futures",
@@ -5987,34 +5035,17 @@ dependencies = [
  "pharos",
  "rustc_version",
  "send_wrapper",
- "thiserror 1.0.69",
+ "thiserror 2.0.12",
  "wasm-bindgen",
  "wasm-bindgen-futures",
  "web-sys",
 ]
 
-[[package]]
-name = "x509-parser"
-version = "0.16.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69"
-dependencies = [
- "asn1-rs",
- "data-encoding",
- "der-parser",
- "lazy_static",
- "nom",
- "oid-registry",
- "rusticata-macros",
- "thiserror 1.0.69",
- "time",
-]
-
 [[package]]
 name = "xml-rs"
-version = "0.8.25"
+version = "0.8.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c5b940ebc25896e71dd073bad2dbaa2abfe97b0a391415e22ad1326d9c54e3c4"
+checksum = "a62ce76d9b56901b19a74f19431b0d8b3bc7ca4ad685a746dfd78ca8f4fc6bda"
 
 [[package]]
 name = "xmltree"
@@ -6036,9 +5067,9 @@ dependencies = [
 
 [[package]]
 name = "yoke"
-version = "0.7.5"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40"
+checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc"
 dependencies = [
  "serde",
  "stable_deref_trait",
@@ -6048,13 +5079,13 @@ dependencies = [
 
 [[package]]
 name = "yoke-derive"
-version = "0.7.5"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154"
+checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
  "synstructure",
 ]
 
@@ -6066,42 +5097,22 @@ checksum = "2164e798d9e3d84ee2c91139ace54638059a3b23e361f5c11781c2c6459bde0f"
 
 [[package]]
 name = "zerocopy"
-version = "0.7.35"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
-dependencies = [
- "zerocopy-derive 0.7.35",
-]
-
-[[package]]
-name = "zerocopy"
-version = "0.8.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd97444d05a4328b90e75e503a34bad781f14e28a823ad3557f0750df1ebcbc6"
-dependencies = [
- "zerocopy-derive 0.8.23",
-]
-
-[[package]]
-name = "zerocopy-derive"
-version = "0.7.35"
+version = "0.8.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
+checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f"
 dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.100",
+ "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.8.23"
+version = "0.8.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6352c01d0edd5db859a63e2605f4ea3183ddbd15e2c4a9e7d32184df75e4f154"
+checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
 
 [[package]]
@@ -6121,7 +5132,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
  "synstructure",
 ]
 
@@ -6131,11 +5142,22 @@ version = "1.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
 
+[[package]]
+name = "zerotrie"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595"
+dependencies = [
+ "displaydoc",
+ "yoke",
+ "zerofrom",
+]
+
 [[package]]
 name = "zerovec"
-version = "0.10.4"
+version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079"
+checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428"
 dependencies = [
  "yoke",
  "zerofrom",
@@ -6144,11 +5166,11 @@ dependencies = [
 
 [[package]]
 name = "zerovec-derive"
-version = "0.10.3"
+version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6"
+checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.104",
 ]
diff --git a/Cargo.toml b/Cargo.toml
index 8438532df..2e9bd6c77 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,186 +1,73 @@
 [package]
 name = "iroh-blobs"
-version = "0.35.0"
+version = "0.90.0"
 edition = "2021"
-readme = "README.md"
-description = "blob and collection transfer support for iroh"
+description = "content-addressed blobs for iroh"
 license = "MIT OR Apache-2.0"
 authors = ["dignifiedquire <me@dignifiedquire.com>", "n0 team"]
-repository = "https://github.com/n0-computer/iroh-blobs"
-keywords = ["hashing", "quic", "blake3"]
+repository = "https://github.com/n0-computer/blobs2"
+keywords = ["hashing", "quic", "blake3", "streaming"]
 
 # Sadly this also needs to be updated in .github/workflows/ci.yml
-rust-version = "1.81"
+rust-version = "1.85"
 
 [dependencies]
-anyhow = { version = "1" }
-async-channel = "2.3.1"
-bao-tree = { version = "0.15.1", features = [
-    "tokio_fsm",
-    "validate",
-], default-features = false }
-blake3 = { version = "1.8" }
-bytes = { version = "1.7", features = ["serde"] }
-chrono = "0.4.31"
-clap = { version = "4.5.20", features = ["derive"], optional = true }
-data-encoding = { version = "2.3.3" }
-derive_more = { version = "1.0.0", features = [
-    "debug",
-    "display",
-    "deref",
-    "deref_mut",
-    "from",
-    "try_into",
-    "into",
-] }
-futures-buffered = "0.2.4"
-futures-lite = "2.3"
-futures-util = { version = "0.3.30", optional = true }
-genawaiter = { version = "0.99.1", features = ["futures03"] }
-hashlink = { version = "0.9.0", optional = true }
+anyhow = "1.0.95"
+bao-tree = { version = "0.15.1", features = ["experimental-mixed", "tokio_fsm", "validate", "serde"], default-features = false }
+bytes = { version = "1", features = ["serde"] }
+derive_more = { version = "2.0.1", features = ["from", "try_from", "into", "debug", "display", "deref", "deref_mut"] }
+futures-lite = "2.6.0"
+quinn = { package = "iroh-quinn", version = "0.14.0" }
+n0-future = "0.1.2"
+n0-snafu = "0.2.0"
+range-collections = { version = "0.4.6", features = ["serde"] }
+redb = { version = "=2.4" }
+smallvec = { version = "1", features = ["serde", "const_new"] }
+snafu = "0.8.5"
+tokio = { version = "1.43.0", features = ["full"] }
+tokio-util = { version = "0.7.13", features = ["full"] }
+tracing = "0.1.41"
+iroh-io = "0.6.1"
+rand = "0.8.5"
 hex = "0.4.3"
-indicatif = { version = "0.17.8", optional = true }
-iroh-base = "0.35"
-iroh-io = { version = "0.6.0", features = ["stats"] }
-iroh-metrics = { version = "0.34", default-features = false }
-iroh = "0.35"
-nested_enum_utils = { version = "0.1.0", optional = true }
-num_cpus = "1.15.0"
-oneshot = "0.1.8"
-parking_lot = { version = "0.12.1", optional = true }
-portable-atomic = { version = "1", optional = true }
-postcard = { version = "1", default-features = false, features = [
-    "alloc",
-    "use-std",
-    "experimental-derive",
-] }
-quic-rpc = { version = "0.20", optional = true }
-quic-rpc-derive = { version = "0.20", optional = true }
-rand = "0.8"
-range-collections = "0.4.0"
-redb = { version = "=2.4", optional = true }
-reflink-copy = { version = "0.1.8", optional = true }
-self_cell = "1.0.1"
-serde = { version = "1", features = ["derive"] }
-serde-error = "0.1.3"
-smallvec = { version = "1.10.0", features = ["serde", "const_new"] }
-strum = { version = "0.26.3", optional = true }
-ssh-key = { version = "0.6", optional = true, features = ["ed25519"] }
-tempfile = { version = "3.10.0", optional = true }
-thiserror = "2"
-tokio = { version = "1", features = ["fs"] }
-tokio-util = { version = "0.7", features = ["io-util", "io"] }
-tracing = "0.1"
-tracing-futures = "0.2.5"
-walkdir = { version = "2.5.0", optional = true }
-
-# Examples
-console = { version = "0.15.8", optional = true }
-tracing-test = "0.2.5"
+serde = "1.0.217"
+postcard = { version = "1.1.1", features = ["experimental-derive", "use-std"] }
+data-encoding = "2.8.0"
+chrono = "0.4.39"
+nested_enum_utils = "0.2.1"
+ref-cast = "1.0.24"
+arrayvec = "0.7.6"
+iroh = "0.90"
+self_cell = "1.1.0"
+genawaiter = { version = "0.99.1", features = ["futures03"] }
+iroh-base = "0.90"
+reflink-copy = "0.1.24"
+irpc = { version = "0.5.0", features = ["rpc", "quinn_endpoint_setup", "message_spans", "stream", "derive"], default-features = false }
+iroh-metrics = { version = "0.35" }
+hashlink = "0.10.0"
+futures-buffered = "0.2.11"
 
 [dev-dependencies]
-http-body = "1.0"
-iroh = { version = "0.35", features = ["test-utils"] }
-quinn = { package = "iroh-quinn", version = "0.13", features = ["ring"] }
-futures-buffered = "0.2.4"
-proptest = "1.0.0"
-serde_json = "1.0.107"
-serde_test = "1.0.176"
-testresult = "0.4.0"
-tokio = { version = "1", features = ["macros", "test-util"] }
-tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-rcgen = "0.13"
-rustls = { version = "0.23", default-features = false, features = ["ring"] }
-tempfile = "3.10.0"
-futures-util = "0.3.30"
-testdir = "0.9.1"
+clap = { version = "4.5.31", features = ["derive"] }
+hex = "0.4.3"
+iroh-test = "0.31.0"
+proptest = "1.6.0"
+serde_json = "1.0.138"
+serde_test = "1.0.177"
+tempfile = "3.17.1"
+test-strategy = "0.4.0"
+testresult = "0.4.1"
+tracing-subscriber = { version = "0.3.19", features = ["fmt"] }
+tracing-test = "0.2.5"
+walkdir = "2.5.0"
 
 [features]
-default = ["fs-store", "net_protocol", "rpc"]
-downloader = ["dep:parking_lot", "tokio-util/time", "dep:hashlink"]
-net_protocol = ["downloader", "dep:futures-util"]
-fs-store = ["dep:reflink-copy", "redb", "dep:tempfile"]
-metrics = ["iroh-metrics/metrics"]
-redb = ["dep:redb"]
-cli = ["rpc", "dep:clap", "dep:indicatif", "dep:console"]
-rpc = [
-  "dep:quic-rpc",
-  "dep:quic-rpc-derive",
-  "dep:nested_enum_utils",
-  "dep:strum",
-  "dep:futures-util",
-  "dep:portable-atomic",
-  "dep:walkdir",
-  "dep:ssh-key",
-  "downloader",
-]
-
-example-iroh = [
-  "dep:clap",
-  "dep:indicatif",
-  "dep:console",
-  "iroh/discovery-local-network"
-]
-test = ["quic-rpc/quinn-transport", "quic-rpc/test-utils"]
-
-[package.metadata.docs.rs]
-all-features = true
-rustdoc-args = ["--cfg", "iroh_docsrs"]
-
-[[example]]
-name = "provide-bytes"
-
-[[example]]
-name = "fetch-fsm"
-
-[[example]]
-name = "fetch-stream"
-
-[[example]]
-name = "transfer"
-required-features = ["rpc"]
-
-[[example]]
-name = "hello-world-fetch"
-required-features = ["example-iroh"]
-
-[[example]]
-name = "hello-world-provide"
-required-features = ["example-iroh"]
-
-[[example]]
-name = "discovery-local-network"
-required-features = ["example-iroh"]
-
-[[example]]
-name = "custom-protocol"
-required-features = ["example-iroh"]
-
-[lints.rust]
-missing_debug_implementations = "warn"
-
-# We use this --cfg for documenting the cargo features on which an API
-# is available.  To preview this locally use: RUSTFLAGS="--cfg
-# iroh_docsrs cargo +nightly doc --all-features".  We use our own
-# iroh_docsrs instead of the common docsrs to avoid also enabling this
-# feature in any dependencies, because some indirect dependencies
-# require a feature enabled when using `--cfg docsrs` which we can not
-# do.  To enable for a crate set `#![cfg_attr(iroh_docsrs,
-# feature(doc_cfg))]` in the crate.
-unexpected_cfgs = { level = "warn", check-cfg = ["cfg(iroh_docsrs)"] }
-
-[lints.clippy]
-unused-async = "warn"
-
-[profile.dev-ci]
-inherits = 'dev'
-opt-level = 1
-
-[profile.optimized-release]
-inherits = 'release'
-debug = false
-lto = true
-debug-assertions = false
-opt-level = 3
-panic = 'abort'
-incremental = false
+hide-proto-docs = []
+metrics = []
+default = ["hide-proto-docs"]
+
+# [patch.crates-io]
+# iroh = { git = "https://github.com/n0-computer/iroh.git", branch = "ramfox/ticket-error" }
+# iroh-base = { git = "https://github.com/n0-computer/iroh.git", branch = "ramfox/ticket-error" }
+# irpc = { git = "https://github.com/n0-computer/irpc.git", branch = "iroh-quinn-latest" }
+# irpc-derive = { git = "https://github.com/n0-computer/irpc.git", branch = "iroh-quinn-latest" }
diff --git a/DESIGN.md b/DESIGN.md
new file mode 100644
index 000000000..198028363
--- /dev/null
+++ b/DESIGN.md
@@ -0,0 +1,142 @@
+# Blob store trade offs
+
+BLAKE3 hashed data and BLAKE3 verified streaming/bao are fully deterministic.
+
+The iroh blobs protocol is implemented in this spirit. Even for complex requests like requests that involve multiple ranges or entire hash sequences, for every request, there is exactly one sequence of bytes that is the correct answer. And the requester will notice if data is incorrect after at most 16 KiB of data.
+
+So why is designing a blob store for BLAKE3 hashed data so complex? What are the challenges?
+
+## Terminology
+
+The job of a blob store is to store two pieces of data per hash, the actual data itself and an `outboard` containing the BLAKE3 hash tree that connects each chunk of data to the root. Data and outboard are kept separate so that the data can be used as-is.
+
+## In memory store
+
+In the simplest case, you store data in memory. If you want to serve complete blobs, you just need a map from hash to data and outboard stored in a cheaply cloneable container like [bytes::Bytes]. Even if you want to also handle incomplete blobs, you use a map from hash and chunk number to (<= 1024 byte) data block, and that's it. Such a store is relatively easy to implement using a crate like [bao-tree] and a simple [BTreeMap] wrapped in a [Mutex].
+
+But the whole point of a blob store - in particular for iroh-blobs, where we don't have an upper size limit for blobs - is to store *a lot* of data persistently. So while an in memory store implementation is included in iroh-blobs, an in memory store won't do for all use cases.
+
+## Ok, fine. Let's use a database
+
+We want a database that works on the various platforms we support for iroh, and that does not come with a giant dependency tree or non-rust dependencies. So our current choice is an embedded database called [redb]. But as we will see the exact choice of database does not matter that much.
+
+A naive implementation of a block store using a relational or key value database would just duplicate the above approach. A map from hash and chunk number to a <= 1024 byte data block, except now in a persistent table.
+
+But this approach has multiple problems. While it would work very well for small blobs, it would be very slow for large blobs. First of all, the database will have significant storage overhead for each block. Second, operating systems are very efficient in quickly accessing large files. Whenever you do disk IO on a computer, you will always go through the file system. So anything you add on top will only slow things down.
+
+E.g. playing back videos from this database would be orders of magnitude slower than playing back from disk. And any interaction with the data would have to go through the iroh node, while with a file system we could directly use the file on disk.
+
+## So, no database?
+
+This is a perfectly valid approach when dealing exclusively with complete, large files. You just store the data file and the outboard file in the file system, using the hash as file name. But this approach has multiple downsides as well.
+
+First of all, it is very inefficient if you deal with a large number of tiny blobs, like we frequently do when working with [iroh-docs] or [iroh-willow] documents. Just the file system metadata for a tiny file will vastly exceed the storage needed for the data itself.
+
+Also, now we are very much dependent on the quirks of whatever file system our target operating system has. Many older file systems like FAT32 or EXT2 are notoriously bad in handling directories with millions of files. And we can't just limit ourselves to e.g. linux servers with modern file systems, since we also want to support mobile platforms and windows PCs.
+
+And last but not least, creating a the metadata for a file is very expensive compared to writing a few bytes. We would be limited to a pathetically low download speed when bulk downloading millions of blobs, like for example an iroh collection containing the linux source code. For very small files embedded databases are [frequently faster](https://www.sqlite.org/fasterthanfs.html) than the file system.
+
+There are additional problems when dealing with large incomplete files. Enough to fill a book. More on that later.
+
+## What about a hybrid approach
+
+It seems that databases are good for small blobs, and the file system works great for large blobs. So what about a hybrid approach where small blobs are stored entirely inline in a database, but large blobs are stored as files in the file system? This is complex, but provides very good performance both for millions of small blobs and for a few giant blobs.
+
+We won't ever have a directory with millions of files, since tiny files never make it to the file system in the first place. And we won't need to read giant files through the bottleneck of a database, so access to large files will be as fast and as concurrent as the hardware allows.
+
+This is exactly the approach that we currently take in the iroh-blobs file based store. For both data files and outboards, there is a configurable threshold below which the data will be stored inline in the database. The configuration can be useful for debugging and testing, e.g. you can force all data on disk by setting both thresholds to 0. But the defaults of 16 KiB for inline outboard and data size make a lot of sense and will rarely have to be changed in production.
+
+While this is the most complex approach, there is really no alternative if you want good performance both for a large number of tiny blobs and a small number of giant blobs.
+
+# The hybrid blob store in detail
+
+### Sizes
+
+A complication with the hybrid approach is that when we have a hash to get data for, we do not know its size. So we can not immediately decide whether it should go into a file or the database. Even once we request the data from a remote peer, we don't know the exact size. A remote node could lie about the size, e.g. tell us that the blob has an absurdly large size. As we get validated chunks the uncertainty about the size decreases. After the first validated chunk of data, the uncertainty is only a factor of 2, and as we receive the last chunk we know the exact size.
+
+This is not the end of the world, but it does mean that we need to keep the data for a hash in some weird indeterminate state before we have enough data to decide if it will be stored in memory or on disk.
+
+### Metadata
+
+We want to have the ability to quickly list all blobs or determine if we have data for a blob. Small blobs live completely in the database, so for these this is not a problem. But even for large blobs we want to keep a bit of metadata in the database as well, such as whether they are complete and if not if we at least have a verified size. But we also don't want to keep more than this in the metadata database, so we don't have to update it frequently as new data comes in. An individual database update is very fast, but syncing the update to disk is extremely slow no matter how small the update is. So the secret to fast download speeds is to not touch the metadata database at all if possible.
+
+### The metadata table
+
+The metadata table is a mapping from a BLAKE3 hash to an entry state. The rough entry state can be either `complete` or `partial`.
+
+#### Complete entry state
+
+Data can either be `inline` in the metadata db, `owned` in a canonical location in the file system (in this case we don't have to store the path since it can be computed from the hash), or `external` in a number of user owned paths that have to be explicitly stored. In the `owned` or `external` case we store the size in the metadata, since getting it from a file is an io operation with non-zero cost.
+
+An outboard can either be `inline` in the metadata db, `owned` in a canonical location in the file system, or `not needed` in case the data is less large than a chunk group size. Any data <= 16 KiB will have an outboard of size 0 which we don't bother to store at all. Outboards are never in user owned paths, so we never have to store a path for them.
+
+#### Partial entry state
+
+For a partial entry, there are just two possible states. Once we have the last chunk we have a verified size, so we store it here. If we do not yet have the last chunks, we do not have a size.
+
+### Inline data and inline outboard table
+
+The inline data table is kept separate from the metadata table since it will tend to be much bigger. Mixing inline data with metadata would make iterating over metadata entries slower. For the same reason, the outboard table is separate. You could argue that the inline outboard and data should be combined in a single table, but I decided against it to keep a simple table structure. Also, for the default settings if the data is inline, no outboard will be needed at all. Likewise when an outboard is needed, the data won't be inline.
+
+### Tags table
+
+The tags table is just a simple CRUD table that is completely independent of the other tables. It is just kept together with the other tables for simplicity.
+
+
+todo: move this section!
+
+The way this is currently implemented is as follows: as we start downloading a new blob from a remote, the information about this blob is kept fully in memory. The database is not touched at all, which means that the data would be gone after a crash. On the plus side, no matter what lies a remote node tells us, they will not create much work for us.
+
+As soon as we know that the data is going to be larger than the inlining threshold, we create an entry in the db marking the data as present but incomplete, and create files for the data and outboard. There is also an additional file which contains the most precisely known size.
+
+As soon as the blob becomes complete, an entry is created that marks the data as complete. At this point the inlining rules will be applied. Data that is below the inline threshold will be moved into the metadata database, likewise with the outboard.
+
+## Blob lifecycle
+
+There are two fundamentally different ways how data can be added to a blob store. 
+
+### Adding local files by name
+
+If we add data locally, e.g. from a file, we have the data but don't know the hash. We have to compute the complete outboard, the root hash, and then atomically move the file into the store under the root hash. Depending on the size of the file, data and outboard will end up in the file system or in the database. If there was partial data there before, we can just replace it with the new complete data and outboard. Once the data is stored under the hash, neither the data nor the outboard will be changed.
+
+### Syncing remote blobs by hash
+
+If we sync data from a remote node, we do know the hash but don't have the data. In case the blob is small, we can request and atomically write the data, so we have a similar situation as above. But as soon as the data is larger than a chunk group size (16 KiB), we will have a situation where the data has to be written incrementally. This is much more complex than adding local files. We now have to keep track of which chunks of the blob we have locally, and which chunks of the blob we can *prove* to have locally (not the same thing if you have a chunk group size > 1).
+
+### Blob deletion
+
+On creation, blobs are tagged with a temporary tag that prevents them from being deleted for as long as the process lives. They can then be tagged with a persistent tag that prevents them from being deleted even after a restart. And last but not least, large groups of blobs can be protected from deletion in bulk by putting a sequence of hashes into a blob and tagging that blob as a hash sequence.
+
+We also provide a way to explicitly delete blobs by hash, but that is meant to be used only in case of an emergency. You have some data that you want **gone** no matter how dire the consequences are.
+
+Deletion is always *complete*. There is currently no mechanism to protect or delete ranges of a file. This makes a number of things easier. A chunk of a blob can only go from not present (all zeroes) to present (whatever the correct data is for the chunk), not the other way. And this means that all changes due to syncing from a remote source commute, which makes dealing with concurrent downloads from multiple sources much easier.
+
+Deletion runs in the background, and conceptually you should think of it as always being on. Currently there is a small delay between a blob being untagged and it being deleted, but in the future blobs will be immediately deleted as soon as they are no longer tagged.
+
+### Incomplete files and the file system
+
+File system APIs are not very rich. They don't provide any transactional semantics or even ordering guarantees. And what little guarantees they provide is different from platform to platform.
+
+This is not much of a problem when dealing with *complete* files. You have a file for the data and a file for the outboard, and that's it. You never write to them, and reliably *reading* ranges from files is something even the most ancient file system like FAT32 or EXT2 can reliably handle.
+
+But it gets much more complex when dealing with *incomplete* files. E.g. you start downloading a large file from a remote node, and add chunks one by one (or in the case of n0 flavoured bao in chunk groups of 16 KiB).
+
+The challenge now is to write to the file system in such a way that the data remains consistent even when the write operation is rudely interrupted by the process being killed or the entire computer crashing. Losing a bit of data that has been written immediately before the crash is tolerable. But we don't *ever* want to have a situation after a crash where we *think* we have some data, but actually we are missing something about the data or the outboard that is needed to verify the data.
+
+BLAKE3 is very fast, so for small to medium files we can just validate the blob on node startup. For every chunk of data in the file, compute the hash and check it against the data in the outboard. This functionality exists in the [bao-tree] crate. It will give you a bitfield for every chunk that you have locally. But not even BLAKE3 is fast enough to do this for giant files like ML models or disk images. For such files it would take many *minutes* at startup to compute the validity bitfield, so we need a way to persist it.
+
+And that comes with its own rabbit hole of problems. As it turns out, [files are hard](https://danluu.com/file-consistency/). In particular writing to files.
+
+## Files are hard
+
+As discussed above, we want to keep track of data and outboard in the file system for large blobs. Let's say we use three files, `<hash>.data`, `<hash>.outboard` and `<hash>.bitfield`. As we receive chunks of verified data via [bao], we write the hashes to the outboard file, the data to the data file, and then update the bitfield file with the newly valid chunks. But we need to do the update in *exactly this order*, otherwise we might have a situation where the bitfield file has been updated before the data and outboard file, meaning that the bitfield file is inconsistent with the other files. The only way to fix this would be to do a full validation, which is what we wanted to avoid.
+
+We can of course enforce order by calling `fsync` on the data and outboard file *before* updating the bitfield file, after each chunk update. But that is horribly expensive no matter how fast your computer is. A sync to file system will take on the order of milliseconds, even on SATA SSDs. So clearly that is not an option. If you sync after each chunk, and a sync takes 5ms, your write speed is now limited to 200 kilobytes per second. Even if you sync after each 16 KiB chunk group, your download speed is now limited to 3.2 megabytes per second.
+
+But if we don't do the sync, there is no guarantee whatsoever in what order or if at all the data makes it to disk. So in case of a crash our bitfield could be inconsistent with the data, so we would be better off not persisting the bitfield at all.
+
+Most operating systems have platform dependent ways to finely control syncing for memory mapped files. E.g. on POSIX systems there is the `msync` API and on windows the similar `FlushViewOfFile` API. But first of all we are not confident that memory mapping will work consistently on all platforms we are targeting. And also, this API is not as useful as it might seem.
+
+While it provides fine grained control over syncing individual ranges of memory mapped files, it does not provide a way to guarantee that *no* sync happens unless you explicitly call it. So if you map the three files in memory and then update them, even if you don't call `msync` at all there is no guarantee in which order the data makes it to disk. The operating system page cache does not know about files or about write order, it will persist pages to disk in whatever order is most convenient for its internal data structures.
+
+So what's the solution? We can write to the data and outboard file in any order and without any sync calls as often as we want. These changes will be invisible after a crash provided that the bitmap file is not updated.
\ No newline at end of file
diff --git a/README.md b/README.md
index a87df9fb5..b6a4313cf 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,7 @@
 # iroh-blobs
 
+**NOTE: this version of iroh-blobs is not yet considered production quality. For now, if you need production quality, use iroh-blobs 0.35**
+
 This crate provides blob and blob sequence transfer support for iroh. It implements a simple request-response protocol based on BLAKE3 verified streaming.
 
 A request describes data in terms of BLAKE3 hashes and byte ranges. It is possible to request blobs or ranges of blobs, as well as entire sequences of blobs in one request.
diff --git a/deny.toml b/deny.toml
index 8f6f12246..026724f8d 100644
--- a/deny.toml
+++ b/deny.toml
@@ -29,6 +29,7 @@ allow = [
     "MPL-2.0",
     "Unicode-3.0",
     "Unlicense",
+    "CDLA-Permissive-2.0",
 ]
 
 [[licenses.clarify]]
diff --git a/examples/custom-protocol.rs b/examples/custom-protocol.rs
deleted file mode 100644
index f3255f8ad..000000000
--- a/examples/custom-protocol.rs
+++ /dev/null
@@ -1,300 +0,0 @@
-//! Example for adding a custom protocol to a iroh node.
-//!
-//! We are building a very simple custom protocol here, and make our iroh nodes speak this protocol
-//! in addition to the built-in protocols (blobs, gossip, docs).
-//!
-//! Our custom protocol allows querying the blob store of other nodes for text matches. For
-//! this, we keep a very primitive index of the UTF-8 text of our blobs.
-//!
-//! The example is contrived - we only use memory nodes, and our database is a hashmap in a mutex,
-//! and our queries just match if the query string appears as-is in a blob.
-//! Nevertheless, this shows how powerful systems can be built with custom protocols by also using
-//! the existing iroh protocols (blobs in this case).
-//!
-//! ## Usage
-//!
-//! In one terminal, run
-//!
-//!     cargo run --example custom-protocol --features=examples  -- listen "hello-world" "foo-bar" "hello-moon"
-//!
-//! This spawns an iroh nodes with three blobs. It will print the node's node id.
-//!
-//! In another terminal, run
-//!
-//!     cargo run --example custom-protocol --features=examples  -- query <node-id> hello
-//!
-//! Replace <node-id> with the node id from above. This will connect to the listening node with our
-//! custom protocol and query for the string `hello`. The listening node will return a list of
-//! blob hashes that contain `hello`. We will then download all these blobs with iroh-blobs,
-//! and then print a list of the hashes with their content.
-//!
-//! For this example, this will print:
-//!
-//!     moobakc6gao3ufmk: hello moon
-//!     25eyd35hbigiqc4n: hello world
-//!
-//! That's it! Follow along in the code below, we added a bunch of comments to explain things.
-
-use std::{
-    collections::HashMap,
-    sync::{Arc, Mutex},
-};
-
-use anyhow::Result;
-use clap::Parser;
-use futures_lite::future::Boxed as BoxedFuture;
-use iroh::{
-    endpoint::Connection,
-    protocol::{ProtocolHandler, Router},
-    Endpoint, NodeId,
-};
-use iroh_blobs::{net_protocol::Blobs, rpc::client::blobs::MemClient, Hash};
-use tracing_subscriber::{prelude::*, EnvFilter};
-
-#[derive(Debug, Parser)]
-pub struct Cli {
-    #[clap(subcommand)]
-    command: Command,
-}
-
-#[derive(Debug, Parser)]
-pub enum Command {
-    /// Spawn a node in listening mode.
-    Listen {
-        /// Each text string will be imported as a blob and inserted into the search database.
-        text: Vec<String>,
-    },
-    /// Query a remote node for data and print the results.
-    Query {
-        /// The node id of the node we want to query.
-        node_id: NodeId,
-        /// The text we want to match.
-        query: String,
-    },
-}
-
-/// Each custom protocol is identified by its ALPN string.
-///
-/// The ALPN, or application-layer protocol negotiation, is exchanged in the connection handshake,
-/// and the connection is aborted unless both nodes pass the same bytestring.
-const ALPN: &[u8] = b"iroh-example/text-search/0";
-
-#[tokio::main]
-async fn main() -> Result<()> {
-    setup_logging();
-    let args = Cli::parse();
-
-    // Build a in-memory node. For production code, you'd want a persistent node instead usually.
-    let endpoint = Endpoint::builder().bind().await?;
-    let builder = Router::builder(endpoint);
-    let blobs = Blobs::memory().build(builder.endpoint());
-    let builder = builder.accept(iroh_blobs::ALPN, blobs.clone());
-    let blobs_client = blobs.client();
-
-    // Build our custom protocol handler. The `builder` exposes access to various subsystems in the
-    // iroh node. In our case, we need a blobs client and the endpoint.
-    let proto = BlobSearch::new(blobs_client.clone(), builder.endpoint().clone());
-
-    // Add our protocol, identified by our ALPN, to the node, and spawn the node.
-    let builder = builder.accept(ALPN, proto.clone());
-    let node = builder.spawn();
-
-    match args.command {
-        Command::Listen { text } => {
-            let node_id = node.endpoint().node_id();
-            println!("our node id: {node_id}");
-
-            // Insert the text strings as blobs and index them.
-            for text in text.into_iter() {
-                proto.insert_and_index(text).await?;
-            }
-
-            // Wait for Ctrl-C to be pressed.
-            tokio::signal::ctrl_c().await?;
-        }
-        Command::Query { node_id, query } => {
-            // Query the remote node.
-            // This will send the query over our custom protocol, read hashes on the reply stream,
-            // and download each hash over iroh-blobs.
-            let hashes = proto.query_remote(node_id, &query).await?;
-
-            // Print out our query results.
-            for hash in hashes {
-                read_and_print(blobs_client, hash).await?;
-            }
-        }
-    }
-
-    node.shutdown().await?;
-
-    Ok(())
-}
-
-#[derive(Debug, Clone)]
-struct BlobSearch {
-    blobs: MemClient,
-    endpoint: Endpoint,
-    index: Arc<Mutex<HashMap<String, Hash>>>,
-}
-
-impl ProtocolHandler for BlobSearch {
-    /// The `accept` method is called for each incoming connection for our ALPN.
-    ///
-    /// The returned future runs on a newly spawned tokio task, so it can run as long as
-    /// the connection lasts.
-    fn accept(&self, connection: Connection) -> BoxedFuture<Result<()>> {
-        let this = self.clone();
-        // We have to return a boxed future from the handler.
-        Box::pin(async move {
-            // We can get the remote's node id from the connection.
-            let node_id = connection.remote_node_id()?;
-            println!("accepted connection from {node_id}");
-
-            // Our protocol is a simple request-response protocol, so we expect the
-            // connecting peer to open a single bi-directional stream.
-            let (mut send, mut recv) = connection.accept_bi().await?;
-
-            // We read the query from the receive stream, while enforcing a max query length.
-            let query_bytes = recv.read_to_end(64).await?;
-
-            // Now, we can perform the actual query on our local database.
-            let query = String::from_utf8(query_bytes)?;
-            let hashes = this.query_local(&query);
-
-            // We want to return a list of hashes. We do the simplest thing possible, and just send
-            // one hash after the other. Because the hashes have a fixed size of 32 bytes, this is
-            // very easy to parse on the other end.
-            for hash in hashes {
-                send.write_all(hash.as_bytes()).await?;
-            }
-
-            // By calling `finish` on the send stream we signal that we will not send anything
-            // further, which makes the receive stream on the other end terminate.
-            send.finish()?;
-            // By calling stopped we wait until the remote iroh Endpoint has acknowledged
-            // all data.  This does not mean the remote application has received all data
-            // from the Endpoint.
-            send.stopped().await?;
-            Ok(())
-        })
-    }
-}
-
-impl BlobSearch {
-    /// Create a new protocol handler.
-    pub fn new(blobs: MemClient, endpoint: Endpoint) -> Arc<Self> {
-        Arc::new(Self {
-            blobs,
-            endpoint,
-            index: Default::default(),
-        })
-    }
-
-    /// Query a remote node, download all matching blobs and print the results.
-    pub async fn query_remote(&self, node_id: NodeId, query: &str) -> Result<Vec<Hash>> {
-        // Establish a connection to our node.
-        // We use the default node discovery in iroh, so we can connect by node id without
-        // providing further information.
-        let conn = self.endpoint.connect(node_id, ALPN).await?;
-
-        // Open a bi-directional in our connection.
-        let (mut send, mut recv) = conn.open_bi().await?;
-
-        // Send our query.
-        send.write_all(query.as_bytes()).await?;
-
-        // Finish the send stream, signalling that no further data will be sent.
-        // This makes the `read_to_end` call on the accepting side terminate.
-        send.finish()?;
-        // By calling stopped we wait until the remote iroh Endpoint has acknowledged all
-        // data.  This does not mean the remote application has received all data from the
-        // Endpoint.
-        send.stopped().await?;
-
-        // In this example, we simply collect all results into a vector.
-        // For real protocols, you'd usually want to return a stream of results instead.
-        let mut out = vec![];
-
-        // The response is sent as a list of 32-byte long hashes.
-        // We simply read one after the other into a byte buffer.
-        let mut hash_bytes = [0u8; 32];
-        loop {
-            // Read 32 bytes from the stream.
-            match recv.read_exact(&mut hash_bytes).await {
-                // FinishedEarly means that the remote side did not send further data,
-                // so in this case we break our loop.
-                Err(iroh::endpoint::ReadExactError::FinishedEarly(_)) => break,
-                // Other errors are connection errors, so we bail.
-                Err(err) => return Err(err.into()),
-                Ok(_) => {}
-            };
-            // Upcast the raw bytes to the `Hash` type.
-            let hash = Hash::from_bytes(hash_bytes);
-            // Download the content via iroh-blobs.
-            self.blobs.download(hash, node_id.into()).await?.await?;
-            // Add the blob to our local database.
-            self.add_to_index(hash).await?;
-            out.push(hash);
-        }
-        Ok(out)
-    }
-
-    /// Query the local database.
-    ///
-    /// Returns the list of hashes of blobs which contain `query` literally.
-    pub fn query_local(&self, query: &str) -> Vec<Hash> {
-        let db = self.index.lock().unwrap();
-        db.iter()
-            .filter_map(|(text, hash)| text.contains(query).then_some(*hash))
-            .collect::<Vec<_>>()
-    }
-
-    /// Insert a text string into the database.
-    ///
-    /// This first imports the text as a blob into the iroh blob store, and then inserts a
-    /// reference to that hash in our (primitive) text database.
-    pub async fn insert_and_index(&self, text: String) -> Result<Hash> {
-        let hash = self.blobs.add_bytes(text.into_bytes()).await?.hash;
-        self.add_to_index(hash).await?;
-        Ok(hash)
-    }
-
-    /// Index a blob which is already in our blob store.
-    ///
-    /// This only indexes complete blobs that are smaller than 1KiB.
-    ///
-    /// Returns `true` if the blob was indexed.
-    async fn add_to_index(&self, hash: Hash) -> Result<bool> {
-        let mut reader = self.blobs.read(hash).await?;
-        // Skip blobs larger than 1KiB.
-        if reader.size() > 1024 * 1024 {
-            return Ok(false);
-        }
-        let bytes = reader.read_to_bytes().await?;
-        match String::from_utf8(bytes.to_vec()) {
-            Ok(text) => {
-                let mut db = self.index.lock().unwrap();
-                db.insert(text, hash);
-                Ok(true)
-            }
-            Err(_err) => Ok(false),
-        }
-    }
-}
-
-/// Read a blob from the local blob store and print it to STDOUT.
-async fn read_and_print(blobs: &MemClient, hash: Hash) -> Result<()> {
-    let content = blobs.read_to_bytes(hash).await?;
-    let message = String::from_utf8(content.to_vec())?;
-    println!("{}: {message}", hash.fmt_short());
-    Ok(())
-}
-
-/// Set the RUST_LOG env var to one of {debug,info,warn} to see logging.
-fn setup_logging() {
-    tracing_subscriber::registry()
-        .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr))
-        .with(EnvFilter::from_default_env())
-        .try_init()
-        .ok();
-}
diff --git a/examples/discovery-local-network.rs b/examples/discovery-local-network.rs
deleted file mode 100644
index bf8fccbcf..000000000
--- a/examples/discovery-local-network.rs
+++ /dev/null
@@ -1,266 +0,0 @@
-//! Example that runs and iroh node with local node discovery and no relay server
-//!
-//! Run the follow command to run the "accept" side, that hosts the content:
-//!  $ cargo run --example discovery_local_network --features="discovery-local-network" -- accept [FILE_PATH]
-//! Wait for output that looks like the following:
-//!  $ cargo run --example discovery_local_network --features="discovery-local-network" -- connect [NODE_ID] [HASH] -o [FILE_PATH]
-//! Run that command on another machine in the same local network, replacing [FILE_PATH] to the path on which you want to save the transferred content.
-use std::path::PathBuf;
-
-use anyhow::ensure;
-use clap::{Parser, Subcommand};
-use iroh::{
-    discovery::mdns::MdnsDiscovery, protocol::Router, Endpoint, NodeAddr, PublicKey, RelayMode,
-    SecretKey,
-};
-use iroh_blobs::{net_protocol::Blobs, rpc::client::blobs::WrapOption, Hash};
-use tracing_subscriber::{prelude::*, EnvFilter};
-
-use self::progress::show_download_progress;
-
-// set the RUST_LOG env var to one of {debug,info,warn} to see logging info
-pub fn setup_logging() {
-    tracing_subscriber::registry()
-        .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr))
-        .with(EnvFilter::from_default_env())
-        .try_init()
-        .ok();
-}
-
-#[derive(Debug, Parser)]
-#[command(version, about)]
-pub struct Cli {
-    #[clap(subcommand)]
-    command: Commands,
-}
-
-#[derive(Subcommand, Clone, Debug)]
-pub enum Commands {
-    /// Launch an iroh node and provide the content at the given path
-    Accept {
-        /// path to the file you want to provide
-        path: PathBuf,
-    },
-    /// Get the node_id and hash string from a node running accept in the local network
-    /// Download the content from that node.
-    Connect {
-        /// Node ID of a node on the local network
-        node_id: PublicKey,
-        /// Hash of content you want to download from the node
-        hash: Hash,
-        /// save the content to a file
-        #[clap(long, short)]
-        out: Option<PathBuf>,
-    },
-}
-
-#[tokio::main]
-async fn main() -> anyhow::Result<()> {
-    setup_logging();
-    let cli = Cli::parse();
-
-    let key = SecretKey::generate(rand::rngs::OsRng);
-    let discovery = MdnsDiscovery::new(key.public())?;
-
-    println!("Starting iroh node with mdns discovery...");
-    // create a new node
-    let endpoint = Endpoint::builder()
-        .secret_key(key)
-        .discovery(Box::new(discovery))
-        .relay_mode(RelayMode::Disabled)
-        .bind()
-        .await?;
-    let builder = Router::builder(endpoint);
-    let blobs = Blobs::memory().build(builder.endpoint());
-    let builder = builder.accept(iroh_blobs::ALPN, blobs.clone());
-    let node = builder.spawn();
-    let blobs_client = blobs.client();
-
-    match &cli.command {
-        Commands::Accept { path } => {
-            if !path.is_file() {
-                println!("Content must be a file.");
-                node.shutdown().await?;
-                return Ok(());
-            }
-            let absolute = path.canonicalize()?;
-            println!("Adding {} as {}...", path.display(), absolute.display());
-            let stream = blobs_client
-                .add_from_path(
-                    absolute,
-                    true,
-                    iroh_blobs::util::SetTagOption::Auto,
-                    WrapOption::NoWrap,
-                )
-                .await?;
-            let outcome = stream.finish().await?;
-            println!("To fetch the blob:\n\tcargo run --example discovery_local_network --features=\"discovery-local-network\" -- connect {} {} -o [FILE_PATH]", node.endpoint().node_id(), outcome.hash);
-            tokio::signal::ctrl_c().await?;
-            node.shutdown().await?;
-            std::process::exit(0);
-        }
-        Commands::Connect { node_id, hash, out } => {
-            println!("NodeID: {}", node.endpoint().node_id());
-            let mut stream = blobs_client
-                .download(*hash, NodeAddr::new(*node_id))
-                .await?;
-            show_download_progress(*hash, &mut stream).await?;
-            if let Some(path) = out {
-                let absolute = std::env::current_dir()?.join(path);
-                ensure!(!absolute.is_dir(), "output must not be a directory");
-                tracing::info!(
-                    "exporting {hash} to {} -> {}",
-                    path.display(),
-                    absolute.display()
-                );
-                let stream = blobs_client
-                    .export(
-                        *hash,
-                        absolute,
-                        iroh_blobs::store::ExportFormat::Blob,
-                        iroh_blobs::store::ExportMode::Copy,
-                    )
-                    .await?;
-                stream.await?;
-            }
-        }
-    }
-    Ok(())
-}
-
-mod progress {
-    use anyhow::{bail, Result};
-    use console::style;
-    use futures_lite::{Stream, StreamExt};
-    use indicatif::{
-        HumanBytes, HumanDuration, MultiProgress, ProgressBar, ProgressDrawTarget, ProgressState,
-        ProgressStyle,
-    };
-    use iroh_blobs::{
-        get::{db::DownloadProgress, progress::BlobProgress, Stats},
-        Hash,
-    };
-
-    pub async fn show_download_progress(
-        hash: Hash,
-        mut stream: impl Stream<Item = Result<DownloadProgress>> + Unpin,
-    ) -> Result<()> {
-        eprintln!("Fetching: {}", hash);
-        let mp = MultiProgress::new();
-        mp.set_draw_target(ProgressDrawTarget::stderr());
-        let op = mp.add(make_overall_progress());
-        let ip = mp.add(make_individual_progress());
-        op.set_message(format!("{} Connecting ...\n", style("[1/3]").bold().dim()));
-        let mut seq = false;
-        while let Some(x) = stream.next().await {
-            match x? {
-                DownloadProgress::InitialState(state) => {
-                    if state.connected {
-                        op.set_message(format!("{} Requesting ...\n", style("[2/3]").bold().dim()));
-                    }
-                    if let Some(count) = state.root.child_count {
-                        op.set_message(format!(
-                            "{} Downloading {} blob(s)\n",
-                            style("[3/3]").bold().dim(),
-                            count + 1,
-                        ));
-                        op.set_length(count + 1);
-                        op.reset();
-                        op.set_position(state.current.map(u64::from).unwrap_or(0));
-                        seq = true;
-                    }
-                    if let Some(blob) = state.get_current() {
-                        if let Some(size) = blob.size {
-                            ip.set_length(size.value());
-                            ip.reset();
-                            match blob.progress {
-                                BlobProgress::Pending => {}
-                                BlobProgress::Progressing(offset) => ip.set_position(offset),
-                                BlobProgress::Done => ip.finish_and_clear(),
-                            }
-                            if !seq {
-                                op.finish_and_clear();
-                            }
-                        }
-                    }
-                }
-                DownloadProgress::FoundLocal { .. } => {}
-                DownloadProgress::Connected => {
-                    op.set_message(format!("{} Requesting ...\n", style("[2/3]").bold().dim()));
-                }
-                DownloadProgress::FoundHashSeq { children, .. } => {
-                    op.set_message(format!(
-                        "{} Downloading {} blob(s)\n",
-                        style("[3/3]").bold().dim(),
-                        children + 1,
-                    ));
-                    op.set_length(children + 1);
-                    op.reset();
-                    seq = true;
-                }
-                DownloadProgress::Found { size, child, .. } => {
-                    if seq {
-                        op.set_position(child.into());
-                    } else {
-                        op.finish_and_clear();
-                    }
-                    ip.set_length(size);
-                    ip.reset();
-                }
-                DownloadProgress::Progress { offset, .. } => {
-                    ip.set_position(offset);
-                }
-                DownloadProgress::Done { .. } => {
-                    ip.finish_and_clear();
-                }
-                DownloadProgress::AllDone(Stats {
-                    bytes_read,
-                    elapsed,
-                    ..
-                }) => {
-                    op.finish_and_clear();
-                    eprintln!(
-                        "Transferred {} in {}, {}/s",
-                        HumanBytes(bytes_read),
-                        HumanDuration(elapsed),
-                        HumanBytes((bytes_read as f64 / elapsed.as_secs_f64()) as u64)
-                    );
-                    break;
-                }
-                DownloadProgress::Abort(e) => {
-                    bail!("download aborted: {}", e);
-                }
-            }
-        }
-        Ok(())
-    }
-    fn make_overall_progress() -> ProgressBar {
-        let pb = ProgressBar::hidden();
-        pb.enable_steady_tick(std::time::Duration::from_millis(100));
-        pb.set_style(
-            ProgressStyle::with_template(
-                "{msg}{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {pos}/{len}",
-            )
-            .unwrap()
-            .progress_chars("#>-"),
-        );
-        pb
-    }
-
-    fn make_individual_progress() -> ProgressBar {
-        let pb = ProgressBar::hidden();
-        pb.enable_steady_tick(std::time::Duration::from_millis(100));
-        pb.set_style(
-        ProgressStyle::with_template("{msg}{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({eta})")
-            .unwrap()
-            .with_key(
-                "eta",
-                |state: &ProgressState, w: &mut dyn std::fmt::Write| {
-                    write!(w, "{:.1}s", state.eta().as_secs_f64()).unwrap()
-                },
-            )
-            .progress_chars("#>-"),
-    );
-        pb
-    }
-}
diff --git a/examples/fetch-fsm.rs b/examples/fetch-fsm.rs
deleted file mode 100644
index 4e0422043..000000000
--- a/examples/fetch-fsm.rs
+++ /dev/null
@@ -1,160 +0,0 @@
-//! An example how to download a single blob or collection from a node and write it to stdout using the `get` finite state machine directly.
-//!
-//! Since this example does not use [`iroh-net::Endpoint`], it does not do any holepunching, and so will only work locally or between two processes that have public IP addresses.
-//!
-//! Run the provide-bytes example first. It will give instructions on how to run this example properly.
-use std::str::FromStr;
-
-use anyhow::{Context, Result};
-use iroh_blobs::{
-    get::fsm::{AtInitial, ConnectedNext, EndBlobNext},
-    hashseq::HashSeq,
-    protocol::GetRequest,
-    BlobFormat,
-};
-use iroh_io::ConcatenateSliceWriter;
-use tracing_subscriber::{prelude::*, EnvFilter};
-
-const EXAMPLE_ALPN: &[u8] = b"n0/iroh/examples/bytes/0";
-
-// set the RUST_LOG env var to one of {debug,info,warn} to see logging info
-pub fn setup_logging() {
-    tracing_subscriber::registry()
-        .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr))
-        .with(EnvFilter::from_default_env())
-        .try_init()
-        .ok();
-}
-
-#[tokio::main]
-async fn main() -> Result<()> {
-    println!("\nfetch fsm example!");
-    setup_logging();
-    let args: Vec<_> = std::env::args().collect();
-    if args.len() != 2 {
-        anyhow::bail!("usage: fetch-fsm [TICKET]");
-    }
-    let ticket =
-        iroh_blobs::ticket::BlobTicket::from_str(&args[1]).context("unable to parse [TICKET]")?;
-
-    let (node, hash, format) = ticket.into_parts();
-
-    // create an endpoint to listen for incoming connections
-    let endpoint = iroh::Endpoint::builder()
-        .relay_mode(iroh::RelayMode::Disabled)
-        .alpns(vec![EXAMPLE_ALPN.into()])
-        .bind()
-        .await?;
-    println!(
-        "\nlistening on {:?}",
-        endpoint.node_addr().await?.direct_addresses
-    );
-    println!("fetching hash {hash} from {:?}", node.node_id);
-
-    // connect
-    let connection = endpoint.connect(node, EXAMPLE_ALPN).await?;
-
-    match format {
-        BlobFormat::HashSeq => {
-            // create a request for a collection
-            let request = GetRequest::all(hash);
-            // create the initial state of the finite state machine
-            let initial = iroh_blobs::get::fsm::start(connection, request);
-
-            write_collection(initial).await
-        }
-        BlobFormat::Raw => {
-            // create a request for a single blob
-            let request = GetRequest::single(hash);
-            // create the initial state of the finite state machine
-            let initial = iroh_blobs::get::fsm::start(connection, request);
-
-            write_blob(initial).await
-        }
-    }
-}
-
-async fn write_blob(initial: AtInitial) -> Result<()> {
-    // connect (create a stream pair)
-    let connected = initial.next().await?;
-
-    // we expect a start root message, since we requested a single blob
-    let ConnectedNext::StartRoot(start_root) = connected.next().await? else {
-        panic!("expected start root")
-    };
-    // we can just call next to proceed to the header, since we know the root hash
-    let header = start_root.next();
-
-    // we need to wrap stdout in a struct that implements AsyncSliceWriter. Since we can not
-    // seek in stdout we use ConcatenateSliceWriter which just concatenates all the writes.
-    let writer = ConcatenateSliceWriter::new(tokio::io::stdout());
-
-    // make the spacing nicer in the terminal
-    println!();
-    // use the utility function write_all to write the entire blob
-    let end = header.write_all(writer).await?;
-
-    // we requested a single blob, so we expect to enter the closing state
-    let EndBlobNext::Closing(closing) = end.next() else {
-        panic!("expected closing")
-    };
-
-    // close the connection and get the stats
-    let _stats = closing.next().await?;
-    Ok(())
-}
-
-async fn write_collection(initial: AtInitial) -> Result<()> {
-    // connect
-    let connected = initial.next().await?;
-    // read the first bytes
-    let ConnectedNext::StartRoot(start_root) = connected.next().await? else {
-        anyhow::bail!("failed to parse collection");
-    };
-    // check that we requested the whole collection
-    if !start_root.ranges().is_all() {
-        anyhow::bail!("collection was not requested completely");
-    }
-
-    // move to the header
-    let header: iroh_blobs::get::fsm::AtBlobHeader = start_root.next();
-    let (root_end, hashes_bytes) = header.concatenate_into_vec().await?;
-    let next = root_end.next();
-    let EndBlobNext::MoreChildren(at_meta) = next else {
-        anyhow::bail!("missing meta blob, got {next:?}");
-    };
-    // parse the hashes from the hash sequence bytes
-    let hashes = HashSeq::try_from(bytes::Bytes::from(hashes_bytes))
-        .context("failed to parse hashes")?
-        .into_iter()
-        .collect::<Vec<_>>();
-    let meta_hash = hashes.first().context("missing meta hash")?;
-
-    let (meta_end, _meta_bytes) = at_meta.next(*meta_hash).concatenate_into_vec().await?;
-    let mut curr = meta_end.next();
-    let closing = loop {
-        match curr {
-            EndBlobNext::MoreChildren(more) => {
-                let Some(hash) = hashes.get(more.child_offset() as usize) else {
-                    break more.finish();
-                };
-                let header = more.next(*hash);
-
-                // we need to wrap stdout in a struct that implements AsyncSliceWriter. Since we can not
-                // seek in stdout we use ConcatenateSliceWriter which just concatenates all the writes.
-                let writer = ConcatenateSliceWriter::new(tokio::io::stdout());
-
-                // use the utility function write_all to write the entire blob
-                let end = header.write_all(writer).await?;
-                println!();
-                curr = end.next();
-            }
-            EndBlobNext::Closing(closing) => {
-                break closing;
-            }
-        }
-    };
-    // close the connection
-    let _stats = closing.next().await?;
-    Ok(())
-}
diff --git a/examples/fetch-stream.rs b/examples/fetch-stream.rs
deleted file mode 100644
index f9405abca..000000000
--- a/examples/fetch-stream.rs
+++ /dev/null
@@ -1,228 +0,0 @@
-//! An example how to download a single blob or collection from a node and write it to stdout, using a helper method to turn the `get` finite state machine into a stream.
-//!
-//! Since this example does not use [`iroh-net::Endpoint`], it does not do any holepunching, and so will only work locally or between two processes that have public IP addresses.
-//!
-//! Run the provide-bytes example first. It will give instructions on how to run this example properly.
-use std::{io, str::FromStr};
-
-use anyhow::{Context, Result};
-use bao_tree::io::fsm::BaoContentItem;
-use bytes::Bytes;
-use futures_lite::{Stream, StreamExt};
-use genawaiter::sync::{Co, Gen};
-use iroh_blobs::{
-    get::fsm::{AtInitial, BlobContentNext, ConnectedNext, EndBlobNext},
-    hashseq::HashSeq,
-    protocol::GetRequest,
-    BlobFormat,
-};
-use tokio::io::AsyncWriteExt;
-use tracing_subscriber::{prelude::*, EnvFilter};
-
-const EXAMPLE_ALPN: &[u8] = b"n0/iroh/examples/bytes/0";
-
-// set the RUST_LOG env var to one of {debug,info,warn} to see logging info
-pub fn setup_logging() {
-    tracing_subscriber::registry()
-        .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr))
-        .with(EnvFilter::from_default_env())
-        .try_init()
-        .ok();
-}
-
-#[tokio::main]
-async fn main() -> Result<()> {
-    println!("\nfetch stream example!");
-    setup_logging();
-    let args: Vec<_> = std::env::args().collect();
-    if args.len() != 2 {
-        anyhow::bail!("usage: fetch-stream [TICKET]");
-    }
-    let ticket =
-        iroh_blobs::ticket::BlobTicket::from_str(&args[1]).context("unable to parse [TICKET]")?;
-
-    let (node, hash, format) = ticket.into_parts();
-
-    // create an endpoint to listen for incoming connections
-    let endpoint = iroh::Endpoint::builder()
-        .relay_mode(iroh::RelayMode::Disabled)
-        .alpns(vec![EXAMPLE_ALPN.into()])
-        .bind()
-        .await?;
-    println!(
-        "\nlistening on {:?}",
-        endpoint.node_addr().await?.direct_addresses
-    );
-    println!("fetching hash {hash} from {:?}", node.node_id);
-
-    // connect
-    let connection = endpoint.connect(node, EXAMPLE_ALPN).await?;
-
-    let mut stream = match format {
-        BlobFormat::HashSeq => {
-            // create a request for a collection
-            let request = GetRequest::all(hash);
-
-            // create the initial state of the finite state machine
-            let initial = iroh_blobs::get::fsm::start(connection, request);
-
-            // create a stream that yields all the data of the blob
-            stream_children(initial).boxed_local()
-        }
-        BlobFormat::Raw => {
-            // create a request for a single blob
-            let request = GetRequest::single(hash);
-
-            // create the initial state of the finite state machine
-            let initial = iroh_blobs::get::fsm::start(connection, request);
-
-            // create a stream that yields all the data of the blob
-            stream_blob(initial).boxed_local()
-        }
-    };
-    while let Some(item) = stream.next().await {
-        let item = item?;
-        tokio::io::stdout().write_all(&item).await?;
-        println!();
-    }
-    Ok(())
-}
-
-/// Stream the response for a request for a single blob.
-///
-/// If the request was for a part of the blob, this will stream just the requested
-/// blocks.
-///
-/// This will stream the root blob and close the connection.
-fn stream_blob(initial: AtInitial) -> impl Stream<Item = io::Result<Bytes>> + 'static {
-    async fn inner(initial: AtInitial, co: &Co<io::Result<Bytes>>) -> io::Result<()> {
-        // connect
-        let connected = initial.next().await?;
-        // read the first bytes
-        let ConnectedNext::StartRoot(start_root) = connected.next().await? else {
-            return Err(io::Error::new(io::ErrorKind::Other, "expected start root"));
-        };
-        // move to the header
-        let header = start_root.next();
-        // get the size of the content
-        let (mut content, _size) = header.next().await?;
-        // manually loop over the content and yield all data
-        let done = loop {
-            match content.next().await {
-                BlobContentNext::More((next, data)) => {
-                    if let BaoContentItem::Leaf(leaf) = data? {
-                        // yield the data
-                        co.yield_(Ok(leaf.data)).await;
-                    }
-                    content = next;
-                }
-                BlobContentNext::Done(done) => {
-                    // we are done with the root blob
-                    break done;
-                }
-            }
-        };
-        // close the connection even if there is more data
-        let closing = match done.next() {
-            EndBlobNext::Closing(closing) => closing,
-            EndBlobNext::MoreChildren(more) => more.finish(),
-        };
-        // close the connection
-        let _stats = closing.next().await?;
-        Ok(())
-    }
-
-    Gen::new(|co| async move {
-        if let Err(e) = inner(initial, &co).await {
-            co.yield_(Err(e)).await;
-        }
-    })
-}
-
-/// Stream the response for a request for an iroh collection and its children.
-///
-/// If the request was for a part of the children, this will stream just the requested
-/// blocks.
-///
-/// The root blob is not streamed. It must be fully included in the response.
-fn stream_children(initial: AtInitial) -> impl Stream<Item = io::Result<Bytes>> + 'static {
-    async fn inner(initial: AtInitial, co: &Co<io::Result<Bytes>>) -> io::Result<()> {
-        // connect
-        let connected = initial.next().await?;
-        // read the first bytes
-        let ConnectedNext::StartRoot(start_root) = connected.next().await? else {
-            return Err(io::Error::new(
-                io::ErrorKind::Other,
-                "failed to parse collection",
-            ));
-        };
-        // check that we requested the whole collection
-        if !start_root.ranges().is_all() {
-            return Err(io::Error::new(
-                io::ErrorKind::Other,
-                "collection was not requested completely",
-            ));
-        }
-        // move to the header
-        let header: iroh_blobs::get::fsm::AtBlobHeader = start_root.next();
-        let (root_end, hashes_bytes) = header.concatenate_into_vec().await?;
-
-        // parse the hashes from the hash sequence bytes
-        let hashes = HashSeq::try_from(bytes::Bytes::from(hashes_bytes))
-            .map_err(|e| {
-                io::Error::new(io::ErrorKind::Other, format!("failed to parse hashes: {e}"))
-            })?
-            .into_iter()
-            .collect::<Vec<_>>();
-
-        let next = root_end.next();
-        let EndBlobNext::MoreChildren(at_meta) = next else {
-            return Err(io::Error::new(io::ErrorKind::Other, "missing meta blob"));
-        };
-        let meta_hash = hashes
-            .first()
-            .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "missing meta link"))?;
-        let (meta_end, _meta_bytes) = at_meta.next(*meta_hash).concatenate_into_vec().await?;
-        let mut curr = meta_end.next();
-        let closing = loop {
-            match curr {
-                EndBlobNext::MoreChildren(more) => {
-                    let Some(hash) = hashes.get(more.child_offset() as usize) else {
-                        break more.finish();
-                    };
-                    let header = more.next(*hash);
-                    let (mut content, _size) = header.next().await?;
-                    // manually loop over the content and yield all data
-                    let done = loop {
-                        match content.next().await {
-                            BlobContentNext::More((next, data)) => {
-                                if let BaoContentItem::Leaf(leaf) = data? {
-                                    // yield the data
-                                    co.yield_(Ok(leaf.data)).await;
-                                }
-                                content = next;
-                            }
-                            BlobContentNext::Done(done) => {
-                                // we are done with the root blob
-                                break done;
-                            }
-                        }
-                    };
-                    curr = done.next();
-                }
-                EndBlobNext::Closing(closing) => {
-                    break closing;
-                }
-            }
-        };
-        // close the connection
-        let _stats = closing.next().await?;
-        Ok(())
-    }
-
-    Gen::new(|co| async move {
-        if let Err(e) = inner(initial, &co).await {
-            co.yield_(Err(e)).await;
-        }
-    })
-}
diff --git a/examples/hello-world-fetch.rs b/examples/hello-world-fetch.rs
deleted file mode 100644
index 93ae9bdea..000000000
--- a/examples/hello-world-fetch.rs
+++ /dev/null
@@ -1,89 +0,0 @@
-//! An example that fetches an iroh blob and prints the contents.
-//! Will only work with blobs and collections that contain text, and is meant as a companion to the `hello-world-get` examples.
-//!
-//! This is using an in memory database and a random node id.
-//! Run the `provide` example, which will give you instructions on how to run this example.
-use std::{env, str::FromStr};
-
-use anyhow::{bail, ensure, Context, Result};
-use iroh::{protocol::Router, Endpoint};
-use iroh_blobs::{net_protocol::Blobs, ticket::BlobTicket, BlobFormat};
-use tracing_subscriber::{prelude::*, EnvFilter};
-
-// set the RUST_LOG env var to one of {debug,info,warn} to see logging info
-pub fn setup_logging() {
-    tracing_subscriber::registry()
-        .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr))
-        .with(EnvFilter::from_default_env())
-        .try_init()
-        .ok();
-}
-
-#[tokio::main]
-async fn main() -> Result<()> {
-    setup_logging();
-    println!("\n'Hello World' fetch example!");
-    // get the ticket
-    let args: Vec<String> = env::args().collect();
-
-    if args.len() != 2 {
-        bail!("expected one argument [BLOB_TICKET]\n\nGet a ticket by running the follow command in a separate terminal:\n\n`cargo run --example hello-world-provide`");
-    }
-
-    // deserialize ticket string into a ticket
-    let ticket =
-        BlobTicket::from_str(&args[1]).context("failed parsing blob ticket\n\nGet a ticket by running the follow command in a separate terminal:\n\n`cargo run --example hello-world-provide`")?;
-
-    // create a new node
-    let endpoint = Endpoint::builder().bind().await?;
-    let builder = Router::builder(endpoint);
-    let blobs = Blobs::memory().build(builder.endpoint());
-    let builder = builder.accept(iroh_blobs::ALPN, blobs.clone());
-    let node = builder.spawn();
-    let blobs_client = blobs.client();
-
-    println!("fetching hash:  {}", ticket.hash());
-    println!("node id:        {}", node.endpoint().node_id());
-    println!("node listening addresses:");
-    let addrs = node.endpoint().node_addr().await?;
-    for addr in addrs.direct_addresses() {
-        println!("\t{:?}", addr);
-    }
-    println!(
-        "node relay server url: {:?}",
-        node.endpoint()
-            .home_relay()
-            .get()?
-            .expect("a default relay url should be provided")
-            .to_string()
-    );
-
-    // If the `BlobFormat` is `Raw`, we have the hash for a single blob, and simply need to read the blob using the `blobs` API on the client to get the content.
-    ensure!(
-        ticket.format() == BlobFormat::Raw,
-        "'Hello World' example expects to fetch a single blob, but the ticket indicates a collection.",
-    );
-
-    // `download` returns a stream of `DownloadProgress` events. You can iterate through these updates to get progress
-    // on the state of your download.
-    let download_stream = blobs_client
-        .download(ticket.hash(), ticket.node_addr().clone())
-        .await?;
-
-    // You can also just `await` the stream, which will poll the `DownloadProgress` stream for you.
-    let outcome = download_stream.await.context("unable to download hash")?;
-
-    println!(
-        "\ndownloaded {} bytes from node {}",
-        outcome.downloaded_size,
-        ticket.node_addr().node_id
-    );
-
-    // Get the content we have just fetched from the iroh database.
-
-    let bytes = blobs_client.read_to_bytes(ticket.hash()).await?;
-    let s = std::str::from_utf8(&bytes).context("unable to parse blob as as utf-8 string")?;
-    println!("{s}");
-
-    Ok(())
-}
diff --git a/examples/hello-world-provide.rs b/examples/hello-world-provide.rs
deleted file mode 100644
index a89803047..000000000
--- a/examples/hello-world-provide.rs
+++ /dev/null
@@ -1,61 +0,0 @@
-//! The smallest possible example to spin up a node and serve a single blob.
-//!
-//! This is using an in memory database and a random node id.
-//! run this example from the project root:
-//!     $ cargo run --example hello-world-provide
-use iroh::{protocol::Router, Endpoint};
-use iroh_blobs::{net_protocol::Blobs, ticket::BlobTicket};
-use tracing_subscriber::{prelude::*, EnvFilter};
-
-// set the RUST_LOG env var to one of {debug,info,warn} to see logging info
-pub fn setup_logging() {
-    tracing_subscriber::registry()
-        .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr))
-        .with(EnvFilter::from_default_env())
-        .try_init()
-        .ok();
-}
-
-#[tokio::main]
-async fn main() -> anyhow::Result<()> {
-    setup_logging();
-    println!("'Hello World' provide example!");
-
-    // create a new node
-    let endpoint = Endpoint::builder().bind().await?;
-    let builder = Router::builder(endpoint);
-    let blobs = Blobs::memory().build(builder.endpoint());
-    let builder = builder.accept(iroh_blobs::ALPN, blobs.clone());
-    let blobs_client = blobs.client();
-    let node = builder.spawn();
-
-    // add some data and remember the hash
-    let res = blobs_client.add_bytes("Hello, world!").await?;
-
-    // create a ticket
-    let addr = node.endpoint().node_addr().await?;
-    let ticket = BlobTicket::new(addr, res.hash, res.format)?;
-
-    // print some info about the node
-    println!("serving hash:    {}", ticket.hash());
-    println!("node id:         {}", ticket.node_addr().node_id);
-    println!("node listening addresses:");
-    for addr in ticket.node_addr().direct_addresses() {
-        println!("\t{:?}", addr);
-    }
-    println!(
-        "node relay server url: {:?}",
-        ticket
-            .node_addr()
-            .relay_url()
-            .expect("a default relay url should be provided")
-            .to_string()
-    );
-    // print the ticket, containing all the above information
-    println!("\nin another terminal, run:");
-    println!("\t cargo run --example hello-world-fetch {}", ticket);
-    // block until SIGINT is received (ctrl+c)
-    tokio::signal::ctrl_c().await?;
-    node.shutdown().await?;
-    Ok(())
-}
diff --git a/examples/provide-bytes.rs b/examples/provide-bytes.rs
deleted file mode 100644
index 7f120ed3b..000000000
--- a/examples/provide-bytes.rs
+++ /dev/null
@@ -1,124 +0,0 @@
-//! An example that provides a blob or a collection over a Quinn connection.
-//!
-//! Since this example does not use [`iroh-net::Endpoint`], it does not do any holepunching, and so will only work locally or between two processes that have public IP addresses.
-//!
-//! Run this example with
-//!    cargo run --example provide-bytes blob
-//! To provide a blob (single file)
-//!
-//! Run this example with
-//!    cargo run --example provide-bytes collection
-//! To provide a collection (multiple blobs)
-use anyhow::Result;
-use iroh_blobs::{format::collection::Collection, util::local_pool::LocalPool, BlobFormat, Hash};
-use tracing::warn;
-use tracing_subscriber::{prelude::*, EnvFilter};
-
-const EXAMPLE_ALPN: &[u8] = b"n0/iroh/examples/bytes/0";
-
-// set the RUST_LOG env var to one of {debug,info,warn} to see logging info
-pub fn setup_logging() {
-    tracing_subscriber::registry()
-        .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr))
-        .with(EnvFilter::from_default_env())
-        .try_init()
-        .ok();
-}
-
-#[tokio::main]
-async fn main() -> Result<()> {
-    let args: Vec<_> = std::env::args().collect();
-    if args.len() != 2 {
-        anyhow::bail!(
-            "usage: provide-bytes [FORMAT], where [FORMAT] is either 'blob' or 'collection'\n\nThe 'blob' example demonstrates sending a single blob of bytes. The 'collection' example demonstrates sending multiple blobs of bytes, grouped together in a 'collection'."
-        );
-    }
-    let format = {
-        if args[1] != "blob" && args[1] != "collection" {
-            anyhow::bail!(
-                "expected either 'blob' or 'collection' for FORMAT argument, got {}",
-                args[1]
-            );
-        }
-        args[1].clone()
-    };
-    println!("\nprovide bytes {format} example!");
-
-    let (db, hash, format) = if format == "collection" {
-        let (mut db, names) = iroh_blobs::store::readonly_mem::Store::new([
-            ("blob1", b"the first blob of bytes".to_vec()),
-            ("blob2", b"the second blob of bytes".to_vec()),
-        ]); // create a collection
-        let collection: Collection = names
-            .into_iter()
-            .map(|(name, hash)| (name, Hash::from(hash)))
-            .collect();
-        // add it to the db
-        let hash = db.insert_many(collection.to_blobs()).unwrap();
-        (db, hash, BlobFormat::HashSeq)
-    } else {
-        // create a new database and add a blob
-        let (db, names) =
-            iroh_blobs::store::readonly_mem::Store::new([("hello", b"Hello World!".to_vec())]);
-
-        // get the hash of the content
-        let hash = names.get("hello").unwrap();
-        (db, Hash::from(hash.as_bytes()), BlobFormat::Raw)
-    };
-
-    // create an endpoint to listen for incoming connections
-    let endpoint = iroh::Endpoint::builder()
-        .relay_mode(iroh::RelayMode::Disabled)
-        .alpns(vec![EXAMPLE_ALPN.into()])
-        .bind()
-        .await?;
-    let addr = endpoint.node_addr().await?;
-    println!("\nlistening on {:?}", addr.direct_addresses);
-    println!("providing hash {hash}");
-
-    let ticket = iroh_blobs::ticket::BlobTicket::new(addr, hash, format)?;
-
-    println!("\nfetch the content using a finite state machine by running the following example:\n\ncargo run --example fetch-fsm {ticket}");
-    println!("\nfetch the content using a stream by running the following example:\n\ncargo run --example fetch-stream {ticket}\n");
-
-    // create a new local pool handle with 1 worker thread
-    let lp = LocalPool::single();
-
-    let accept_task = tokio::spawn(async move {
-        while let Some(incoming) = endpoint.accept().await {
-            println!("connection incoming");
-
-            let conn = match incoming.accept() {
-                Ok(conn) => conn,
-                Err(err) => {
-                    warn!("incoming connection failed: {err:#}");
-                    // we can carry on in these cases:
-                    // this can be caused by retransmitted datagrams
-                    continue;
-                }
-            };
-            let db = db.clone();
-            let lp = lp.clone();
-
-            // spawn a task to handle the connection
-            tokio::spawn(async move {
-                let conn = match conn.await {
-                    Ok(conn) => conn,
-                    Err(err) => {
-                        warn!("Error connecting: {err:#}");
-                        return;
-                    }
-                };
-                iroh_blobs::provider::handle_connection(conn, db, Default::default(), lp).await
-            });
-        }
-    });
-
-    match tokio::signal::ctrl_c().await {
-        Ok(()) => {
-            accept_task.abort();
-            Ok(())
-        }
-        Err(e) => Err(anyhow::anyhow!("unable to listen for ctrl-c: {e}")),
-    }
-}
diff --git a/examples/random_store.rs b/examples/random_store.rs
new file mode 100644
index 000000000..74e18919f
--- /dev/null
+++ b/examples/random_store.rs
@@ -0,0 +1,292 @@
+use std::{env, path::PathBuf, str::FromStr};
+
+use anyhow::{Context, Result};
+use clap::{Parser, Subcommand};
+use iroh::{SecretKey, Watcher};
+use iroh_base::ticket::NodeTicket;
+use iroh_blobs::{
+    api::downloader::Shuffled,
+    provider::Event,
+    store::fs::FsStore,
+    test::{add_hash_sequences, create_random_blobs},
+    HashAndFormat,
+};
+use n0_future::StreamExt;
+use rand::{rngs::StdRng, Rng, SeedableRng};
+use tokio::{signal::ctrl_c, sync::mpsc};
+use tracing::info;
+
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+pub struct Args {
+    /// Commands to run
+    #[command(subcommand)]
+    pub command: Commands,
+}
+
+#[derive(Parser, Debug)]
+pub struct CommonArgs {
+    /// Random seed for reproducible results
+    #[arg(long)]
+    pub seed: Option<u64>,
+
+    /// Path for store, none for in-memory store
+    #[arg(long)]
+    pub path: Option<PathBuf>,
+}
+
+#[derive(Subcommand, Debug)]
+pub enum Commands {
+    /// Provide content to the network
+    Provide(ProvideArgs),
+    /// Request content from the network
+    Request(RequestArgs),
+}
+
+#[derive(Parser, Debug)]
+pub struct ProvideArgs {
+    #[command(flatten)]
+    pub common: CommonArgs,
+
+    /// Number of blobs to generate
+    #[arg(long, default_value_t = 100)]
+    pub num_blobs: usize,
+
+    /// Size of each blob in bytes
+    #[arg(long, default_value_t = 100000)]
+    pub blob_size: usize,
+
+    /// Number of hash sequences
+    #[arg(long, default_value_t = 1)]
+    pub hash_seqs: usize,
+
+    /// Size of each hash sequence
+    #[arg(long, default_value_t = 100)]
+    pub hash_seq_size: usize,
+
+    /// Size of each hash sequence
+    #[arg(long, default_value_t = false)]
+    pub allow_push: bool,
+}
+
+#[derive(Parser, Debug)]
+pub struct RequestArgs {
+    #[command(flatten)]
+    pub common: CommonArgs,
+
+    /// Hash of the blob to request
+    #[arg(long)]
+    pub content: Vec<HashAndFormat>,
+
+    /// Nodes to request from
+    pub nodes: Vec<NodeTicket>,
+
+    /// Split large requests
+    #[arg(long, default_value_t = false)]
+    pub split: bool,
+}
+
+pub fn get_or_generate_secret_key() -> Result<SecretKey> {
+    if let Ok(secret) = env::var("IROH_SECRET") {
+        // Parse the secret key from string
+        SecretKey::from_str(&secret).context("Invalid secret key format")
+    } else {
+        // Generate a new random key
+        let secret_key = SecretKey::generate(&mut rand::thread_rng());
+        let secret_key_str = hex::encode(secret_key.to_bytes());
+        println!("Generated new random secret key");
+        println!("To reuse this key, set the IROH_SECRET={secret_key_str}");
+        Ok(secret_key)
+    }
+}
+
+pub fn dump_provider_events(
+    allow_push: bool,
+) -> (
+    tokio::task::JoinHandle<()>,
+    mpsc::Sender<iroh_blobs::provider::Event>,
+) {
+    let (tx, mut rx) = mpsc::channel(100);
+    let dump_task = tokio::spawn(async move {
+        while let Some(event) = rx.recv().await {
+            match event {
+                Event::ClientConnected {
+                    node_id,
+                    connection_id,
+                    permitted,
+                } => {
+                    permitted.send(true).await.ok();
+                    println!("Client connected: {node_id} {connection_id}");
+                }
+                Event::GetRequestReceived {
+                    connection_id,
+                    request_id,
+                    hash,
+                    ranges,
+                } => {
+                    println!(
+                        "Get request received: {connection_id} {request_id} {hash} {ranges:?}"
+                    );
+                }
+                Event::TransferCompleted {
+                    connection_id,
+                    request_id,
+                    stats,
+                } => {
+                    println!("Transfer completed: {connection_id} {request_id} {stats:?}");
+                }
+                Event::TransferAborted {
+                    connection_id,
+                    request_id,
+                    stats,
+                } => {
+                    println!("Transfer aborted: {connection_id} {request_id} {stats:?}");
+                }
+                Event::TransferProgress {
+                    connection_id,
+                    request_id,
+                    index,
+                    end_offset,
+                } => {
+                    info!("Transfer progress: {connection_id} {request_id} {index} {end_offset}");
+                }
+                Event::PushRequestReceived {
+                    connection_id,
+                    request_id,
+                    hash,
+                    ranges,
+                    permitted,
+                } => {
+                    if allow_push {
+                        permitted.send(true).await.ok();
+                        println!(
+                            "Push request received: {connection_id} {request_id} {hash} {ranges:?}"
+                        );
+                    } else {
+                        permitted.send(false).await.ok();
+                        println!(
+                            "Push request denied: {connection_id} {request_id} {hash} {ranges:?}"
+                        );
+                    }
+                }
+                _ => {
+                    info!("Received event: {:?}", event);
+                }
+            }
+        }
+    });
+    (dump_task, tx)
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    tracing_subscriber::fmt::init();
+    let args = Args::parse();
+    match args.command {
+        Commands::Provide(args) => provide(args).await,
+        Commands::Request(args) => request(args).await,
+    }
+}
+
+async fn provide(args: ProvideArgs) -> anyhow::Result<()> {
+    println!("{args:?}");
+    let tempdir = if args.common.path.is_none() {
+        Some(tempfile::tempdir_in(".").context("Failed to create temporary directory")?)
+    } else {
+        None
+    };
+    let path = args
+        .common
+        .path
+        .unwrap_or_else(|| tempdir.as_ref().unwrap().path().to_path_buf());
+    let store = FsStore::load(&path).await?;
+    println!("Using store at: {}", path.display());
+    let mut rng = match args.common.seed {
+        Some(seed) => StdRng::seed_from_u64(seed),
+        None => StdRng::from_entropy(),
+    };
+    let blobs = create_random_blobs(
+        &store,
+        args.num_blobs,
+        |_, rand| rand.gen_range(1..=args.blob_size),
+        &mut rng,
+    )
+    .await?;
+    let hs = add_hash_sequences(
+        &store,
+        &blobs,
+        args.hash_seqs,
+        |_, rand| rand.gen_range(1..=args.hash_seq_size),
+        &mut rng,
+    )
+    .await?;
+    println!(
+        "Created {} blobs and {} hash sequences",
+        blobs.len(),
+        hs.len()
+    );
+    for (i, info) in blobs.iter().enumerate() {
+        println!("blob {i} {}", info.hash_and_format());
+    }
+    for (i, info) in hs.iter().enumerate() {
+        println!("hash_seq {i} {}", info.hash_and_format());
+    }
+    let secret_key = get_or_generate_secret_key()?;
+    let endpoint = iroh::Endpoint::builder()
+        .secret_key(secret_key)
+        .bind()
+        .await?;
+    let (dump_task, events_tx) = dump_provider_events(args.allow_push);
+    let blobs = iroh_blobs::net_protocol::Blobs::new(&store, endpoint.clone(), Some(events_tx));
+    let router = iroh::protocol::Router::builder(endpoint.clone())
+        .accept(iroh_blobs::ALPN, blobs)
+        .spawn();
+    let addr = router.endpoint().node_addr().initialized().await?;
+    let ticket = NodeTicket::from(addr.clone());
+    println!("Node address: {addr:?}");
+    println!("ticket:\n{ticket}");
+    ctrl_c().await?;
+    router.shutdown().await?;
+    dump_task.abort();
+    Ok(())
+}
+
+async fn request(args: RequestArgs) -> anyhow::Result<()> {
+    println!("{args:?}");
+    let tempdir = if args.common.path.is_none() {
+        Some(tempfile::tempdir_in(".").context("Failed to create temporary directory")?)
+    } else {
+        None
+    };
+    let path = args
+        .common
+        .path
+        .unwrap_or_else(|| tempdir.as_ref().unwrap().path().to_path_buf());
+    let store = FsStore::load(&path).await?;
+    println!("Using store at: {}", path.display());
+    let endpoint = iroh::Endpoint::builder().bind().await?;
+    let downloader = store.downloader(&endpoint);
+    for ticket in &args.nodes {
+        endpoint.add_node_addr(ticket.node_addr().clone())?;
+    }
+    let nodes = args
+        .nodes
+        .iter()
+        .map(|ticket| ticket.node_addr().node_id)
+        .collect::<Vec<_>>();
+    for content in args.content {
+        let mut progress = downloader
+            .download(content, Shuffled::new(nodes.clone()))
+            .stream()
+            .await?;
+        while let Some(event) = progress.next().await {
+            info!("Progress: {:?}", event);
+        }
+    }
+    let hashes = store.list().hashes().await?;
+    for hash in hashes {
+        println!("Got {hash}");
+    }
+    store.dump().await?;
+    Ok(())
+}
diff --git a/examples/request.rs b/examples/request.rs
new file mode 100644
index 000000000..3239eee89
--- /dev/null
+++ b/examples/request.rs
@@ -0,0 +1,4 @@
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    Ok(())
+}
diff --git a/examples/transfer.rs b/examples/transfer.rs
deleted file mode 100644
index 60f9e3874..000000000
--- a/examples/transfer.rs
+++ /dev/null
@@ -1,107 +0,0 @@
-use std::path::PathBuf;
-
-use anyhow::Result;
-use iroh::{protocol::Router, Endpoint};
-use iroh_blobs::{
-    net_protocol::Blobs,
-    rpc::client::blobs::WrapOption,
-    store::{ExportFormat, ExportMode},
-    ticket::BlobTicket,
-    util::SetTagOption,
-};
-
-#[tokio::main]
-async fn main() -> Result<()> {
-    // Create an endpoint, it allows creating and accepting
-    // connections in the iroh p2p world
-    let endpoint = Endpoint::builder().discovery_n0().bind().await?;
-    // We initialize the Blobs protocol in-memory
-    let blobs = Blobs::memory().build(&endpoint);
-
-    // Now we build a router that accepts blobs connections & routes them
-    // to the blobs protocol.
-    let router = Router::builder(endpoint)
-        .accept(iroh_blobs::ALPN, blobs.clone())
-        .spawn();
-
-    // We use a blobs client to interact with the blobs protocol we're running locally:
-    let blobs_client = blobs.client();
-
-    // Grab all passed in arguments, the first one is the binary itself, so we skip it.
-    let args: Vec<String> = std::env::args().skip(1).collect();
-    // Convert to &str, so we can pattern-match easily:
-    let arg_refs: Vec<&str> = args.iter().map(String::as_str).collect();
-
-    match arg_refs.as_slice() {
-        ["send", filename] => {
-            let filename: PathBuf = filename.parse()?;
-            let abs_path = std::path::absolute(&filename)?;
-
-            println!("Hashing file.");
-
-            // keep the file in place and link it, instead of copying it into the in-memory blobs database
-            let in_place = true;
-            let blob = blobs_client
-                .add_from_path(abs_path, in_place, SetTagOption::Auto, WrapOption::NoWrap)
-                .await?
-                .finish()
-                .await?;
-
-            let node_id = router.endpoint().node_id();
-            let ticket = BlobTicket::new(node_id.into(), blob.hash, blob.format)?;
-
-            println!("File hashed. Fetch this file by running:");
-            println!(
-                "cargo run --example transfer -- receive {ticket} {}",
-                filename.display()
-            );
-
-            tokio::signal::ctrl_c().await?;
-        }
-        ["receive", ticket, filename] => {
-            let filename: PathBuf = filename.parse()?;
-            let abs_path = std::path::absolute(filename)?;
-            let ticket: BlobTicket = ticket.parse()?;
-
-            println!("Starting download.");
-
-            blobs_client
-                .download(ticket.hash(), ticket.node_addr().clone())
-                .await?
-                .finish()
-                .await?;
-
-            println!("Finished download.");
-            println!("Copying to destination.");
-
-            blobs_client
-                .export(
-                    ticket.hash(),
-                    abs_path,
-                    ExportFormat::Blob,
-                    ExportMode::Copy,
-                )
-                .await?
-                .finish()
-                .await?;
-
-            println!("Finished copying.");
-        }
-        _ => {
-            println!("Couldn't parse command line arguments: {args:?}");
-            println!("Usage:");
-            println!("    # to send:");
-            println!("    cargo run --example transfer -- send [FILE]");
-            println!("    # this will print a ticket.");
-            println!();
-            println!("    # to receive:");
-            println!("    cargo run --example transfer -- receive [TICKET] [FILE]");
-        }
-    }
-
-    // Gracefully shut down the node
-    println!("Shutting down.");
-    router.shutdown().await?;
-
-    Ok(())
-}
diff --git a/proptest-regressions/protocol/range_spec.txt b/proptest-regressions/protocol/range_spec.txt
index 8558e5a40..ff5fb3fab 100644
--- a/proptest-regressions/protocol/range_spec.txt
+++ b/proptest-regressions/protocol/range_spec.txt
@@ -4,5 +4,6 @@
 #
 # It is recommended to check this file in to source control so that
 # everyone who runs the test benefits from these saved cases.
-cc 7375b003a63bfe725eb4bcb2f266fae6afd9b3c921f9c2018f97daf6ef05a364 # shrinks to ranges = [RangeSet{ChunkNum(0)..ChunkNum(1)}, RangeSet{}]
-cc 23322efa46881646f1468137a688e66aee7ec2a3d01895ccad851d442a7828af # shrinks to ranges = [RangeSet{}, RangeSet{ChunkNum(0)..ChunkNum(1)}]
+cc 5ff4de8531a81c637b4d202c97b724a41a989bc6894464e84db5ac2a519c08a9 # shrinks to ranges = [RangeSet{1..2}]
+cc 50cb338763aa276705bb970d57d3d87e834f31a7e57bba810f46690c6d1e9955 # shrinks to ranges = [RangeSet{7..98}, RangeSet{7..98}]
+cc 8579821a8bde7872fed2cfa38e8a5923706b9915f3920e9c2d101a06bc789309 # shrinks to ranges = []
diff --git a/proptest-regressions/provider.txt b/proptest-regressions/provider.txt
deleted file mode 100644
index 9471db846..000000000
--- a/proptest-regressions/provider.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-# Seeds for failure cases proptest has generated in the past. It is
-# automatically read and these particular cases re-run before any
-# novel cases are generated.
-#
-# It is recommended to check this file in to source control so that
-# everyone who runs the test benefits from these saved cases.
-cc 25ec044e2b84054195984d7e04b93d9b39e2cc25eaee4037dc1be9398f9fd4b4 # shrinks to db = Database(RwLock { data: {}, poisoned: false, .. })
diff --git a/release.toml b/release.toml
deleted file mode 100644
index b4dc18381..000000000
--- a/release.toml
+++ /dev/null
@@ -1 +0,0 @@
-pre-release-hook = ["git", "cliff", "--prepend", "CHANGELOG.md", "--tag", "{{version}}", "--unreleased" ]
diff --git a/src/api.rs b/src/api.rs
new file mode 100644
index 000000000..94f34adea
--- /dev/null
+++ b/src/api.rs
@@ -0,0 +1,322 @@
+//! The user facing API of the store.
+//!
+//! This API is both for interacting with an in-process store and for interacting
+//! with a remote store via rpc calls.
+use std::{io, net::SocketAddr, ops::Deref, sync::Arc};
+
+use iroh::Endpoint;
+use irpc::rpc::{listen, Handler};
+use n0_snafu::SpanTrace;
+use nested_enum_utils::common_fields;
+use proto::{Request, ShutdownRequest, SyncDbRequest};
+use ref_cast::RefCast;
+use serde::{Deserialize, Serialize};
+use snafu::{Backtrace, IntoError, Snafu};
+use tags::Tags;
+
+pub mod blobs;
+pub mod downloader;
+pub mod proto;
+pub mod remote;
+pub mod tags;
+pub use crate::{store::util::Tag, util::temp_tag::TempTag};
+
+pub(crate) type ApiClient = irpc::Client<proto::Command, proto::Request, proto::StoreService>;
+
+#[common_fields({
+    backtrace: Option<Backtrace>,
+    #[snafu(implicit)]
+    span_trace: SpanTrace,
+})]
+#[allow(missing_docs)]
+#[non_exhaustive]
+#[derive(Debug, Snafu)]
+pub enum RequestError {
+    /// Request failed due to rpc error.
+    #[snafu(display("rpc error: {source}"))]
+    Rpc { source: irpc::Error },
+    /// Request failed due an actual error.
+    #[snafu(display("inner error: {source}"))]
+    Inner { source: Error },
+}
+
+impl From<irpc::Error> for RequestError {
+    fn from(value: irpc::Error) -> Self {
+        RpcSnafu.into_error(value)
+    }
+}
+
+impl From<Error> for RequestError {
+    fn from(value: Error) -> Self {
+        InnerSnafu.into_error(value)
+    }
+}
+
+impl From<io::Error> for RequestError {
+    fn from(value: io::Error) -> Self {
+        InnerSnafu.into_error(value.into())
+    }
+}
+
+impl From<irpc::channel::RecvError> for RequestError {
+    fn from(value: irpc::channel::RecvError) -> Self {
+        RpcSnafu.into_error(value.into())
+    }
+}
+
+pub type RequestResult<T> = std::result::Result<T, RequestError>;
+
+#[common_fields({
+    backtrace: Option<Backtrace>,
+    #[snafu(implicit)]
+    span_trace: SpanTrace,
+})]
+#[allow(missing_docs)]
+#[non_exhaustive]
+#[derive(Debug, Snafu)]
+pub enum ExportBaoError {
+    #[snafu(display("send error: {source}"))]
+    Send { source: irpc::channel::SendError },
+    #[snafu(display("recv error: {source}"))]
+    Recv { source: irpc::channel::RecvError },
+    #[snafu(display("request error: {source}"))]
+    Request { source: irpc::RequestError },
+    #[snafu(display("io error: {source}"))]
+    ExportBaoIo { source: io::Error },
+    #[snafu(display("encode error: {source}"))]
+    ExportBaoInner { source: bao_tree::io::EncodeError },
+}
+
+impl From<ExportBaoError> for Error {
+    fn from(e: ExportBaoError) -> Self {
+        match e {
+            ExportBaoError::Send { source, .. } => Self::Io(source.into()),
+            ExportBaoError::Recv { source, .. } => Self::Io(source.into()),
+            ExportBaoError::Request { source, .. } => Self::Io(source.into()),
+            ExportBaoError::ExportBaoIo { source, .. } => Self::Io(source),
+            ExportBaoError::ExportBaoInner { source, .. } => Self::Io(source.into()),
+        }
+    }
+}
+
+impl From<irpc::Error> for ExportBaoError {
+    fn from(e: irpc::Error) -> Self {
+        match e {
+            irpc::Error::Recv(e) => RecvSnafu.into_error(e),
+            irpc::Error::Send(e) => SendSnafu.into_error(e),
+            irpc::Error::Request(e) => RequestSnafu.into_error(e),
+            irpc::Error::Write(e) => ExportBaoIoSnafu.into_error(e.into()),
+        }
+    }
+}
+
+impl From<io::Error> for ExportBaoError {
+    fn from(value: io::Error) -> Self {
+        ExportBaoIoSnafu.into_error(value)
+    }
+}
+
+impl From<irpc::channel::RecvError> for ExportBaoError {
+    fn from(value: irpc::channel::RecvError) -> Self {
+        RecvSnafu.into_error(value)
+    }
+}
+
+impl From<irpc::channel::SendError> for ExportBaoError {
+    fn from(value: irpc::channel::SendError) -> Self {
+        SendSnafu.into_error(value)
+    }
+}
+
+impl From<irpc::RequestError> for ExportBaoError {
+    fn from(value: irpc::RequestError) -> Self {
+        RequestSnafu.into_error(value)
+    }
+}
+
+impl From<bao_tree::io::EncodeError> for ExportBaoError {
+    fn from(value: bao_tree::io::EncodeError) -> Self {
+        ExportBaoInnerSnafu.into_error(value)
+    }
+}
+
+pub type ExportBaoResult<T> = std::result::Result<T, ExportBaoError>;
+
+#[derive(Debug, derive_more::Display, derive_more::From, Serialize, Deserialize)]
+pub enum Error {
+    #[serde(with = "crate::util::serde::io_error_serde")]
+    Io(io::Error),
+}
+
+impl Error {
+    pub fn io(
+        kind: io::ErrorKind,
+        msg: impl Into<Box<dyn std::error::Error + Send + Sync>>,
+    ) -> Self {
+        Self::Io(io::Error::new(kind, msg.into()))
+    }
+
+    pub fn other<E>(msg: E) -> Self
+    where
+        E: Into<Box<dyn std::error::Error + Send + Sync>>,
+    {
+        Self::Io(io::Error::other(msg.into()))
+    }
+}
+
+impl From<irpc::Error> for Error {
+    fn from(e: irpc::Error) -> Self {
+        Self::Io(e.into())
+    }
+}
+
+impl From<RequestError> for Error {
+    fn from(e: RequestError) -> Self {
+        match e {
+            RequestError::Rpc { source, .. } => Self::Io(source.into()),
+            RequestError::Inner { source, .. } => source,
+        }
+    }
+}
+
+impl From<irpc::channel::RecvError> for Error {
+    fn from(e: irpc::channel::RecvError) -> Self {
+        Self::Io(e.into())
+    }
+}
+
+impl From<irpc::rpc::WriteError> for Error {
+    fn from(e: irpc::rpc::WriteError) -> Self {
+        Self::Io(e.into())
+    }
+}
+
+impl From<irpc::RequestError> for Error {
+    fn from(e: irpc::RequestError) -> Self {
+        Self::Io(e.into())
+    }
+}
+
+impl From<irpc::channel::SendError> for Error {
+    fn from(e: irpc::channel::SendError) -> Self {
+        Self::Io(e.into())
+    }
+}
+
+impl std::error::Error for Error {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        match self {
+            Error::Io(e) => Some(e),
+        }
+    }
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+/// The main entry point for the store API.
+#[derive(Debug, Clone, ref_cast::RefCast)]
+#[repr(transparent)]
+pub struct Store {
+    client: ApiClient,
+}
+
+impl Deref for Store {
+    type Target = blobs::Blobs;
+
+    fn deref(&self) -> &Self::Target {
+        blobs::Blobs::ref_from_sender(&self.client)
+    }
+}
+
+impl Store {
+    /// The tags API.
+    pub fn tags(&self) -> &Tags {
+        Tags::ref_from_sender(&self.client)
+    }
+
+    /// The blobs API.
+    pub fn blobs(&self) -> &blobs::Blobs {
+        blobs::Blobs::ref_from_sender(&self.client)
+    }
+
+    /// API for getting blobs from a *single* remote node.
+    pub fn remote(&self) -> &remote::Remote {
+        remote::Remote::ref_from_sender(&self.client)
+    }
+
+    /// Create a downloader for more complex downloads.
+    ///
+    /// Unlike the other APIs, this creates an object that has internal state,
+    /// so don't create it ad hoc but store it somewhere if you need it multiple
+    /// times.
+    pub fn downloader(&self, endpoint: &Endpoint) -> downloader::Downloader {
+        downloader::Downloader::new(self, endpoint)
+    }
+
+    /// Connect to a remote store as a rpc client.
+    pub fn connect(endpoint: quinn::Endpoint, addr: SocketAddr) -> Self {
+        let sender = irpc::Client::quinn(endpoint, addr);
+        Store::from_sender(sender)
+    }
+
+    /// Listen on a quinn endpoint for incoming rpc connections.
+    pub async fn listen(self, endpoint: quinn::Endpoint) {
+        let local = self.client.local().unwrap().clone();
+        let handler: Handler<Request> = Arc::new(move |req, rx, tx| {
+            let local = local.clone();
+            Box::pin({
+                match req {
+                    Request::SetTag(msg) => local.send((msg, tx)),
+                    Request::CreateTag(msg) => local.send((msg, tx)),
+                    Request::DeleteTags(msg) => local.send((msg, tx)),
+                    Request::RenameTag(msg) => local.send((msg, tx)),
+                    Request::ListTags(msg) => local.send((msg, tx)),
+
+                    Request::ListTempTags(msg) => local.send((msg, tx)),
+                    Request::CreateTempTag(msg) => local.send((msg, tx)),
+
+                    Request::BlobStatus(msg) => local.send((msg, tx)),
+
+                    Request::ImportBytes(msg) => local.send((msg, tx)),
+                    Request::ImportByteStream(msg) => local.send((msg, tx, rx)),
+                    Request::ImportBao(msg) => local.send((msg, tx, rx)),
+                    Request::ImportPath(msg) => local.send((msg, tx)),
+                    Request::ListBlobs(msg) => local.send((msg, tx)),
+                    Request::DeleteBlobs(msg) => local.send((msg, tx)),
+                    Request::Batch(msg) => local.send((msg, tx, rx)),
+
+                    Request::ExportBao(msg) => local.send((msg, tx)),
+                    Request::ExportRanges(msg) => local.send((msg, tx)),
+                    Request::ExportPath(msg) => local.send((msg, tx)),
+
+                    Request::Observe(msg) => local.send((msg, tx)),
+
+                    Request::ClearProtected(msg) => local.send((msg, tx)),
+                    Request::SyncDb(msg) => local.send((msg, tx)),
+                    Request::Shutdown(msg) => local.send((msg, tx)),
+                }
+            })
+        });
+        listen::<Request>(endpoint, handler).await
+    }
+
+    pub async fn sync_db(&self) -> RequestResult<()> {
+        let msg = SyncDbRequest;
+        self.client.rpc(msg).await??;
+        Ok(())
+    }
+
+    pub async fn shutdown(&self) -> irpc::Result<()> {
+        let msg = ShutdownRequest;
+        self.client.rpc(msg).await?;
+        Ok(())
+    }
+
+    pub(crate) fn from_sender(client: ApiClient) -> Self {
+        Self { client }
+    }
+
+    pub(crate) fn ref_from_sender(client: &ApiClient) -> &Self {
+        Self::ref_cast(client)
+    }
+}
diff --git a/src/api/blobs.rs b/src/api/blobs.rs
new file mode 100644
index 000000000..942daf0a0
--- /dev/null
+++ b/src/api/blobs.rs
@@ -0,0 +1,1135 @@
+//! API to interact with a local blob store
+//!
+//! This API is for local interactions with the blob store, such as importing
+//! and exporting blobs, observing the bitfield of a blob, and deleting blobs.
+//!
+//! The main entry point is the [`Blobs`] struct.
+use std::{
+    collections::BTreeMap,
+    future::{Future, IntoFuture},
+    io,
+    num::NonZeroU64,
+    path::{Path, PathBuf},
+    pin::Pin,
+};
+
+pub use bao_tree::io::mixed::EncodedItem;
+use bao_tree::{
+    io::{
+        fsm::{ResponseDecoder, ResponseDecoderNext},
+        BaoContentItem, Leaf,
+    },
+    BaoTree, ChunkNum, ChunkRanges,
+};
+use bytes::Bytes;
+use genawaiter::sync::Gen;
+use iroh_io::{AsyncStreamReader, TokioStreamReader};
+use irpc::channel::{mpsc, oneshot};
+use n0_future::{future, stream, Stream, StreamExt};
+use quinn::SendStream;
+use range_collections::{range_set::RangeSetRange, RangeSet2};
+use ref_cast::RefCast;
+use tokio::io::AsyncWriteExt;
+use tracing::trace;
+
+// Public reexports from the proto module.
+//
+// Due to the fact that the proto module is hidden from docs by default,
+// these will appear in the docs as if they were declared here.
+pub use super::proto::{
+    AddProgressItem, Bitfield, BlobDeleteRequest as DeleteOptions, BlobStatus,
+    ExportBaoRequest as ExportBaoOptions, ExportMode, ExportPathRequest as ExportOptions,
+    ExportProgressItem, ExportRangesRequest as ExportRangesOptions,
+    ImportBaoRequest as ImportBaoOptions, ImportMode, ObserveRequest as ObserveOptions,
+};
+use super::{
+    proto::{
+        BatchResponse, BlobStatusRequest, ClearProtectedRequest, CreateTempTagRequest,
+        ExportBaoRequest, ExportRangesItem, ImportBaoRequest, ImportByteStreamRequest,
+        ImportBytesRequest, ImportPathRequest, ListRequest, Scope,
+    },
+    remote::HashSeqChunk,
+    tags::TagInfo,
+    ApiClient, RequestResult, Tags,
+};
+use crate::{
+    api::proto::{BatchRequest, ImportByteStreamUpdate},
+    provider::StreamContext,
+    store::IROH_BLOCK_SIZE,
+    util::temp_tag::TempTag,
+    BlobFormat, Hash, HashAndFormat,
+};
+
+/// Options for adding bytes.
+#[derive(Debug)]
+pub struct AddBytesOptions {
+    pub data: Bytes,
+    pub format: BlobFormat,
+}
+
+impl<T: Into<Bytes>> From<(T, BlobFormat)> for AddBytesOptions {
+    fn from(item: (T, BlobFormat)) -> Self {
+        let (data, format) = item;
+        Self {
+            data: data.into(),
+            format,
+        }
+    }
+}
+
+/// Blobs API
+#[derive(Debug, Clone, ref_cast::RefCast)]
+#[repr(transparent)]
+pub struct Blobs {
+    client: ApiClient,
+}
+
+impl Blobs {
+    pub(crate) fn ref_from_sender(sender: &ApiClient) -> &Self {
+        Self::ref_cast(sender)
+    }
+
+    pub async fn batch(&self) -> irpc::Result<Batch<'_>> {
+        let msg = BatchRequest;
+        trace!("{msg:?}");
+        let (tx, rx) = self.client.client_streaming(msg, 32).await?;
+        let scope = rx.await?;
+
+        Ok(Batch {
+            scope,
+            blobs: self,
+            _tx: tx,
+        })
+    }
+
+    pub async fn delete_with_opts(&self, options: DeleteOptions) -> RequestResult<()> {
+        trace!("{options:?}");
+        self.client.rpc(options).await??;
+        Ok(())
+    }
+
+    pub async fn delete(
+        &self,
+        hashes: impl IntoIterator<Item = impl Into<Hash>>,
+    ) -> RequestResult<()> {
+        self.delete_with_opts(DeleteOptions {
+            hashes: hashes.into_iter().map(Into::into).collect(),
+            force: false,
+        })
+        .await
+    }
+
+    pub fn add_slice(&self, data: impl AsRef<[u8]>) -> AddProgress {
+        let options = ImportBytesRequest {
+            data: Bytes::copy_from_slice(data.as_ref()),
+            format: crate::BlobFormat::Raw,
+            scope: Scope::GLOBAL,
+        };
+        self.add_bytes_impl(options)
+    }
+
+    pub fn add_bytes(&self, data: impl Into<bytes::Bytes>) -> AddProgress {
+        let options = ImportBytesRequest {
+            data: data.into(),
+            format: crate::BlobFormat::Raw,
+            scope: Scope::GLOBAL,
+        };
+        self.add_bytes_impl(options)
+    }
+
+    pub fn add_bytes_with_opts(&self, options: impl Into<AddBytesOptions>) -> AddProgress {
+        let options = options.into();
+        let request = ImportBytesRequest {
+            data: options.data,
+            format: options.format,
+            scope: Scope::GLOBAL,
+        };
+        self.add_bytes_impl(request)
+    }
+
+    fn add_bytes_impl(&self, options: ImportBytesRequest) -> AddProgress {
+        trace!("{options:?}");
+        let this = self.clone();
+        let stream = Gen::new(|co| async move {
+            let mut receiver = match this.client.server_streaming(options, 32).await {
+                Ok(receiver) => receiver,
+                Err(cause) => {
+                    co.yield_(AddProgressItem::Error(cause.into())).await;
+                    return;
+                }
+            };
+            loop {
+                match receiver.recv().await {
+                    Ok(Some(item)) => co.yield_(item).await,
+                    Err(cause) => {
+                        co.yield_(AddProgressItem::Error(cause.into())).await;
+                        break;
+                    }
+                    Ok(None) => break,
+                }
+            }
+        });
+        AddProgress::new(self, stream)
+    }
+
+    pub fn add_path_with_opts(&self, options: impl Into<AddPathOptions>) -> AddProgress {
+        let options = options.into();
+        self.add_path_with_opts_impl(ImportPathRequest {
+            path: options.path,
+            mode: options.mode,
+            format: options.format,
+            scope: Scope::GLOBAL,
+        })
+    }
+
+    fn add_path_with_opts_impl(&self, options: ImportPathRequest) -> AddProgress {
+        trace!("{:?}", options);
+        let client = self.client.clone();
+        let stream = Gen::new(|co| async move {
+            let mut receiver = match client.server_streaming(options, 32).await {
+                Ok(receiver) => receiver,
+                Err(cause) => {
+                    co.yield_(AddProgressItem::Error(cause.into())).await;
+                    return;
+                }
+            };
+            loop {
+                match receiver.recv().await {
+                    Ok(Some(item)) => co.yield_(item).await,
+                    Err(cause) => {
+                        co.yield_(AddProgressItem::Error(cause.into())).await;
+                        break;
+                    }
+                    Ok(None) => break,
+                }
+            }
+        });
+        AddProgress::new(self, stream)
+    }
+
+    pub fn add_path(&self, path: impl AsRef<Path>) -> AddProgress {
+        self.add_path_with_opts(AddPathOptions {
+            path: path.as_ref().to_owned(),
+            mode: ImportMode::Copy,
+            format: BlobFormat::Raw,
+        })
+    }
+
+    pub async fn add_stream(
+        &self,
+        data: impl Stream<Item = io::Result<Bytes>> + Send + Sync + 'static,
+    ) -> AddProgress {
+        let inner = ImportByteStreamRequest {
+            format: crate::BlobFormat::Raw,
+            scope: Scope::default(),
+        };
+        let client = self.client.clone();
+        let stream = Gen::new(|co| async move {
+            let (sender, mut receiver) = match client.bidi_streaming(inner, 32, 32).await {
+                Ok(x) => x,
+                Err(cause) => {
+                    co.yield_(AddProgressItem::Error(cause.into())).await;
+                    return;
+                }
+            };
+            let recv = async {
+                loop {
+                    match receiver.recv().await {
+                        Ok(Some(item)) => co.yield_(item).await,
+                        Err(cause) => {
+                            co.yield_(AddProgressItem::Error(cause.into())).await;
+                            break;
+                        }
+                        Ok(None) => break,
+                    }
+                }
+            };
+            let send = async {
+                tokio::pin!(data);
+                while let Some(item) = data.next().await {
+                    sender.send(ImportByteStreamUpdate::Bytes(item?)).await?;
+                }
+                sender.send(ImportByteStreamUpdate::Done).await?;
+                anyhow::Ok(())
+            };
+            let _ = tokio::join!(send, recv);
+        });
+        AddProgress::new(self, stream)
+    }
+
+    pub fn export_ranges(
+        &self,
+        hash: impl Into<Hash>,
+        ranges: impl Into<RangeSet2<u64>>,
+    ) -> ExportRangesProgress {
+        self.export_ranges_with_opts(ExportRangesOptions {
+            hash: hash.into(),
+            ranges: ranges.into(),
+        })
+    }
+
+    pub fn export_ranges_with_opts(&self, options: ExportRangesOptions) -> ExportRangesProgress {
+        trace!("{options:?}");
+        ExportRangesProgress::new(
+            options.ranges.clone(),
+            self.client.server_streaming(options, 32),
+        )
+    }
+
+    pub fn export_bao_with_opts(
+        &self,
+        options: ExportBaoOptions,
+        local_update_cap: usize,
+    ) -> ExportBaoProgress {
+        trace!("{options:?}");
+        ExportBaoProgress::new(self.client.server_streaming(options, local_update_cap))
+    }
+
+    pub fn export_bao(
+        &self,
+        hash: impl Into<Hash>,
+        ranges: impl Into<ChunkRanges>,
+    ) -> ExportBaoProgress {
+        self.export_bao_with_opts(
+            ExportBaoRequest {
+                hash: hash.into(),
+                ranges: ranges.into(),
+            },
+            32,
+        )
+    }
+
+    /// Export a single chunk from the given hash, at the given offset.
+    pub async fn export_chunk(
+        &self,
+        hash: impl Into<Hash>,
+        offset: u64,
+    ) -> super::ExportBaoResult<Leaf> {
+        let base = ChunkNum::full_chunks(offset);
+        let ranges = ChunkRanges::from(base..base + 1);
+        let mut stream = self.export_bao(hash, ranges).stream();
+        while let Some(item) = stream.next().await {
+            match item {
+                EncodedItem::Leaf(leaf) => return Ok(leaf),
+                EncodedItem::Parent(_) => {}
+                EncodedItem::Size(_) => {}
+                EncodedItem::Done => break,
+                EncodedItem::Error(cause) => return Err(cause.into()),
+            }
+        }
+        Err(io::Error::other("unexpected end of stream").into())
+    }
+
+    /// Get the entire blob into a Bytes
+    ///
+    /// This will run out of memory when called for very large blobs, so be careful!
+    pub async fn get_bytes(&self, hash: impl Into<Hash>) -> super::ExportBaoResult<Bytes> {
+        self.export_bao(hash.into(), ChunkRanges::all())
+            .data_to_bytes()
+            .await
+    }
+
+    /// Observe the bitfield of the given hash.
+    pub fn observe(&self, hash: impl Into<Hash>) -> ObserveProgress {
+        self.observe_with_opts(ObserveOptions { hash: hash.into() })
+    }
+
+    pub fn observe_with_opts(&self, options: ObserveOptions) -> ObserveProgress {
+        trace!("{:?}", options);
+        if options.hash == Hash::EMPTY {
+            return ObserveProgress::new(async move {
+                let (tx, rx) = mpsc::channel(1);
+                tx.send(Bitfield::complete(0)).await.ok();
+                Ok(rx)
+            });
+        }
+        ObserveProgress::new(self.client.server_streaming(options, 32))
+    }
+
+    pub fn export_with_opts(&self, options: ExportOptions) -> ExportProgress {
+        trace!("{:?}", options);
+        ExportProgress::new(self.client.server_streaming(options, 32))
+    }
+
+    pub fn export(&self, hash: impl Into<Hash>, target: impl AsRef<Path>) -> ExportProgress {
+        let options = ExportOptions {
+            hash: hash.into(),
+            mode: ExportMode::Copy,
+            target: target.as_ref().to_owned(),
+        };
+        self.export_with_opts(options)
+    }
+
+    /// Import BaoContentItems from a stream.
+    ///
+    /// The store assumes that these are already verified and in the correct order.
+    #[cfg_attr(feature = "hide-proto-docs", doc(hidden))]
+    pub async fn import_bao(
+        &self,
+        hash: impl Into<Hash>,
+        size: NonZeroU64,
+        local_update_cap: usize,
+    ) -> irpc::Result<ImportBaoHandle> {
+        let options = ImportBaoRequest {
+            hash: hash.into(),
+            size,
+        };
+        self.import_bao_with_opts(options, local_update_cap).await
+    }
+
+    #[cfg_attr(feature = "hide-proto-docs", doc(hidden))]
+    pub async fn import_bao_with_opts(
+        &self,
+        options: ImportBaoOptions,
+        local_update_cap: usize,
+    ) -> irpc::Result<ImportBaoHandle> {
+        trace!("{:?}", options);
+        ImportBaoHandle::new(self.client.client_streaming(options, local_update_cap)).await
+    }
+
+    #[cfg_attr(feature = "hide-proto-docs", doc(hidden))]
+    async fn import_bao_reader<R: AsyncStreamReader>(
+        &self,
+        hash: Hash,
+        ranges: ChunkRanges,
+        mut reader: R,
+    ) -> RequestResult<R> {
+        let size = u64::from_le_bytes(reader.read::<8>().await.map_err(super::Error::other)?);
+        let Some(size) = NonZeroU64::new(size) else {
+            return if hash == Hash::EMPTY {
+                Ok(reader)
+            } else {
+                Err(super::Error::other("invalid size for hash").into())
+            };
+        };
+        let tree = BaoTree::new(size.get(), IROH_BLOCK_SIZE);
+        let mut decoder = ResponseDecoder::new(hash.into(), ranges, tree, reader);
+        let options = ImportBaoOptions { hash, size };
+        let handle = self.import_bao_with_opts(options, 32).await?;
+        let driver = async move {
+            let reader = loop {
+                match decoder.next().await {
+                    ResponseDecoderNext::More((rest, item)) => {
+                        handle.tx.send(item?).await?;
+                        decoder = rest;
+                    }
+                    ResponseDecoderNext::Done(reader) => break reader,
+                };
+            };
+            drop(handle.tx);
+            io::Result::Ok(reader)
+        };
+        let fut = async move { handle.rx.await.map_err(io::Error::other)? };
+        let (reader, res) = tokio::join!(driver, fut);
+        res?;
+        Ok(reader?)
+    }
+
+    #[cfg_attr(feature = "hide-proto-docs", doc(hidden))]
+    pub async fn import_bao_quinn(
+        &self,
+        hash: Hash,
+        ranges: ChunkRanges,
+        stream: &mut iroh::endpoint::RecvStream,
+    ) -> RequestResult<()> {
+        let reader = TokioStreamReader::new(stream);
+        self.import_bao_reader(hash, ranges, reader).await?;
+        Ok(())
+    }
+
+    #[cfg_attr(feature = "hide-proto-docs", doc(hidden))]
+    pub async fn import_bao_bytes(
+        &self,
+        hash: Hash,
+        ranges: ChunkRanges,
+        data: impl Into<Bytes>,
+    ) -> RequestResult<()> {
+        self.import_bao_reader(hash, ranges, data.into()).await?;
+        Ok(())
+    }
+
+    pub fn list(&self) -> BlobsListProgress {
+        let msg = ListRequest;
+        let client = self.client.clone();
+        BlobsListProgress::new(client.server_streaming(msg, 32))
+    }
+
+    pub async fn status(&self, hash: impl Into<Hash>) -> irpc::Result<BlobStatus> {
+        let hash = hash.into();
+        let msg = BlobStatusRequest { hash };
+        self.client.rpc(msg).await
+    }
+
+    pub async fn has(&self, hash: impl Into<Hash>) -> irpc::Result<bool> {
+        match self.status(hash).await? {
+            BlobStatus::Complete { .. } => Ok(true),
+            _ => Ok(false),
+        }
+    }
+
+    pub(crate) async fn clear_protected(&self) -> RequestResult<()> {
+        let msg = ClearProtectedRequest;
+        self.client.rpc(msg).await??;
+        Ok(())
+    }
+}
+
+/// A progress handle for a batch scoped add operation.
+pub struct BatchAddProgress<'a>(AddProgress<'a>);
+
+impl<'a> IntoFuture for BatchAddProgress<'a> {
+    type Output = RequestResult<TempTag>;
+
+    type IntoFuture = Pin<Box<dyn Future<Output = Self::Output> + Send + 'a>>;
+
+    fn into_future(self) -> Self::IntoFuture {
+        Box::pin(self.temp_tag())
+    }
+}
+
+impl<'a> BatchAddProgress<'a> {
+    pub async fn with_named_tag(self, name: impl AsRef<[u8]>) -> RequestResult<HashAndFormat> {
+        self.0.with_named_tag(name).await
+    }
+
+    pub async fn with_tag(self) -> RequestResult<TagInfo> {
+        self.0.with_tag().await
+    }
+
+    pub async fn stream(self) -> impl Stream<Item = AddProgressItem> {
+        self.0.stream().await
+    }
+
+    pub async fn temp_tag(self) -> RequestResult<TempTag> {
+        self.0.temp_tag().await
+    }
+}
+
+/// A batch of operations that modify the blob store.
+pub struct Batch<'a> {
+    scope: Scope,
+    blobs: &'a Blobs,
+    _tx: mpsc::Sender<BatchResponse>,
+}
+
+impl<'a> Batch<'a> {
+    pub fn add_bytes(&self, data: impl Into<Bytes>) -> BatchAddProgress {
+        let options = ImportBytesRequest {
+            data: data.into(),
+            format: crate::BlobFormat::Raw,
+            scope: self.scope,
+        };
+        BatchAddProgress(self.blobs.add_bytes_impl(options))
+    }
+
+    pub fn add_bytes_with_opts(&self, options: impl Into<AddBytesOptions>) -> BatchAddProgress {
+        let options = options.into();
+        BatchAddProgress(self.blobs.add_bytes_impl(ImportBytesRequest {
+            data: options.data,
+            format: options.format,
+            scope: self.scope,
+        }))
+    }
+
+    pub fn add_slice(&self, data: impl AsRef<[u8]>) -> BatchAddProgress {
+        let options = ImportBytesRequest {
+            data: Bytes::copy_from_slice(data.as_ref()),
+            format: crate::BlobFormat::Raw,
+            scope: self.scope,
+        };
+        BatchAddProgress(self.blobs.add_bytes_impl(options))
+    }
+
+    pub fn add_path_with_opts(&self, options: impl Into<AddPathOptions>) -> BatchAddProgress {
+        let options = options.into();
+        BatchAddProgress(self.blobs.add_path_with_opts_impl(ImportPathRequest {
+            path: options.path,
+            mode: options.mode,
+            format: options.format,
+            scope: self.scope,
+        }))
+    }
+
+    pub async fn temp_tag(&self, value: impl Into<HashAndFormat>) -> irpc::Result<TempTag> {
+        let value = value.into();
+        let msg = CreateTempTagRequest {
+            scope: self.scope,
+            value,
+        };
+        self.blobs.client.rpc(msg).await
+    }
+}
+
+/// Options for adding data from a file system path.
+#[derive(Debug)]
+pub struct AddPathOptions {
+    pub path: PathBuf,
+    pub format: BlobFormat,
+    pub mode: ImportMode,
+}
+
+/// A progress handle for an import operation.
+///
+/// Internally this is a stream of [`AddProgressItem`] items. Working with this
+/// stream directly can be inconvenient, so this struct provides some convenience
+/// methods to work with the result.
+///
+/// It also implements [`IntoFuture`], so you can await it to get the [`TempTag`] that
+/// contains the hash of the added content and also protects the content.
+///
+/// If you want access to the stream, you can use the [`AddProgress::stream`] method.
+pub struct AddProgress<'a> {
+    blobs: &'a Blobs,
+    inner: stream::Boxed<AddProgressItem>,
+}
+
+impl<'a> IntoFuture for AddProgress<'a> {
+    type Output = RequestResult<TagInfo>;
+
+    type IntoFuture = Pin<Box<dyn Future<Output = Self::Output> + Send + 'a>>;
+
+    fn into_future(self) -> Self::IntoFuture {
+        Box::pin(self.with_tag())
+    }
+}
+
+impl<'a> AddProgress<'a> {
+    fn new(blobs: &'a Blobs, stream: impl Stream<Item = AddProgressItem> + Send + 'static) -> Self {
+        Self {
+            blobs,
+            inner: Box::pin(stream),
+        }
+    }
+
+    pub async fn temp_tag(self) -> RequestResult<TempTag> {
+        let mut stream = self.inner;
+        while let Some(item) = stream.next().await {
+            match item {
+                AddProgressItem::Done(tt) => return Ok(tt),
+                AddProgressItem::Error(e) => return Err(e.into()),
+                _ => {}
+            }
+        }
+        Err(super::Error::other("unexpected end of stream").into())
+    }
+
+    pub async fn with_named_tag(self, name: impl AsRef<[u8]>) -> RequestResult<HashAndFormat> {
+        let blobs = self.blobs.clone();
+        let tt = self.temp_tag().await?;
+        let haf = *tt.hash_and_format();
+        let tags = Tags::ref_from_sender(&blobs.client);
+        tags.set(name, *tt.hash_and_format()).await?;
+        drop(tt);
+        Ok(haf)
+    }
+
+    pub async fn with_tag(self) -> RequestResult<TagInfo> {
+        let blobs = self.blobs.clone();
+        let tt = self.temp_tag().await?;
+        let hash = *tt.hash();
+        let format = tt.format();
+        let tags = Tags::ref_from_sender(&blobs.client);
+        let name = tags.create(*tt.hash_and_format()).await?;
+        drop(tt);
+        Ok(TagInfo { name, hash, format })
+    }
+
+    pub async fn stream(self) -> impl Stream<Item = AddProgressItem> {
+        self.inner
+    }
+}
+
+/// An observe result. Awaiting this will return the current state.
+///
+/// Calling [`ObserveProgress::stream`] will return a stream of updates, where
+/// the first item is the current state and subsequent items are updates.
+pub struct ObserveProgress {
+    inner: future::Boxed<irpc::Result<mpsc::Receiver<Bitfield>>>,
+}
+
+impl IntoFuture for ObserveProgress {
+    type Output = RequestResult<Bitfield>;
+
+    type IntoFuture = Pin<Box<dyn Future<Output = Self::Output> + Send>>;
+
+    fn into_future(self) -> Self::IntoFuture {
+        Box::pin(async move {
+            let mut rx = self.inner.await?;
+            match rx.recv().await? {
+                Some(bitfield) => Ok(bitfield),
+                None => Err(super::Error::other("unexpected end of stream").into()),
+            }
+        })
+    }
+}
+
+impl ObserveProgress {
+    fn new(
+        fut: impl Future<Output = irpc::Result<mpsc::Receiver<Bitfield>>> + Send + 'static,
+    ) -> Self {
+        Self {
+            inner: Box::pin(fut),
+        }
+    }
+
+    pub async fn await_completion(self) -> RequestResult<Bitfield> {
+        let mut stream = self.stream().await?;
+        while let Some(item) = stream.next().await {
+            if item.is_complete() {
+                return Ok(item);
+            }
+        }
+        Err(super::Error::other("unexpected end of stream").into())
+    }
+
+    /// Returns an infinite stream of bitfields. The first bitfield is the
+    /// current state, and the following bitfields are updates.
+    ///
+    /// Once a blob is complete, there will be no more updates.
+    pub async fn stream(self) -> irpc::Result<impl Stream<Item = Bitfield>> {
+        let mut rx = self.inner.await?;
+        Ok(Gen::new(|co| async move {
+            while let Ok(Some(item)) = rx.recv().await {
+                co.yield_(item).await;
+            }
+        }))
+    }
+}
+
+/// A progress handle for an export operation.
+///
+/// Internally this is a stream of [`ExportProgress`] items. Working with this
+/// stream directly can be inconvenient, so this struct provides some convenience
+/// methods to work with the result.
+///
+/// To get the underlying stream, use the [`ExportProgress::stream`] method.
+///
+/// It also implements [`IntoFuture`], so you can await it to get the size of the
+/// exported blob.
+pub struct ExportProgress {
+    inner: future::Boxed<irpc::Result<mpsc::Receiver<ExportProgressItem>>>,
+}
+
+impl IntoFuture for ExportProgress {
+    type Output = RequestResult<u64>;
+
+    type IntoFuture = Pin<Box<dyn Future<Output = Self::Output> + Send>>;
+
+    fn into_future(self) -> Self::IntoFuture {
+        Box::pin(self.finish())
+    }
+}
+
+impl ExportProgress {
+    fn new(
+        fut: impl Future<Output = irpc::Result<mpsc::Receiver<ExportProgressItem>>> + Send + 'static,
+    ) -> Self {
+        Self {
+            inner: Box::pin(fut),
+        }
+    }
+
+    pub async fn stream(self) -> impl Stream<Item = ExportProgressItem> {
+        Gen::new(|co| async move {
+            let mut rx = match self.inner.await {
+                Ok(rx) => rx,
+                Err(e) => {
+                    co.yield_(ExportProgressItem::Error(e.into())).await;
+                    return;
+                }
+            };
+            while let Ok(Some(item)) = rx.recv().await {
+                co.yield_(item).await;
+            }
+        })
+    }
+
+    pub async fn finish(self) -> RequestResult<u64> {
+        let mut rx = self.inner.await?;
+        let mut size = None;
+        loop {
+            match rx.recv().await? {
+                Some(ExportProgressItem::Done) => break,
+                Some(ExportProgressItem::Size(s)) => size = Some(s),
+                Some(ExportProgressItem::Error(cause)) => return Err(cause.into()),
+                _ => {}
+            }
+        }
+        if let Some(size) = size {
+            Ok(size)
+        } else {
+            Err(super::Error::other("unexpected end of stream").into())
+        }
+    }
+}
+
+/// A handle for an ongoing bao import operation.
+pub struct ImportBaoHandle {
+    pub tx: mpsc::Sender<BaoContentItem>,
+    pub rx: oneshot::Receiver<super::Result<()>>,
+}
+
+impl ImportBaoHandle {
+    pub(crate) async fn new(
+        fut: impl Future<
+                Output = irpc::Result<(
+                    mpsc::Sender<BaoContentItem>,
+                    oneshot::Receiver<super::Result<()>>,
+                )>,
+            > + Send
+            + 'static,
+    ) -> irpc::Result<Self> {
+        let (tx, rx) = fut.await?;
+        Ok(Self { tx, rx })
+    }
+}
+
+/// A progress handle for a blobs list operation.
+pub struct BlobsListProgress {
+    inner: future::Boxed<irpc::Result<mpsc::Receiver<super::Result<Hash>>>>,
+}
+
+impl BlobsListProgress {
+    fn new(
+        fut: impl Future<Output = irpc::Result<mpsc::Receiver<super::Result<Hash>>>> + Send + 'static,
+    ) -> Self {
+        Self {
+            inner: Box::pin(fut),
+        }
+    }
+
+    pub async fn hashes(self) -> RequestResult<Vec<Hash>> {
+        let mut rx: mpsc::Receiver<Result<Hash, super::Error>> = self.inner.await?;
+        let mut hashes = Vec::new();
+        while let Some(item) = rx.recv().await? {
+            hashes.push(item?);
+        }
+        Ok(hashes)
+    }
+
+    pub async fn stream(self) -> irpc::Result<impl Stream<Item = super::Result<Hash>>> {
+        let mut rx = self.inner.await?;
+        Ok(Gen::new(|co| async move {
+            while let Ok(Some(item)) = rx.recv().await {
+                co.yield_(item).await;
+            }
+        }))
+    }
+}
+
+/// A progress handle for a bao export operation.
+///
+/// Internally, this is a stream of [`EncodedItem`]s. Using this stream directly
+/// is often inconvenient, so there are a number of higher level methods to
+/// process the stream.
+///
+/// You can get access to the underlying stream using the [`ExportBaoProgress::stream`] method.
+pub struct ExportRangesProgress {
+    ranges: RangeSet2<u64>,
+    inner: future::Boxed<irpc::Result<mpsc::Receiver<ExportRangesItem>>>,
+}
+
+impl ExportRangesProgress {
+    fn new(
+        ranges: RangeSet2<u64>,
+        fut: impl Future<Output = irpc::Result<mpsc::Receiver<ExportRangesItem>>> + Send + 'static,
+    ) -> Self {
+        Self {
+            ranges,
+            inner: Box::pin(fut),
+        }
+    }
+}
+
+impl ExportRangesProgress {
+    /// A raw stream of [`ExportRangesItem`]s.
+    ///
+    /// Ranges will be rounded up to chunk boundaries. So if you request a
+    /// range of 0..100, you will get the entire first chunk, 0..1024.
+    ///
+    /// It is up to the caller to clip the ranges to the requested ranges.
+    pub async fn stream(self) -> impl Stream<Item = ExportRangesItem> {
+        Gen::new(|co| async move {
+            let mut rx = match self.inner.await {
+                Ok(rx) => rx,
+                Err(e) => {
+                    co.yield_(ExportRangesItem::Error(e.into())).await;
+                    return;
+                }
+            };
+            while let Ok(Some(item)) = rx.recv().await {
+                co.yield_(item).await;
+            }
+        })
+    }
+
+    /// Concatenate all the data into a single `Bytes`.
+    pub async fn concatenate(self) -> RequestResult<Vec<u8>> {
+        let mut rx = self.inner.await?;
+        let mut data = BTreeMap::new();
+        while let Some(item) = rx.recv().await? {
+            match item {
+                ExportRangesItem::Size(_) => {}
+                ExportRangesItem::Data(leaf) => {
+                    data.insert(leaf.offset, leaf.data);
+                }
+                ExportRangesItem::Error(cause) => return Err(cause.into()),
+            }
+        }
+        let mut res = Vec::new();
+        for range in self.ranges.iter() {
+            let (start, end) = match range {
+                RangeSetRange::RangeFrom(range) => (*range.start, u64::MAX),
+                RangeSetRange::Range(range) => (*range.start, *range.end),
+            };
+            for (offset, data) in data.iter() {
+                let cstart = *offset;
+                let cend = *offset + (data.len() as u64);
+                if cstart >= end || cend <= start {
+                    continue;
+                }
+                let start = start.max(cstart);
+                let end = end.min(cend);
+                let data = &data[(start - cstart) as usize..(end - cstart) as usize];
+                res.extend_from_slice(data);
+            }
+        }
+        Ok(res)
+    }
+}
+
+/// A progress handle for a bao export operation.
+///
+/// Internally, this is a stream of [`EncodedItem`]s. Using this stream directly
+/// is often inconvenient, so there are a number of higher level methods to
+/// process the stream.
+///
+/// You can get access to the underlying stream using the [`ExportBaoProgress::stream`] method.
+pub struct ExportBaoProgress {
+    inner: future::Boxed<irpc::Result<mpsc::Receiver<EncodedItem>>>,
+}
+
+impl ExportBaoProgress {
+    fn new(
+        fut: impl Future<Output = irpc::Result<mpsc::Receiver<EncodedItem>>> + Send + 'static,
+    ) -> Self {
+        Self {
+            inner: Box::pin(fut),
+        }
+    }
+
+    /// Interprets this blob as a hash sequence and returns a stream of hashes.
+    ///
+    /// Errors will be reported, but the iterator will nevertheless continue.
+    /// If you get an error despite having asked for ranges that should be present,
+    /// this means that the data is corrupted. It can still make sense to continue
+    /// to get all non-corrupted sections.
+    pub fn hashes_with_index(
+        self,
+    ) -> impl Stream<Item = std::result::Result<(u64, Hash), anyhow::Error>> {
+        let mut stream = self.stream();
+        Gen::new(|co| async move {
+            while let Some(item) = stream.next().await {
+                let leaf = match item {
+                    EncodedItem::Leaf(leaf) => leaf,
+                    EncodedItem::Error(e) => {
+                        co.yield_(Err(e.into())).await;
+                        continue;
+                    }
+                    _ => continue,
+                };
+                let slice = match HashSeqChunk::try_from(leaf) {
+                    Ok(slice) => slice,
+                    Err(e) => {
+                        co.yield_(Err(e)).await;
+                        continue;
+                    }
+                };
+                let offset = slice.base();
+                for (o, hash) in slice.into_iter().enumerate() {
+                    co.yield_(Ok((offset + o as u64, hash))).await;
+                }
+            }
+        })
+    }
+
+    /// Same as [`Self::hashes_with_index`], but without the indexes.
+    pub fn hashes(self) -> impl Stream<Item = std::result::Result<Hash, anyhow::Error>> {
+        self.hashes_with_index().map(|x| x.map(|(_, hash)| hash))
+    }
+
+    pub async fn bao_to_vec(self) -> RequestResult<Vec<u8>> {
+        let mut data = Vec::new();
+        let mut stream = self.into_byte_stream();
+        while let Some(item) = stream.next().await {
+            println!("item: {item:?}");
+            data.extend_from_slice(&item?);
+        }
+        Ok(data)
+    }
+
+    pub async fn data_to_bytes(self) -> super::ExportBaoResult<Bytes> {
+        let mut rx = self.inner.await?;
+        let mut data = Vec::new();
+        while let Some(item) = rx.recv().await? {
+            match item {
+                EncodedItem::Leaf(leaf) => {
+                    data.push(leaf.data);
+                }
+                EncodedItem::Parent(_) => {}
+                EncodedItem::Size(_) => {}
+                EncodedItem::Done => break,
+                EncodedItem::Error(cause) => return Err(cause.into()),
+            }
+        }
+        if data.len() == 1 {
+            Ok(data.pop().unwrap())
+        } else {
+            let mut out = Vec::new();
+            for item in data {
+                out.extend_from_slice(&item);
+            }
+            Ok(out.into())
+        }
+    }
+
+    pub async fn data_to_vec(self) -> super::ExportBaoResult<Vec<u8>> {
+        let mut rx = self.inner.await?;
+        let mut data = Vec::new();
+        while let Some(item) = rx.recv().await? {
+            match item {
+                EncodedItem::Leaf(leaf) => {
+                    data.extend_from_slice(&leaf.data);
+                }
+                EncodedItem::Parent(_) => {}
+                EncodedItem::Size(_) => {}
+                EncodedItem::Done => break,
+                EncodedItem::Error(cause) => return Err(cause.into()),
+            }
+        }
+        Ok(data)
+    }
+
+    pub async fn write_quinn(self, target: &mut quinn::SendStream) -> super::ExportBaoResult<()> {
+        let mut rx = self.inner.await?;
+        while let Some(item) = rx.recv().await? {
+            match item {
+                EncodedItem::Size(size) => {
+                    target.write_u64_le(size).await?;
+                }
+                EncodedItem::Parent(parent) => {
+                    let mut data = vec![0u8; 64];
+                    data[..32].copy_from_slice(parent.pair.0.as_bytes());
+                    data[32..].copy_from_slice(parent.pair.1.as_bytes());
+                    target.write_all(&data).await.map_err(io::Error::from)?;
+                }
+                EncodedItem::Leaf(leaf) => {
+                    target
+                        .write_chunk(leaf.data)
+                        .await
+                        .map_err(io::Error::from)?;
+                }
+                EncodedItem::Done => break,
+                EncodedItem::Error(cause) => return Err(cause.into()),
+            }
+        }
+        Ok(())
+    }
+
+    /// Write quinn variant that also feeds a progress writer.
+    pub(crate) async fn write_quinn_with_progress(
+        self,
+        writer: &mut SendStream,
+        progress: &mut impl WriteProgress,
+        hash: &Hash,
+        index: u64,
+    ) -> super::ExportBaoResult<()> {
+        let mut rx = self.inner.await?;
+        while let Some(item) = rx.recv().await? {
+            match item {
+                EncodedItem::Size(size) => {
+                    progress.send_transfer_started(index, hash, size).await;
+                    writer.write_u64_le(size).await?;
+                    progress.log_other_write(8);
+                }
+                EncodedItem::Parent(parent) => {
+                    let mut data = vec![0u8; 64];
+                    data[..32].copy_from_slice(parent.pair.0.as_bytes());
+                    data[32..].copy_from_slice(parent.pair.1.as_bytes());
+                    writer.write_all(&data).await.map_err(io::Error::from)?;
+                    progress.log_other_write(64);
+                }
+                EncodedItem::Leaf(leaf) => {
+                    let len = leaf.data.len();
+                    writer
+                        .write_chunk(leaf.data)
+                        .await
+                        .map_err(io::Error::from)?;
+                    progress.notify_payload_write(index, leaf.offset, len).await;
+                }
+                EncodedItem::Done => break,
+                EncodedItem::Error(cause) => return Err(cause.into()),
+            }
+        }
+        Ok(())
+    }
+
+    pub fn into_byte_stream(self) -> impl Stream<Item = super::Result<Bytes>> {
+        self.stream().filter_map(|item| match item {
+            EncodedItem::Size(size) => {
+                let size = size.to_le_bytes().to_vec().into();
+                Some(Ok(size))
+            }
+            EncodedItem::Parent(parent) => {
+                let mut data = vec![0u8; 64];
+                data[..32].copy_from_slice(parent.pair.0.as_bytes());
+                data[32..].copy_from_slice(parent.pair.1.as_bytes());
+                Some(Ok(data.into()))
+            }
+            EncodedItem::Leaf(leaf) => Some(Ok(leaf.data)),
+            EncodedItem::Done => None,
+            EncodedItem::Error(cause) => Some(Err(super::Error::other(cause))),
+        })
+    }
+
+    pub fn stream(self) -> impl Stream<Item = EncodedItem> {
+        Gen::new(|co| async move {
+            let mut rx = match self.inner.await {
+                Ok(rx) => rx,
+                Err(cause) => {
+                    co.yield_(EncodedItem::Error(io::Error::other(cause).into()))
+                        .await;
+                    return;
+                }
+            };
+            while let Ok(Some(item)) = rx.recv().await {
+                co.yield_(item).await;
+            }
+        })
+    }
+}
+
+pub(crate) trait WriteProgress {
+    /// Notify the progress writer that a payload write has happened.
+    async fn notify_payload_write(&mut self, index: u64, offset: u64, len: usize);
+
+    /// Log a write of some other data.
+    fn log_other_write(&mut self, len: usize);
+
+    /// Notify the progress writer that a transfer has started.
+    async fn send_transfer_started(&mut self, index: u64, hash: &Hash, size: u64);
+}
+
+impl WriteProgress for StreamContext {
+    async fn notify_payload_write(&mut self, index: u64, offset: u64, len: usize) {
+        StreamContext::notify_payload_write(self, index, offset, len);
+    }
+
+    fn log_other_write(&mut self, len: usize) {
+        StreamContext::log_other_write(self, len);
+    }
+
+    async fn send_transfer_started(&mut self, index: u64, hash: &Hash, size: u64) {
+        StreamContext::send_transfer_started(self, index, hash, size).await
+    }
+}
diff --git a/src/api/downloader.rs b/src/api/downloader.rs
new file mode 100644
index 000000000..c86d88999
--- /dev/null
+++ b/src/api/downloader.rs
@@ -0,0 +1,843 @@
+//! API for downloads from multiple nodes.
+use std::{
+    collections::{HashMap, HashSet},
+    fmt::Debug,
+    future::{Future, IntoFuture},
+    io,
+    ops::Deref,
+    sync::Arc,
+    time::{Duration, SystemTime},
+};
+
+use anyhow::bail;
+use genawaiter::sync::Gen;
+use iroh::{endpoint::Connection, Endpoint, NodeId};
+use irpc::{channel::mpsc, rpc_requests};
+use n0_future::{future, stream, BufferedStreamExt, Stream, StreamExt};
+use rand::seq::SliceRandom;
+use serde::{de::Error, Deserialize, Serialize};
+use tokio::{sync::Mutex, task::JoinSet};
+use tokio_util::time::FutureExt;
+use tracing::{info, instrument::Instrument, warn};
+
+use super::{remote::GetConnection, Store};
+use crate::{
+    protocol::{GetManyRequest, GetRequest},
+    util::sink::{Drain, IrpcSenderRefSink, Sink, TokioMpscSenderSink},
+    BlobFormat, Hash, HashAndFormat,
+};
+
+#[derive(Debug, Clone)]
+pub struct Downloader {
+    client: irpc::Client<SwarmMsg, SwarmProtocol, DownloaderService>,
+}
+
+#[derive(Debug, Clone)]
+pub struct DownloaderService;
+
+impl irpc::Service for DownloaderService {}
+
+#[rpc_requests(DownloaderService, message = SwarmMsg, alias = "Msg")]
+#[derive(Debug, Serialize, Deserialize)]
+enum SwarmProtocol {
+    #[rpc(tx = mpsc::Sender<DownloadProgessItem>)]
+    Download(DownloadRequest),
+}
+
+struct DownloaderActor {
+    store: Store,
+    pool: ConnectionPool,
+    tasks: JoinSet<()>,
+    running: HashSet<tokio::task::Id>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub enum DownloadProgessItem {
+    #[serde(skip)]
+    Error(anyhow::Error),
+    TryProvider {
+        id: NodeId,
+        request: Arc<GetRequest>,
+    },
+    ProviderFailed {
+        id: NodeId,
+        request: Arc<GetRequest>,
+    },
+    PartComplete {
+        request: Arc<GetRequest>,
+    },
+    Progress(u64),
+    DownloadError,
+}
+
+impl DownloaderActor {
+    fn new(store: Store, endpoint: Endpoint) -> Self {
+        Self {
+            store,
+            pool: ConnectionPool::new(endpoint, crate::ALPN.to_vec()),
+            tasks: JoinSet::new(),
+            running: HashSet::new(),
+        }
+    }
+
+    async fn run(mut self, mut rx: tokio::sync::mpsc::Receiver<SwarmMsg>) {
+        while let Some(msg) = rx.recv().await {
+            match msg {
+                SwarmMsg::Download(request) => {
+                    self.spawn(handle_download(
+                        self.store.clone(),
+                        self.pool.clone(),
+                        request,
+                    ));
+                }
+            }
+        }
+    }
+
+    fn spawn(&mut self, fut: impl Future<Output = ()> + Send + 'static) {
+        let span = tracing::Span::current();
+        let id = self.tasks.spawn(fut.instrument(span)).id();
+        self.running.insert(id);
+    }
+}
+
+async fn handle_download(store: Store, pool: ConnectionPool, msg: DownloadMsg) {
+    let DownloadMsg { inner, mut tx, .. } = msg;
+    if let Err(cause) = handle_download_impl(store, pool, inner, &mut tx).await {
+        tx.send(DownloadProgessItem::Error(cause)).await.ok();
+    }
+}
+
+async fn handle_download_impl(
+    store: Store,
+    pool: ConnectionPool,
+    request: DownloadRequest,
+    tx: &mut mpsc::Sender<DownloadProgessItem>,
+) -> anyhow::Result<()> {
+    match request.strategy {
+        SplitStrategy::Split => handle_download_split_impl(store, pool, request, tx).await?,
+        SplitStrategy::None => match request.request {
+            FiniteRequest::Get(get) => {
+                let sink = IrpcSenderRefSink(tx).with_map_err(io::Error::other);
+                execute_get(&pool, Arc::new(get), &request.providers, &store, sink).await?;
+            }
+            FiniteRequest::GetMany(_) => {
+                handle_download_split_impl(store, pool, request, tx).await?
+            }
+        },
+    }
+    Ok(())
+}
+
+async fn handle_download_split_impl(
+    store: Store,
+    pool: ConnectionPool,
+    request: DownloadRequest,
+    tx: &mut mpsc::Sender<DownloadProgessItem>,
+) -> anyhow::Result<()> {
+    let providers = request.providers;
+    let requests = split_request(&request.request, &providers, &pool, &store, Drain).await?;
+    let (progress_tx, progress_rx) = tokio::sync::mpsc::channel(32);
+    let mut futs = stream::iter(requests.into_iter().enumerate())
+        .map(|(id, request)| {
+            let pool = pool.clone();
+            let providers = providers.clone();
+            let store = store.clone();
+            let progress_tx = progress_tx.clone();
+            async move {
+                let hash = request.hash;
+                let (tx, rx) = tokio::sync::mpsc::channel::<(usize, DownloadProgessItem)>(16);
+                progress_tx.send(rx).await.ok();
+                let sink = TokioMpscSenderSink(tx)
+                    .with_map_err(io::Error::other)
+                    .with_map(move |x| (id, x));
+                let res = execute_get(&pool, Arc::new(request), &providers, &store, sink).await;
+                (hash, res)
+            }
+        })
+        .buffered_unordered(32);
+    let mut progress_stream = {
+        let mut offsets = HashMap::new();
+        let mut total = 0;
+        into_stream(progress_rx)
+            .flat_map(into_stream)
+            .map(move |(id, item)| match item {
+                DownloadProgessItem::Progress(offset) => {
+                    total += offset;
+                    if let Some(prev) = offsets.insert(id, offset) {
+                        total -= prev;
+                    }
+                    DownloadProgessItem::Progress(total)
+                }
+                x => x,
+            })
+    };
+    loop {
+        tokio::select! {
+            Some(item) = progress_stream.next() => {
+                tx.send(item).await?;
+            },
+            res = futs.next() => {
+                match res {
+                    Some((_hash, Ok(()))) => {
+                    }
+                    Some((_hash, Err(_e))) => {
+                        tx.send(DownloadProgessItem::DownloadError).await?;
+                    }
+                    None => break,
+                }
+            }
+            _ = tx.closed() => {
+                // The sender has been closed, we should stop processing.
+                break;
+            }
+        }
+    }
+    Ok(())
+}
+
+fn into_stream<T>(mut recv: tokio::sync::mpsc::Receiver<T>) -> impl Stream<Item = T> {
+    Gen::new(|co| async move {
+        while let Some(item) = recv.recv().await {
+            co.yield_(item).await;
+        }
+    })
+}
+
+#[derive(Debug, Serialize, Deserialize, derive_more::From)]
+pub enum FiniteRequest {
+    Get(GetRequest),
+    GetMany(GetManyRequest),
+}
+
+pub trait SupportedRequest {
+    fn into_request(self) -> FiniteRequest;
+}
+
+impl<I: Into<Hash>, T: IntoIterator<Item = I>> SupportedRequest for T {
+    fn into_request(self) -> FiniteRequest {
+        let hashes = self.into_iter().map(Into::into).collect::<GetManyRequest>();
+        FiniteRequest::GetMany(hashes)
+    }
+}
+
+impl SupportedRequest for GetRequest {
+    fn into_request(self) -> FiniteRequest {
+        self.into()
+    }
+}
+
+impl SupportedRequest for GetManyRequest {
+    fn into_request(self) -> FiniteRequest {
+        self.into()
+    }
+}
+
+impl SupportedRequest for Hash {
+    fn into_request(self) -> FiniteRequest {
+        GetRequest::blob(self).into()
+    }
+}
+
+impl SupportedRequest for HashAndFormat {
+    fn into_request(self) -> FiniteRequest {
+        (match self.format {
+            BlobFormat::Raw => GetRequest::blob(self.hash),
+            BlobFormat::HashSeq => GetRequest::all(self.hash),
+        })
+        .into()
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct AddProviderRequest {
+    pub hash: Hash,
+    pub providers: Vec<NodeId>,
+}
+
+#[derive(Debug)]
+pub struct DownloadRequest {
+    pub request: FiniteRequest,
+    pub providers: Arc<dyn ContentDiscovery>,
+    pub strategy: SplitStrategy,
+}
+
+impl DownloadRequest {
+    pub fn new(
+        request: impl SupportedRequest,
+        providers: impl ContentDiscovery,
+        strategy: SplitStrategy,
+    ) -> Self {
+        Self {
+            request: request.into_request(),
+            providers: Arc::new(providers),
+            strategy,
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub enum SplitStrategy {
+    None,
+    Split,
+}
+
+impl Serialize for DownloadRequest {
+    fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        Err(serde::ser::Error::custom(
+            "cannot serialize DownloadRequest",
+        ))
+    }
+}
+
+// Implement Deserialize to always fail
+impl<'de> Deserialize<'de> for DownloadRequest {
+    fn deserialize<D>(_deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        Err(D::Error::custom("cannot deserialize DownloadRequest"))
+    }
+}
+
+pub type DownloadOptions = DownloadRequest;
+
+pub struct DownloadProgress {
+    fut: future::Boxed<irpc::Result<mpsc::Receiver<DownloadProgessItem>>>,
+}
+
+impl DownloadProgress {
+    fn new(fut: future::Boxed<irpc::Result<mpsc::Receiver<DownloadProgessItem>>>) -> Self {
+        Self { fut }
+    }
+
+    pub async fn stream(self) -> irpc::Result<impl Stream<Item = DownloadProgessItem> + Unpin> {
+        let rx = self.fut.await?;
+        Ok(Box::pin(rx.into_stream().map(|item| match item {
+            Ok(item) => item,
+            Err(e) => DownloadProgessItem::Error(e.into()),
+        })))
+    }
+
+    async fn complete(self) -> anyhow::Result<()> {
+        let rx = self.fut.await?;
+        let stream = rx.into_stream();
+        tokio::pin!(stream);
+        while let Some(item) = stream.next().await {
+            match item? {
+                DownloadProgessItem::Error(e) => Err(e)?,
+                DownloadProgessItem::DownloadError => anyhow::bail!("Download error"),
+                _ => {}
+            }
+        }
+        Ok(())
+    }
+}
+
+impl IntoFuture for DownloadProgress {
+    type Output = anyhow::Result<()>;
+    type IntoFuture = future::Boxed<Self::Output>;
+
+    fn into_future(self) -> Self::IntoFuture {
+        Box::pin(self.complete())
+    }
+}
+
+impl Downloader {
+    pub fn new(store: &Store, endpoint: &Endpoint) -> Self {
+        let (tx, rx) = tokio::sync::mpsc::channel::<SwarmMsg>(32);
+        let actor = DownloaderActor::new(store.clone(), endpoint.clone());
+        tokio::spawn(actor.run(rx));
+        Self { client: tx.into() }
+    }
+
+    pub fn download(
+        &self,
+        request: impl SupportedRequest,
+        providers: impl ContentDiscovery,
+    ) -> DownloadProgress {
+        let request = request.into_request();
+        let providers = Arc::new(providers);
+        self.download_with_opts(DownloadOptions {
+            request,
+            providers,
+            strategy: SplitStrategy::Split,
+        })
+    }
+
+    pub fn download_with_opts(&self, options: DownloadOptions) -> DownloadProgress {
+        let fut = self.client.server_streaming(options, 32);
+        DownloadProgress::new(Box::pin(fut))
+    }
+}
+
+/// Split a request into multiple requests that can be run in parallel.
+async fn split_request<'a>(
+    request: &'a FiniteRequest,
+    providers: &Arc<dyn ContentDiscovery>,
+    pool: &ConnectionPool,
+    store: &Store,
+    progress: impl Sink<DownloadProgessItem, Error = io::Error>,
+) -> anyhow::Result<Box<dyn Iterator<Item = GetRequest> + Send + 'a>> {
+    Ok(match request {
+        FiniteRequest::Get(req) => {
+            let Some(_first) = req.ranges.iter_infinite().next() else {
+                return Ok(Box::new(std::iter::empty()));
+            };
+            let first = GetRequest::blob(req.hash);
+            execute_get(pool, Arc::new(first), providers, store, progress).await?;
+            let size = store.observe(req.hash).await?.size();
+            anyhow::ensure!(size % 32 == 0, "Size is not a multiple of 32");
+            let n = size / 32;
+            Box::new(
+                req.ranges
+                    .iter_infinite()
+                    .take(n as usize + 1)
+                    .enumerate()
+                    .filter_map(|(i, ranges)| {
+                        if i != 0 && !ranges.is_empty() {
+                            Some(
+                                GetRequest::builder()
+                                    .offset(i as u64, ranges.clone())
+                                    .build(req.hash),
+                            )
+                        } else {
+                            None
+                        }
+                    }),
+            )
+        }
+        FiniteRequest::GetMany(req) => Box::new(
+            req.hashes
+                .iter()
+                .enumerate()
+                .map(|(i, hash)| GetRequest::blob_ranges(*hash, req.ranges[i as u64].clone())),
+        ),
+    })
+}
+
+#[derive(Debug)]
+struct ConnectionPoolInner {
+    alpn: Vec<u8>,
+    endpoint: Endpoint,
+    connections: Mutex<HashMap<NodeId, Arc<Mutex<SlotState>>>>,
+    retry_delay: Duration,
+    connect_timeout: Duration,
+}
+
+#[derive(Debug, Clone)]
+struct ConnectionPool(Arc<ConnectionPoolInner>);
+
+#[derive(Debug, Default)]
+enum SlotState {
+    #[default]
+    Initial,
+    Connected(Connection),
+    AttemptFailed(SystemTime),
+    #[allow(dead_code)]
+    Evil(String),
+}
+
+impl ConnectionPool {
+    fn new(endpoint: Endpoint, alpn: Vec<u8>) -> Self {
+        Self(
+            ConnectionPoolInner {
+                endpoint,
+                alpn,
+                connections: Default::default(),
+                retry_delay: Duration::from_secs(5),
+                connect_timeout: Duration::from_secs(2),
+            }
+            .into(),
+        )
+    }
+
+    pub fn alpn(&self) -> &[u8] {
+        &self.0.alpn
+    }
+
+    pub fn endpoint(&self) -> &Endpoint {
+        &self.0.endpoint
+    }
+
+    pub fn retry_delay(&self) -> Duration {
+        self.0.retry_delay
+    }
+
+    fn dial(&self, id: NodeId) -> DialNode {
+        DialNode {
+            pool: self.clone(),
+            id,
+        }
+    }
+
+    #[allow(dead_code)]
+    async fn mark_evil(&self, id: NodeId, reason: String) {
+        let slot = self
+            .0
+            .connections
+            .lock()
+            .await
+            .entry(id)
+            .or_default()
+            .clone();
+        let mut t = slot.lock().await;
+        *t = SlotState::Evil(reason)
+    }
+
+    #[allow(dead_code)]
+    async fn mark_closed(&self, id: NodeId) {
+        let slot = self
+            .0
+            .connections
+            .lock()
+            .await
+            .entry(id)
+            .or_default()
+            .clone();
+        let mut t = slot.lock().await;
+        *t = SlotState::Initial
+    }
+}
+
+/// Execute a get request sequentially for multiple providers.
+///
+/// It will try each provider in order
+/// until it finds one that can fulfill the request. When trying a new provider,
+/// it takes the progress from the previous providers into account, so e.g.
+/// if the first provider had the first 10% of the data, it will only ask the next
+/// provider for the remaining 90%.
+///
+/// This is fully sequential, so there will only be one request in flight at a time.
+///
+/// If the request is not complete after trying all providers, it will return an error.
+/// If the provider stream never ends, it will try indefinitely.
+async fn execute_get(
+    pool: &ConnectionPool,
+    request: Arc<GetRequest>,
+    providers: &Arc<dyn ContentDiscovery>,
+    store: &Store,
+    mut progress: impl Sink<DownloadProgessItem, Error = io::Error>,
+) -> anyhow::Result<()> {
+    let remote = store.remote();
+    let mut providers = providers.find_providers(request.content());
+    while let Some(provider) = providers.next().await {
+        progress
+            .send(DownloadProgessItem::TryProvider {
+                id: provider,
+                request: request.clone(),
+            })
+            .await?;
+        let mut conn = pool.dial(provider);
+        let local = remote.local_for_request(request.clone()).await?;
+        if local.is_complete() {
+            return Ok(());
+        }
+        let local_bytes = local.local_bytes();
+        let Ok(conn) = conn.connection().await else {
+            progress
+                .send(DownloadProgessItem::ProviderFailed {
+                    id: provider,
+                    request: request.clone(),
+                })
+                .await?;
+            continue;
+        };
+        match remote
+            .execute_get_sink(
+                conn,
+                local.missing(),
+                (&mut progress).with_map(move |x| DownloadProgessItem::Progress(x + local_bytes)),
+            )
+            .await
+        {
+            Ok(_stats) => {
+                progress
+                    .send(DownloadProgessItem::PartComplete {
+                        request: request.clone(),
+                    })
+                    .await?;
+                return Ok(());
+            }
+            Err(_cause) => {
+                progress
+                    .send(DownloadProgessItem::ProviderFailed {
+                        id: provider,
+                        request: request.clone(),
+                    })
+                    .await?;
+                continue;
+            }
+        }
+    }
+    bail!("Unable to download {}", request.hash);
+}
+
+#[derive(Debug, Clone)]
+struct DialNode {
+    pool: ConnectionPool,
+    id: NodeId,
+}
+
+impl DialNode {
+    async fn connection_impl(&self) -> anyhow::Result<Connection> {
+        info!("Getting connection for node {}", self.id);
+        let slot = self
+            .pool
+            .0
+            .connections
+            .lock()
+            .await
+            .entry(self.id)
+            .or_default()
+            .clone();
+        info!("Dialing node {}", self.id);
+        let mut guard = slot.lock().await;
+        match guard.deref() {
+            SlotState::Connected(conn) => {
+                return Ok(conn.clone());
+            }
+            SlotState::AttemptFailed(time) => {
+                let elapsed = time.elapsed().unwrap_or_default();
+                if elapsed <= self.pool.retry_delay() {
+                    bail!(
+                        "Connection attempt failed {} seconds ago",
+                        elapsed.as_secs_f64()
+                    );
+                }
+            }
+            SlotState::Evil(reason) => {
+                bail!("Node is banned due to evil behavior: {reason}");
+            }
+            SlotState::Initial => {}
+        }
+        let res = self
+            .pool
+            .endpoint()
+            .connect(self.id, self.pool.alpn())
+            .timeout(self.pool.0.connect_timeout)
+            .await;
+        match res {
+            Ok(Ok(conn)) => {
+                info!("Connected to node {}", self.id);
+                *guard = SlotState::Connected(conn.clone());
+                Ok(conn)
+            }
+            Ok(Err(e)) => {
+                warn!("Failed to connect to node {}: {}", self.id, e);
+                *guard = SlotState::AttemptFailed(SystemTime::now());
+                Err(e.into())
+            }
+            Err(e) => {
+                warn!("Failed to connect to node {}: {}", self.id, e);
+                *guard = SlotState::AttemptFailed(SystemTime::now());
+                bail!("Failed to connect to node: {}", e);
+            }
+        }
+    }
+}
+
+impl GetConnection for DialNode {
+    fn connection(&mut self) -> impl Future<Output = Result<Connection, anyhow::Error>> + '_ {
+        let this = self.clone();
+        async move { this.connection_impl().await }
+    }
+}
+
+/// Trait for pluggable content discovery strategies.
+pub trait ContentDiscovery: Debug + Send + Sync + 'static {
+    fn find_providers(&self, hash: HashAndFormat) -> n0_future::stream::Boxed<NodeId>;
+}
+
+impl<C, I> ContentDiscovery for C
+where
+    C: Debug + Clone + IntoIterator<Item = I> + Send + Sync + 'static,
+    C::IntoIter: Send + Sync + 'static,
+    I: Into<NodeId> + Send + Sync + 'static,
+{
+    fn find_providers(&self, _: HashAndFormat) -> n0_future::stream::Boxed<NodeId> {
+        let providers = self.clone();
+        n0_future::stream::iter(providers.into_iter().map(Into::into)).boxed()
+    }
+}
+
+#[derive(derive_more::Debug)]
+pub struct Shuffled {
+    nodes: Vec<NodeId>,
+}
+
+impl Shuffled {
+    pub fn new(nodes: Vec<NodeId>) -> Self {
+        Self { nodes }
+    }
+}
+
+impl ContentDiscovery for Shuffled {
+    fn find_providers(&self, _: HashAndFormat) -> n0_future::stream::Boxed<NodeId> {
+        let mut nodes = self.nodes.clone();
+        nodes.shuffle(&mut rand::thread_rng());
+        n0_future::stream::iter(nodes).boxed()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::ops::Deref;
+
+    use bao_tree::ChunkRanges;
+    use iroh::Watcher;
+    use n0_future::StreamExt;
+    use testresult::TestResult;
+
+    use crate::{
+        api::{
+            blobs::AddBytesOptions,
+            downloader::{DownloadOptions, Downloader, Shuffled, SplitStrategy},
+        },
+        hashseq::HashSeq,
+        protocol::{GetManyRequest, GetRequest},
+        tests::node_test_setup_fs,
+    };
+
+    #[tokio::test]
+    #[ignore = "todo"]
+    async fn downloader_get_many_smoke() -> TestResult<()> {
+        let testdir = tempfile::tempdir()?;
+        let (r1, store1, _) = node_test_setup_fs(testdir.path().join("a")).await?;
+        let (r2, store2, _) = node_test_setup_fs(testdir.path().join("b")).await?;
+        let (r3, store3, _) = node_test_setup_fs(testdir.path().join("c")).await?;
+        let tt1 = store1.add_slice("hello world").await?;
+        let tt2 = store2.add_slice("hello world 2").await?;
+        let node1_addr = r1.endpoint().node_addr().initialized().await?;
+        let node1_id = node1_addr.node_id;
+        let node2_addr = r2.endpoint().node_addr().initialized().await?;
+        let node2_id = node2_addr.node_id;
+        let swarm = Downloader::new(&store3, r3.endpoint());
+        r3.endpoint().add_node_addr(node1_addr.clone())?;
+        r3.endpoint().add_node_addr(node2_addr.clone())?;
+        let request = GetManyRequest::builder()
+            .hash(tt1.hash, ChunkRanges::all())
+            .hash(tt2.hash, ChunkRanges::all())
+            .build();
+        let mut progress = swarm
+            .download(request, Shuffled::new(vec![node1_id, node2_id]))
+            .stream()
+            .await?;
+        while let Some(item) = progress.next().await {
+            println!("Got item: {item:?}");
+        }
+        assert_eq!(store3.get_bytes(tt1.hash).await?.deref(), b"hello world");
+        assert_eq!(store3.get_bytes(tt2.hash).await?.deref(), b"hello world 2");
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn downloader_get_smoke() -> TestResult<()> {
+        // tracing_subscriber::fmt::try_init().ok();
+        let testdir = tempfile::tempdir()?;
+        let (r1, store1, _) = node_test_setup_fs(testdir.path().join("a")).await?;
+        let (r2, store2, _) = node_test_setup_fs(testdir.path().join("b")).await?;
+        let (r3, store3, _) = node_test_setup_fs(testdir.path().join("c")).await?;
+        let tt1 = store1.add_slice(vec![1; 10000000]).await?;
+        let tt2 = store2.add_slice(vec![2; 10000000]).await?;
+        let hs = [tt1.hash, tt2.hash].into_iter().collect::<HashSeq>();
+        let root = store1
+            .add_bytes_with_opts(AddBytesOptions {
+                data: hs.clone().into(),
+                format: crate::BlobFormat::HashSeq,
+            })
+            .await?;
+        let node1_addr = r1.endpoint().node_addr().initialized().await?;
+        let node1_id = node1_addr.node_id;
+        let node2_addr = r2.endpoint().node_addr().initialized().await?;
+        let node2_id = node2_addr.node_id;
+        let swarm = Downloader::new(&store3, r3.endpoint());
+        r3.endpoint().add_node_addr(node1_addr.clone())?;
+        r3.endpoint().add_node_addr(node2_addr.clone())?;
+        let request = GetRequest::builder()
+            .root(ChunkRanges::all())
+            .next(ChunkRanges::all())
+            .next(ChunkRanges::all())
+            .build(root.hash);
+        if true {
+            let mut progress = swarm
+                .download_with_opts(DownloadOptions::new(
+                    request,
+                    [node1_id, node2_id],
+                    SplitStrategy::Split,
+                ))
+                .stream()
+                .await?;
+            while let Some(item) = progress.next().await {
+                println!("Got item: {item:?}");
+            }
+        }
+        if false {
+            let conn = r3.endpoint().connect(node1_addr, crate::ALPN).await?;
+            let remote = store3.remote();
+            let _rh = remote
+                .execute_get(
+                    conn.clone(),
+                    GetRequest::builder()
+                        .root(ChunkRanges::all())
+                        .build(root.hash),
+                )
+                .await?;
+            let h1 = remote.execute_get(
+                conn.clone(),
+                GetRequest::builder()
+                    .child(0, ChunkRanges::all())
+                    .build(root.hash),
+            );
+            let h2 = remote.execute_get(
+                conn.clone(),
+                GetRequest::builder()
+                    .child(1, ChunkRanges::all())
+                    .build(root.hash),
+            );
+            h1.await?;
+            h2.await?;
+        }
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn downloader_get_all() -> TestResult<()> {
+        let testdir = tempfile::tempdir()?;
+        let (r1, store1, _) = node_test_setup_fs(testdir.path().join("a")).await?;
+        let (r2, store2, _) = node_test_setup_fs(testdir.path().join("b")).await?;
+        let (r3, store3, _) = node_test_setup_fs(testdir.path().join("c")).await?;
+        let tt1 = store1.add_slice(vec![1; 10000000]).await?;
+        let tt2 = store2.add_slice(vec![2; 10000000]).await?;
+        let hs = [tt1.hash, tt2.hash].into_iter().collect::<HashSeq>();
+        let root = store1
+            .add_bytes_with_opts(AddBytesOptions {
+                data: hs.clone().into(),
+                format: crate::BlobFormat::HashSeq,
+            })
+            .await?;
+        let node1_addr = r1.endpoint().node_addr().initialized().await?;
+        let node1_id = node1_addr.node_id;
+        let node2_addr = r2.endpoint().node_addr().initialized().await?;
+        let node2_id = node2_addr.node_id;
+        let swarm = Downloader::new(&store3, r3.endpoint());
+        r3.endpoint().add_node_addr(node1_addr.clone())?;
+        r3.endpoint().add_node_addr(node2_addr.clone())?;
+        let request = GetRequest::all(root.hash);
+        let mut progress = swarm
+            .download_with_opts(DownloadOptions::new(
+                request,
+                [node1_id, node2_id],
+                SplitStrategy::Split,
+            ))
+            .stream()
+            .await?;
+        while let Some(item) = progress.next().await {
+            println!("Got item: {item:?}");
+        }
+        Ok(())
+    }
+}
diff --git a/src/api/proto.rs b/src/api/proto.rs
new file mode 100644
index 000000000..ed3686e12
--- /dev/null
+++ b/src/api/proto.rs
@@ -0,0 +1,705 @@
+#![cfg_attr(feature = "hide-proto-docs", doc(hidden))]
+//! The protocol that a store implementation needs to implement.
+//!
+//! A store needs to handle [`Request`]s. It is fine to just return an error for some
+//! commands. E.g. an immutable store can just return an error for import commands.
+//!
+//! Each command consists of a serializable request message and channels for updates
+//! and responses. The enum containing the full requests is [`Command`]. These are the
+//! commands you will have to handle in a store actor handler.
+//!
+//! This crate provides a file system based store implementation, [`crate::store::fs::FsStore`],
+//! as well as a mutable in-memory store and an immutable in-memory store.
+//!
+//! The file system store is quite complex and optimized, so to get started take a look at
+//! the much simpler memory store.
+use std::{
+    fmt::{self, Debug},
+    io,
+    num::NonZeroU64,
+    ops::{Bound, RangeBounds},
+    path::PathBuf,
+    pin::Pin,
+};
+
+use arrayvec::ArrayString;
+use bao_tree::{
+    io::{mixed::EncodedItem, BaoContentItem, Leaf},
+    ChunkRanges,
+};
+use bytes::Bytes;
+use irpc::{
+    channel::{mpsc, oneshot},
+    rpc_requests,
+};
+use n0_future::Stream;
+use range_collections::RangeSet2;
+use serde::{Deserialize, Serialize};
+pub(crate) mod bitfield;
+pub use bitfield::Bitfield;
+
+use crate::{store::util::Tag, util::temp_tag::TempTag, BlobFormat, Hash, HashAndFormat};
+
+pub(crate) trait HashSpecific {
+    fn hash(&self) -> Hash;
+
+    fn hash_short(&self) -> ArrayString<10> {
+        self.hash().fmt_short()
+    }
+}
+
+impl HashSpecific for ImportBaoMsg {
+    fn hash(&self) -> crate::Hash {
+        self.inner.hash
+    }
+}
+
+impl HashSpecific for ObserveMsg {
+    fn hash(&self) -> crate::Hash {
+        self.inner.hash
+    }
+}
+
+impl HashSpecific for ExportBaoMsg {
+    fn hash(&self) -> crate::Hash {
+        self.inner.hash
+    }
+}
+
+impl HashSpecific for ExportRangesMsg {
+    fn hash(&self) -> crate::Hash {
+        self.inner.hash
+    }
+}
+
+impl HashSpecific for ExportPathMsg {
+    fn hash(&self) -> crate::Hash {
+        self.inner.hash
+    }
+}
+
+pub type BoxedByteStream = Pin<Box<dyn Stream<Item = io::Result<Bytes>> + Send + Sync + 'static>>;
+
+impl HashSpecific for CreateTagMsg {
+    fn hash(&self) -> crate::Hash {
+        self.inner.value.hash
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct StoreService;
+impl irpc::Service for StoreService {}
+
+#[rpc_requests(StoreService, message = Command, alias = "Msg")]
+#[derive(Debug, Serialize, Deserialize)]
+pub enum Request {
+    #[rpc(tx = mpsc::Sender<super::Result<Hash>>)]
+    ListBlobs(ListRequest),
+    #[rpc(tx = oneshot::Sender<Scope>, rx = mpsc::Receiver<BatchResponse>)]
+    Batch(BatchRequest),
+    #[rpc(tx = oneshot::Sender<super::Result<()>>)]
+    DeleteBlobs(BlobDeleteRequest),
+    #[rpc(rx = mpsc::Receiver<BaoContentItem>, tx = oneshot::Sender<super::Result<()>>)]
+    ImportBao(ImportBaoRequest),
+    #[rpc(tx = mpsc::Sender<EncodedItem>)]
+    ExportBao(ExportBaoRequest),
+    #[rpc(tx = mpsc::Sender<ExportRangesItem>)]
+    ExportRanges(ExportRangesRequest),
+    #[rpc(tx = mpsc::Sender<Bitfield>)]
+    Observe(ObserveRequest),
+    #[rpc(tx = oneshot::Sender<BlobStatus>)]
+    BlobStatus(BlobStatusRequest),
+    #[rpc(tx = mpsc::Sender<AddProgressItem>)]
+    ImportBytes(ImportBytesRequest),
+    #[rpc(rx = mpsc::Receiver<ImportByteStreamUpdate>, tx = mpsc::Sender<AddProgressItem>)]
+    ImportByteStream(ImportByteStreamRequest),
+    #[rpc(tx = mpsc::Sender<AddProgressItem>)]
+    ImportPath(ImportPathRequest),
+    #[rpc(tx = mpsc::Sender<ExportProgressItem>)]
+    ExportPath(ExportPathRequest),
+    #[rpc(tx = oneshot::Sender<Vec<super::Result<TagInfo>>>)]
+    ListTags(ListTagsRequest),
+    #[rpc(tx = oneshot::Sender<super::Result<()>>)]
+    SetTag(SetTagRequest),
+    #[rpc(tx = oneshot::Sender<super::Result<()>>)]
+    DeleteTags(DeleteTagsRequest),
+    #[rpc(tx = oneshot::Sender<super::Result<()>>)]
+    RenameTag(RenameTagRequest),
+    #[rpc(tx = oneshot::Sender<super::Result<Tag>>)]
+    CreateTag(CreateTagRequest),
+    #[rpc(tx = oneshot::Sender<Vec<HashAndFormat>>)]
+    ListTempTags(ListTempTagsRequest),
+    #[rpc(tx = oneshot::Sender<TempTag>)]
+    CreateTempTag(CreateTempTagRequest),
+    #[rpc(tx = oneshot::Sender<super::Result<()>>)]
+    SyncDb(SyncDbRequest),
+    #[rpc(tx = oneshot::Sender<()>)]
+    Shutdown(ShutdownRequest),
+    #[rpc(tx = oneshot::Sender<super::Result<()>>)]
+    ClearProtected(ClearProtectedRequest),
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct SyncDbRequest;
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ShutdownRequest;
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ClearProtectedRequest;
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct BlobStatusRequest {
+    pub hash: Hash,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ListRequest;
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct BatchRequest;
+
+#[derive(Debug, Serialize, Deserialize)]
+pub enum BatchResponse {
+    Drop(HashAndFormat),
+    Ping,
+}
+
+/// Options for force deletion of blobs.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct BlobDeleteRequest {
+    pub hashes: Vec<Hash>,
+    pub force: bool,
+}
+
+/// Import the given bytes.
+#[derive(Serialize, Deserialize)]
+pub struct ImportBytesRequest {
+    pub data: Bytes,
+    pub format: BlobFormat,
+    pub scope: Scope,
+}
+
+impl fmt::Debug for ImportBytesRequest {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("ImportBytes")
+            .field("data", &self.data.len())
+            .field("format", &self.format)
+            .field("scope", &self.scope)
+            .finish()
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ImportPathRequest {
+    pub path: PathBuf,
+    pub mode: ImportMode,
+    pub format: BlobFormat,
+    pub scope: Scope,
+}
+
+/// Import bao encoded data for the given hash with the iroh block size.
+///
+/// The result is just a single item, indicating if a write error occurred.
+/// To observe the incoming data more granularly, use the `Observe` command
+/// concurrently.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ImportBaoRequest {
+    pub hash: Hash,
+    pub size: NonZeroU64,
+}
+
+/// Observe the local bitfield of the given hash.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ObserveRequest {
+    pub hash: Hash,
+}
+
+/// Export the given ranges in bao format, with the iroh block size.
+///
+/// The returned stream should be verified by the store.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ExportBaoRequest {
+    pub hash: Hash,
+    pub ranges: ChunkRanges,
+}
+
+/// Export the given ranges as chunkks, without validation.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ExportRangesRequest {
+    pub hash: Hash,
+    pub ranges: RangeSet2<u64>,
+}
+
+/// Export a file to a target path.
+///
+/// For an incomplete file, the size might be truncated and gaps will be filled
+/// with zeros. If possible, a store implementation should try to write as a
+/// sparse file.
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ExportPathRequest {
+    pub hash: Hash,
+    pub mode: ExportMode,
+    pub target: PathBuf,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ImportByteStreamRequest {
+    pub format: BlobFormat,
+    pub scope: Scope,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub enum ImportByteStreamUpdate {
+    Bytes(Bytes),
+    Done,
+}
+
+/// Options for a list operation.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ListTagsRequest {
+    /// List tags to hash seqs
+    pub hash_seq: bool,
+    /// List tags to raw blobs
+    pub raw: bool,
+    /// Optional from tag (inclusive)
+    pub from: Option<Tag>,
+    /// Optional to tag (exclusive)
+    pub to: Option<Tag>,
+}
+
+impl ListTagsRequest {
+    /// List a range of tags
+    pub fn range<R, E>(range: R) -> Self
+    where
+        R: RangeBounds<E>,
+        E: AsRef<[u8]>,
+    {
+        let (from, to) = tags_from_range(range);
+        Self {
+            from,
+            to,
+            raw: true,
+            hash_seq: true,
+        }
+    }
+
+    /// List tags with a prefix
+    pub fn prefix(prefix: &[u8]) -> Self {
+        let from = Tag::from(prefix);
+        let to = from.next_prefix();
+        Self {
+            raw: true,
+            hash_seq: true,
+            from: Some(from),
+            to,
+        }
+    }
+
+    /// List a single tag
+    pub fn single(name: &[u8]) -> Self {
+        let from = Tag::from(name);
+        Self {
+            to: Some(from.successor()),
+            from: Some(from),
+            raw: true,
+            hash_seq: true,
+        }
+    }
+
+    /// List all tags
+    pub fn all() -> Self {
+        Self {
+            raw: true,
+            hash_seq: true,
+            from: None,
+            to: None,
+        }
+    }
+
+    /// List raw tags
+    pub fn raw() -> Self {
+        Self {
+            raw: true,
+            hash_seq: false,
+            from: None,
+            to: None,
+        }
+    }
+
+    /// List hash seq tags
+    pub fn hash_seq() -> Self {
+        Self {
+            raw: false,
+            hash_seq: true,
+            from: None,
+            to: None,
+        }
+    }
+}
+
+/// Information about a tag.
+#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TagInfo {
+    /// Name of the tag
+    pub name: Tag,
+    /// Format of the data
+    pub format: BlobFormat,
+    /// Hash of the data
+    pub hash: Hash,
+}
+
+impl From<TagInfo> for HashAndFormat {
+    fn from(tag_info: TagInfo) -> Self {
+        HashAndFormat {
+            hash: tag_info.hash,
+            format: tag_info.format,
+        }
+    }
+}
+
+impl TagInfo {
+    /// Create a new tag info.
+    pub fn new(name: impl AsRef<[u8]>, value: impl Into<HashAndFormat>) -> Self {
+        let name = name.as_ref();
+        let value = value.into();
+        Self {
+            name: Tag::from(name),
+            hash: value.hash,
+            format: value.format,
+        }
+    }
+
+    /// Get the hash and format of the tag.
+    pub fn hash_and_format(&self) -> HashAndFormat {
+        HashAndFormat {
+            hash: self.hash,
+            format: self.format,
+        }
+    }
+}
+
+pub(crate) fn tags_from_range<R, E>(range: R) -> (Option<Tag>, Option<Tag>)
+where
+    R: RangeBounds<E>,
+    E: AsRef<[u8]>,
+{
+    let from = match range.start_bound() {
+        Bound::Included(start) => Some(Tag::from(start.as_ref())),
+        Bound::Excluded(start) => Some(Tag::from(start.as_ref()).successor()),
+        Bound::Unbounded => None,
+    };
+    let to = match range.end_bound() {
+        Bound::Included(end) => Some(Tag::from(end.as_ref()).successor()),
+        Bound::Excluded(end) => Some(Tag::from(end.as_ref())),
+        Bound::Unbounded => None,
+    };
+    (from, to)
+}
+
+/// List all temp tags
+#[derive(Debug, Serialize, Deserialize)]
+pub struct CreateTempTagRequest {
+    pub scope: Scope,
+    pub value: HashAndFormat,
+}
+
+/// List all temp tags
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ListTempTagsRequest;
+
+/// Rename a tag atomically
+#[derive(Debug, Serialize, Deserialize)]
+pub struct RenameTagRequest {
+    /// Old tag name
+    pub from: Tag,
+    /// New tag name
+    pub to: Tag,
+}
+
+/// Options for a delete operation.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DeleteTagsRequest {
+    /// Optional from tag (inclusive)
+    pub from: Option<Tag>,
+    /// Optional to tag (exclusive)
+    pub to: Option<Tag>,
+}
+
+impl DeleteTagsRequest {
+    /// Delete a single tag
+    pub fn single(name: &[u8]) -> Self {
+        let name = Tag::from(name);
+        Self {
+            to: Some(name.successor()),
+            from: Some(name),
+        }
+    }
+
+    /// Delete a range of tags
+    pub fn range<R, E>(range: R) -> Self
+    where
+        R: RangeBounds<E>,
+        E: AsRef<[u8]>,
+    {
+        let (from, to) = tags_from_range(range);
+        Self { from, to }
+    }
+
+    /// Delete tags with a prefix
+    pub fn prefix(prefix: &[u8]) -> Self {
+        let from = Tag::from(prefix);
+        let to = from.next_prefix();
+        Self {
+            from: Some(from),
+            to,
+        }
+    }
+}
+
+/// Options for creating a tag or setting it to a new value.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct SetTagRequest {
+    pub name: Tag,
+    pub value: HashAndFormat,
+}
+
+/// Options for creating a tag
+#[derive(Debug, Serialize, Deserialize)]
+pub struct CreateTagRequest {
+    pub value: HashAndFormat,
+}
+
+/// Debug tool to exit the process in the middle of a write transaction, for testing.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ProcessExitRequest {
+    pub code: i32,
+}
+
+/// Progress events for importing from any local source.
+///
+/// For sources with known size such as blobs or files, you will get the events
+/// in the following order:
+///
+/// Size -> CopyProgress(*n) -> CopyDone -> OutboardProgress(*n) -> Done
+///
+/// For sources with unknown size such as streams, you will get the events
+/// in the following order:
+///
+/// CopyProgress(*n) -> Size -> CopyDone -> OutboardProgress(*n) -> Done
+///
+/// Errors can happen at any time, and will be reported as an `Error` event.
+#[derive(Debug, Serialize, Deserialize)]
+pub enum AddProgressItem {
+    /// Progress copying the file into the data directory.
+    ///
+    /// On most modern systems, copying will be done with copy on write,
+    /// so copying will be instantaneous and you won't get any of these.
+    ///
+    /// The number is the *byte offset* of the copy process.
+    ///
+    /// This is an ephemeral progress event, so you can't rely on getting
+    /// regular updates.
+    CopyProgress(u64),
+    /// Size of the file or stream has been determined.
+    ///
+    /// For some input such as blobs or files, the size is immediately known.
+    /// For other inputs such as streams, the size is determined by reading
+    /// the stream to the end.
+    ///
+    /// This is a guaranteed progress event, so you can rely on getting exactly
+    /// one of these.
+    Size(u64),
+    /// The copy part of the import operation is done.
+    ///
+    /// This is a guaranteed progress event, so you can rely on getting exactly
+    /// one of these.
+    CopyDone,
+    /// Progress computing the outboard and root hash of the imported data.
+    ///
+    /// This is an ephemeral progress event, so you can't rely on getting
+    /// regular updates.
+    OutboardProgress(u64),
+    /// The import is done. Once you get this event the data is available
+    /// and protected in the store via the temp tag.
+    ///
+    /// This is a guaranteed progress event, so you can rely on getting exactly
+    /// one of these if the operation was successful.
+    ///
+    /// This is one of the two possible final events. After this event, there
+    /// won't be any more progress events.
+    Done(TempTag),
+    /// The import failed with an error. Partial data will be deleted.
+    ///
+    /// This is a guaranteed progress event, so you can rely on getting exactly
+    /// one of these if the operation was unsuccessful.
+    ///
+    /// This is one of the two possible final events. After this event, there
+    /// won't be any more progress events.
+    Error(#[serde(with = "crate::util::serde::io_error_serde")] io::Error),
+}
+
+impl From<io::Error> for AddProgressItem {
+    fn from(e: io::Error) -> Self {
+        Self::Error(e)
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub enum ExportRangesItem {
+    /// The size of the file being exported.
+    ///
+    /// This is a guaranteed progress event, so you can rely on getting exactly
+    /// one of these.
+    Size(u64),
+    /// A range of the file being exported.
+    Data(Leaf),
+    /// Error while exporting the ranges.
+    Error(super::Error),
+}
+
+impl From<super::Error> for ExportRangesItem {
+    fn from(e: super::Error) -> Self {
+        Self::Error(e)
+    }
+}
+
+impl From<Leaf> for ExportRangesItem {
+    fn from(leaf: Leaf) -> Self {
+        Self::Data(leaf)
+    }
+}
+
+/// Progress events for exporting to a local file.
+///
+/// Exporting does not involve outboard computation, so the events are simpler
+/// than [`AddProgressItem`].
+///
+/// Size -> CopyProgress(*n) -> Done
+///
+/// Errors can happen at any time, and will be reported as an `Error` event.
+#[derive(Debug, Serialize, Deserialize)]
+pub enum ExportProgressItem {
+    /// The size of the file being exported.
+    ///
+    /// This is a guaranteed progress event, so you can rely on getting exactly
+    /// one of these.
+    Size(u64),
+    /// Progress copying the file to the target directory.
+    ///
+    /// On many modern systems, copying will be done with copy on write,
+    /// so copying will be instantaneous and you won't get any of these.
+    ///
+    /// This is an ephemeral progress event, so you can't rely on getting
+    /// regular updates.
+    CopyProgress(u64),
+    /// The export is done. Once you get this event the data is available.
+    ///
+    /// This is a guaranteed progress event, so you can rely on getting exactly
+    /// one of these if the operation was successful.
+    ///
+    /// This is one of the two possible final events. After this event, there
+    /// won't be any more progress events.
+    Done,
+    /// The export failed with an error.
+    ///
+    /// This is a guaranteed progress event, so you can rely on getting exactly
+    /// one of these if the operation was unsuccessful.
+    ///
+    /// This is one of the two possible final events. After this event, there
+    /// won't be any more progress events.
+    Error(super::Error),
+}
+
+impl From<super::Error> for ExportProgressItem {
+    fn from(e: super::Error) -> Self {
+        Self::Error(e)
+    }
+}
+
+/// The import mode describes how files will be imported.
+///
+/// This is a hint to the import trait method. For some implementations, this
+/// does not make any sense. E.g. an in memory implementation will always have
+/// to copy the file into memory. Also, a disk based implementation might choose
+/// to copy small files even if the mode is `Reference`.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+pub enum ImportMode {
+    /// This mode will copy the file into the database before hashing.
+    ///
+    /// This is the safe default because the file can not be accidentally modified
+    /// after it has been imported.
+    #[default]
+    Copy,
+    /// This mode will try to reference the file in place and assume it is unchanged after import.
+    ///
+    /// This has a large performance and storage benefit, but it is less safe since
+    /// the file might be modified after it has been imported.
+    ///
+    /// Stores are allowed to ignore this mode and always copy the file, e.g.
+    /// if the file is very small or if the store does not support referencing files.
+    TryReference,
+}
+
+/// The import mode describes how files will be imported.
+///
+/// This is a hint to the import trait method. For some implementations, this
+/// does not make any sense. E.g. an in memory implementation will always have
+/// to copy the file into memory. Also, a disk based implementation might choose
+/// to copy small files even if the mode is `Reference`.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize, Serialize)]
+pub enum ExportMode {
+    /// This mode will copy the file to the target directory.
+    ///
+    /// This is the safe default because the file can not be accidentally modified
+    /// after it has been exported.
+    #[default]
+    Copy,
+    /// This mode will try to move the file to the target directory and then reference it from
+    /// the database.
+    ///
+    /// This has a large performance and storage benefit, but it is less safe since
+    /// the file might be modified in the target directory after it has been exported.
+    ///
+    /// Stores are allowed to ignore this mode and always copy the file, e.g.
+    /// if the file is very small or if the store does not support referencing files.
+    TryReference,
+}
+
+/// Status information about a blob.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub enum BlobStatus {
+    /// The blob is not stored at all.
+    NotFound,
+    /// The blob is only stored partially.
+    Partial {
+        /// The size of the currently stored partial blob.
+        size: Option<u64>,
+    },
+    /// The blob is stored completely.
+    Complete {
+        /// The size of the blob.
+        size: u64,
+    },
+}
+
+/// A scope for a write operation.
+#[derive(
+    Serialize, Deserialize, Default, Clone, Copy, PartialEq, Eq, Hash, derive_more::Display,
+)]
+pub struct Scope(pub(crate) u64);
+
+impl Scope {
+    pub const GLOBAL: Self = Self(0);
+}
+
+impl std::fmt::Debug for Scope {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        if self.0 == 0 {
+            write!(f, "Global")
+        } else {
+            f.debug_tuple("Scope").field(&self.0).finish()
+        }
+    }
+}
diff --git a/src/api/proto/bitfield.rs b/src/api/proto/bitfield.rs
new file mode 100644
index 000000000..d3ccca66b
--- /dev/null
+++ b/src/api/proto/bitfield.rs
@@ -0,0 +1,325 @@
+use std::{cmp::Ordering, num::NonZeroU64};
+
+use bao_tree::{ChunkNum, ChunkRanges};
+use range_collections::range_set::RangeSetRange;
+use serde::{Deserialize, Deserializer, Serialize};
+use smallvec::SmallVec;
+
+use crate::store::util::{
+    observer::{Combine, CombineInPlace},
+    RangeSetExt,
+};
+
+pub(crate) fn is_validated(size: NonZeroU64, ranges: &ChunkRanges) -> bool {
+    let size = size.get();
+    // ChunkNum::chunks will be at least 1, so this is safe.
+    let last_chunk = ChunkNum::chunks(size) - 1;
+    ranges.contains(&last_chunk)
+}
+
+pub fn is_complete(size: NonZeroU64, ranges: &ChunkRanges) -> bool {
+    let complete = ChunkRanges::from(..ChunkNum::chunks(size.get()));
+    // is_subset is a bit weirdly named. This means that complete is a subset of ranges.
+    complete.is_subset(ranges)
+}
+
+/// The state of a bitfield, or an update to a bitfield
+#[derive(Debug, PartialEq, Eq, Clone, Default)]
+pub struct Bitfield {
+    /// Possible update to the size information. can this be just a u64?
+    pub(crate) size: u64,
+    /// The ranges that were added
+    pub ranges: ChunkRanges,
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Default)]
+pub struct BitfieldState {
+    pub complete: bool,
+    pub validated_size: Option<u64>,
+}
+
+impl Serialize for Bitfield {
+    fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        let mut numbers = SmallVec::<[_; 4]>::new();
+        numbers.push(self.size);
+        numbers.extend(self.ranges.boundaries().iter().map(|x| x.0));
+        numbers.serialize(serializer)
+    }
+}
+
+impl<'de> Deserialize<'de> for Bitfield {
+    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+        let mut numbers: SmallVec<[u64; 4]> = SmallVec::deserialize(deserializer)?;
+
+        // Need at least 1 u64 for size
+        if numbers.is_empty() {
+            return Err(serde::de::Error::custom("Bitfield needs at least size"));
+        }
+
+        // Split: first is size, rest are ranges
+        let size = numbers.remove(0);
+        let mut ranges = SmallVec::<[_; 2]>::new();
+        for num in numbers {
+            ranges.push(ChunkNum(num));
+        }
+        let Some(ranges) = ChunkRanges::new(ranges) else {
+            return Err(serde::de::Error::custom("Boundaries are not sorted"));
+        };
+        Ok(Bitfield::new(ranges, size))
+    }
+}
+
+impl Bitfield {
+    pub(crate) fn new_unchecked(ranges: ChunkRanges, size: u64) -> Self {
+        Self { ranges, size }
+    }
+
+    pub fn new(mut ranges: ChunkRanges, size: u64) -> Self {
+        // for zero size, we have to trust the caller
+        if let Some(size) = NonZeroU64::new(size) {
+            let end = ChunkNum::chunks(size.get());
+            if ChunkRanges::from(..end).is_subset(&ranges) {
+                // complete bitfield, canonicalize to all
+                ranges = ChunkRanges::all();
+            } else if ranges.contains(&(end - 1)) {
+                // validated bitfield, canonicalize to open end
+                ranges |= ChunkRanges::from(end..);
+            }
+        }
+        Self { ranges, size }
+    }
+
+    pub fn size(&self) -> u64 {
+        self.size
+    }
+
+    /// An empty bitfield. This is the neutral element for the combine operation.
+    pub fn empty() -> Self {
+        Self {
+            ranges: ChunkRanges::empty(),
+            size: 0,
+        }
+    }
+
+    /// Create a complete bitfield for the given size
+    pub fn complete(size: u64) -> Self {
+        Self {
+            ranges: ChunkRanges::all(),
+            size,
+        }
+    }
+
+    /// True if the chunk corresponding to the size is included in the ranges
+    pub fn is_validated(&self) -> bool {
+        if let Some(size) = NonZeroU64::new(self.size) {
+            is_validated(size, &self.ranges)
+        } else {
+            self.ranges.is_all()
+        }
+    }
+
+    /// True if all chunks up to size are included in the ranges
+    pub fn is_complete(&self) -> bool {
+        if let Some(size) = NonZeroU64::new(self.size) {
+            is_complete(size, &self.ranges)
+        } else {
+            self.ranges.is_all()
+        }
+    }
+
+    /// Get the validated size if the bitfield is validated
+    pub fn validated_size(&self) -> Option<u64> {
+        if self.is_validated() {
+            Some(self.size)
+        } else {
+            None
+        }
+    }
+
+    /// Total valid bytes in this bitfield.
+    pub fn total_bytes(&self) -> u64 {
+        let mut total = 0;
+        for range in self.ranges.iter() {
+            let (start, end) = match range {
+                RangeSetRange::Range(range) => {
+                    (range.start.to_bytes(), range.end.to_bytes().min(self.size))
+                }
+                RangeSetRange::RangeFrom(range) => (range.start.to_bytes(), self.size),
+            };
+            total += end - start;
+        }
+        total
+    }
+
+    pub fn state(&self) -> BitfieldState {
+        BitfieldState {
+            complete: self.is_complete(),
+            validated_size: self.validated_size(),
+        }
+    }
+
+    /// Update the bitfield with a new value, and gives detailed information about the change.
+    ///
+    /// returns a tuple of (changed, Some((old, new))). If the bitfield changed at all, the flag
+    /// is true. If there was a significant change, the old and new states are returned.
+    pub fn update(&mut self, update: &Bitfield) -> UpdateResult {
+        let s0 = self.state();
+        // todo: this is very inefficient because of all the clones
+        let bitfield1 = self.clone().combine(update.clone());
+        if bitfield1 != *self {
+            let s1 = bitfield1.state();
+            *self = bitfield1;
+            if s0 != s1 {
+                UpdateResult::MajorChange(s0, s1)
+            } else {
+                UpdateResult::MinorChange(s1)
+            }
+        } else {
+            UpdateResult::NoChange(s0)
+        }
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.ranges.is_empty() && self.size == 0
+    }
+
+    /// A diff between two bitfields. This is an inverse of the combine operation.
+    pub fn diff(&self, that: &Bitfield) -> Self {
+        let size = choose_size(self, that);
+        let ranges_diff = &self.ranges ^ &that.ranges;
+        Self {
+            ranges: ranges_diff,
+            size,
+        }
+    }
+}
+
+pub fn choose_size(a: &Bitfield, b: &Bitfield) -> u64 {
+    match (a.ranges.upper_bound(), b.ranges.upper_bound()) {
+        (Some(ac), Some(bc)) => match ac.cmp(&bc) {
+            Ordering::Less => b.size,
+            Ordering::Greater => a.size,
+            Ordering::Equal => a.size.max(b.size),
+        },
+        (Some(_), None) => b.size,
+        (None, Some(_)) => a.size,
+        (None, None) => a.size.max(b.size),
+    }
+}
+
+impl Combine for Bitfield {
+    fn combine(self, that: Self) -> Self {
+        // the size of the chunk with the larger last chunk wins
+        let size = choose_size(&self, &that);
+        let ranges = self.ranges | that.ranges;
+        Self::new(ranges, size)
+    }
+}
+
+impl CombineInPlace for Bitfield {
+    fn combine_with(&mut self, other: Self) -> Self {
+        let new = &other.ranges - &self.ranges;
+        if new.is_empty() {
+            return Bitfield::empty();
+        }
+        self.ranges.union_with(&new);
+        self.size = choose_size(self, &other);
+        Bitfield {
+            ranges: new,
+            size: self.size,
+        }
+    }
+
+    fn is_neutral(&self) -> bool {
+        self.ranges.is_empty() && self.size == 0
+    }
+}
+
+#[allow(clippy::enum_variant_names)]
+#[derive(Debug, Clone, Copy)]
+pub enum UpdateResult {
+    NoChange(BitfieldState),
+    MinorChange(BitfieldState),
+    MajorChange(BitfieldState, BitfieldState),
+}
+
+impl UpdateResult {
+    pub fn new_state(&self) -> &BitfieldState {
+        match self {
+            UpdateResult::NoChange(new) => new,
+            UpdateResult::MinorChange(new) => new,
+            UpdateResult::MajorChange(_, new) => new,
+        }
+    }
+
+    /// True if this change went from non-validated to validated
+    pub fn was_validated(&self) -> bool {
+        match self {
+            UpdateResult::NoChange(_) => false,
+            UpdateResult::MinorChange(_) => false,
+            UpdateResult::MajorChange(old, new) => {
+                new.validated_size.is_some() && old.validated_size.is_none()
+            }
+        }
+    }
+
+    pub fn changed(&self) -> bool {
+        match self {
+            UpdateResult::NoChange(_) => false,
+            UpdateResult::MinorChange(_) => true,
+            UpdateResult::MajorChange(_, _) => true,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use bao_tree::{ChunkNum, ChunkRanges};
+    use proptest::prelude::{prop, Strategy};
+    use smallvec::SmallVec;
+    use test_strategy::proptest;
+
+    use super::Bitfield;
+    use crate::store::util::observer::{Combine, CombineInPlace};
+
+    fn gen_chunk_ranges(max: ChunkNum, k: usize) -> impl Strategy<Value = ChunkRanges> {
+        prop::collection::btree_set(0..=max.0, 0..=k).prop_map(|vec| {
+            let bounds = vec.into_iter().map(ChunkNum).collect::<SmallVec<[_; 2]>>();
+            ChunkRanges::new(bounds).unwrap()
+        })
+    }
+
+    fn gen_bitfields(size: u64, k: usize) -> impl Strategy<Value = Bitfield> {
+        (0..size).prop_flat_map(move |size| {
+            let chunks = ChunkNum::full_chunks(size);
+            gen_chunk_ranges(chunks, k).prop_map(move |ranges| Bitfield::new(ranges, size))
+        })
+    }
+
+    fn gen_non_empty_bitfields(size: u64, k: usize) -> impl Strategy<Value = Bitfield> {
+        gen_bitfields(size, k).prop_filter("non-empty", |x| !x.is_neutral())
+    }
+
+    #[proptest]
+    fn test_combine_empty(#[strategy(gen_non_empty_bitfields(32768, 4))] a: Bitfield) {
+        assert_eq!(a.clone().combine(Bitfield::empty()), a);
+        assert_eq!(Bitfield::empty().combine(a.clone()), a);
+    }
+
+    #[proptest]
+    fn test_combine_order(
+        #[strategy(gen_non_empty_bitfields(32768, 4))] a: Bitfield,
+        #[strategy(gen_non_empty_bitfields(32768, 4))] b: Bitfield,
+    ) {
+        let ab = a.clone().combine(b.clone());
+        let ba = b.combine(a);
+        assert_eq!(ab, ba);
+    }
+
+    #[proptest]
+    fn test_complete_normalized(#[strategy(gen_non_empty_bitfields(32768, 4))] a: Bitfield) {
+        if a.is_complete() {
+            assert_eq!(a.ranges, ChunkRanges::all());
+        }
+    }
+}
diff --git a/src/api/remote.rs b/src/api/remote.rs
new file mode 100644
index 000000000..75e66d0e3
--- /dev/null
+++ b/src/api/remote.rs
@@ -0,0 +1,1270 @@
+//! API for downloading blobs from a single remote node.
+//!
+//! The entry point is the [`Remote`] struct.
+use genawaiter::sync::{Co, Gen};
+use iroh::endpoint::SendStream;
+use irpc::util::{AsyncReadVarintExt, WriteVarintExt};
+use n0_future::{io, Stream, StreamExt};
+use n0_snafu::SpanTrace;
+use nested_enum_utils::common_fields;
+use ref_cast::RefCast;
+use snafu::{Backtrace, IntoError, Snafu};
+
+use super::blobs::Bitfield;
+use crate::{
+    api::{blobs::WriteProgress, ApiClient},
+    get::{fsm::DecodeError, BadRequestSnafu, GetError, GetResult, LocalFailureSnafu, Stats},
+    protocol::{
+        GetManyRequest, ObserveItem, ObserveRequest, PushRequest, Request, RequestType,
+        MAX_MESSAGE_SIZE,
+    },
+    util::sink::{Sink, TokioMpscSenderSink},
+};
+
+/// API to compute request and to download from remote nodes.
+///
+/// Usually you want to first find out what, if any, data you have locally.
+/// This can be done using [`Remote::local`], which inspects the local store
+/// and returns a [`LocalInfo`].
+///
+/// From this you can compute various values such as the number of locally present
+/// bytes. You can also compute a request to get the missing data using [`LocalInfo::missing`].
+///
+/// Once you have a request, you can execute it using [`Remote::execute_get`].
+/// Executing a request will store to the local store, but otherwise does not take
+/// the available data into account.
+///
+/// If you are not interested in the details and just want your data, you can use
+/// [`Remote::fetch`]. This will internally do the dance described above.
+#[derive(Debug, Clone, RefCast)]
+#[repr(transparent)]
+pub struct Remote {
+    client: ApiClient,
+}
+
+#[derive(Debug)]
+pub enum GetProgressItem {
+    /// Progress on the payload bytes read.
+    Progress(u64),
+    /// The request was completed.
+    Done(Stats),
+    /// The request was closed, but not completed.
+    Error(GetError),
+}
+
+impl From<GetResult<Stats>> for GetProgressItem {
+    fn from(res: GetResult<Stats>) -> Self {
+        match res {
+            Ok(stats) => GetProgressItem::Done(stats),
+            Err(e) => GetProgressItem::Error(e),
+        }
+    }
+}
+
+impl TryFrom<GetProgressItem> for GetResult<Stats> {
+    type Error = &'static str;
+
+    fn try_from(item: GetProgressItem) -> Result<Self, Self::Error> {
+        match item {
+            GetProgressItem::Done(stats) => Ok(Ok(stats)),
+            GetProgressItem::Error(e) => Ok(Err(e)),
+            GetProgressItem::Progress(_) => Err("not a final item"),
+        }
+    }
+}
+
+pub struct GetProgress {
+    rx: tokio::sync::mpsc::Receiver<GetProgressItem>,
+    fut: n0_future::boxed::BoxFuture<()>,
+}
+
+impl IntoFuture for GetProgress {
+    type Output = GetResult<Stats>;
+    type IntoFuture = n0_future::boxed::BoxFuture<Self::Output>;
+
+    fn into_future(self) -> n0_future::boxed::BoxFuture<Self::Output> {
+        Box::pin(self.complete())
+    }
+}
+
+impl GetProgress {
+    pub fn stream(self) -> impl Stream<Item = GetProgressItem> {
+        into_stream(self.rx, self.fut)
+    }
+
+    pub async fn complete(self) -> GetResult<Stats> {
+        just_result(self.stream()).await.unwrap_or_else(|| {
+            Err(LocalFailureSnafu
+                .into_error(anyhow::anyhow!("stream closed without result").into()))
+        })
+    }
+}
+
+#[derive(Debug)]
+pub enum PushProgressItem {
+    /// Progress on the payload bytes read.
+    Progress(u64),
+    /// The request was completed.
+    Done(Stats),
+    /// The request was closed, but not completed.
+    Error(anyhow::Error),
+}
+
+impl From<anyhow::Result<Stats>> for PushProgressItem {
+    fn from(res: anyhow::Result<Stats>) -> Self {
+        match res {
+            Ok(stats) => Self::Done(stats),
+            Err(e) => Self::Error(e),
+        }
+    }
+}
+
+impl TryFrom<PushProgressItem> for anyhow::Result<Stats> {
+    type Error = &'static str;
+
+    fn try_from(item: PushProgressItem) -> Result<Self, Self::Error> {
+        match item {
+            PushProgressItem::Done(stats) => Ok(Ok(stats)),
+            PushProgressItem::Error(e) => Ok(Err(e)),
+            PushProgressItem::Progress(_) => Err("not a final item"),
+        }
+    }
+}
+
+pub struct PushProgress {
+    rx: tokio::sync::mpsc::Receiver<PushProgressItem>,
+    fut: n0_future::boxed::BoxFuture<()>,
+}
+
+impl IntoFuture for PushProgress {
+    type Output = anyhow::Result<Stats>;
+    type IntoFuture = n0_future::boxed::BoxFuture<Self::Output>;
+
+    fn into_future(self) -> n0_future::boxed::BoxFuture<Self::Output> {
+        Box::pin(self.complete())
+    }
+}
+
+impl PushProgress {
+    pub fn stream(self) -> impl Stream<Item = PushProgressItem> {
+        into_stream(self.rx, self.fut)
+    }
+
+    pub async fn complete(self) -> anyhow::Result<Stats> {
+        just_result(self.stream())
+            .await
+            .unwrap_or_else(|| Err(anyhow::anyhow!("stream closed without result")))
+    }
+}
+
+async fn just_result<S, R>(stream: S) -> Option<R>
+where
+    S: Stream,
+    R: TryFrom<S::Item>,
+{
+    tokio::pin!(stream);
+    while let Some(item) = stream.next().await {
+        if let Ok(res) = R::try_from(item) {
+            return Some(res);
+        }
+    }
+    None
+}
+
+fn into_stream<T, F>(mut rx: tokio::sync::mpsc::Receiver<T>, fut: F) -> impl Stream<Item = T>
+where
+    F: Future,
+{
+    Gen::new(move |co| async move {
+        tokio::pin!(fut);
+        loop {
+            tokio::select! {
+                biased;
+                item = rx.recv() => {
+                    if let Some(item) = item {
+                        co.yield_(item).await;
+                    } else {
+                        break;
+                    }
+                }
+                _ = &mut fut => {
+                    break;
+                }
+            }
+        }
+        while let Some(item) = rx.recv().await {
+            co.yield_(item).await;
+        }
+    })
+}
+
+/// Local info for a blob or hash sequence.
+///
+/// This can be used to get the amount of missing data, and to construct a
+/// request to get the missing data.
+#[derive(Debug)]
+pub struct LocalInfo {
+    /// The hash for which this is the local info
+    request: Arc<GetRequest>,
+    /// The bitfield for the root hash
+    bitfield: Bitfield,
+    /// Optional - the hash sequence info if this was a request for a hash sequence
+    children: Option<NonRawLocalInfo>,
+}
+
+impl LocalInfo {
+    /// The number of bytes we have locally
+    pub fn local_bytes(&self) -> u64 {
+        let Some(root_requested) = self.requested_root_ranges() else {
+            // empty request requests 0 bytes
+            return 0;
+        };
+        let mut local = self.bitfield.clone();
+        local.ranges.intersection_with(root_requested);
+        let mut res = local.total_bytes();
+        if let Some(children) = &self.children {
+            let Some(max_local_index) = children.hash_seq.keys().next_back() else {
+                // no children
+                return res;
+            };
+            for (offset, ranges) in self.request.ranges.iter_non_empty_infinite() {
+                if offset == 0 {
+                    // skip the root hash
+                    continue;
+                }
+                let child = offset - 1;
+                if child > *max_local_index {
+                    // we are done
+                    break;
+                }
+                let Some(hash) = children.hash_seq.get(&child) else {
+                    continue;
+                };
+                let bitfield = &children.bitfields[hash];
+                let mut local = bitfield.clone();
+                local.ranges.intersection_with(ranges);
+                res += local.total_bytes();
+            }
+        }
+        res
+    }
+
+    /// Number of children in this hash sequence
+    pub fn children(&self) -> Option<u64> {
+        if self.children.is_some() {
+            self.bitfield.validated_size().map(|x| x / 32)
+        } else {
+            Some(0)
+        }
+    }
+
+    /// The requested root ranges.
+    ///
+    /// This will return None if the request is empty, and an empty CHunkRanges
+    /// if no ranges were requested for the root hash.
+    fn requested_root_ranges(&self) -> Option<&ChunkRanges> {
+        self.request.ranges.iter().next()
+    }
+
+    /// True if the data is complete.
+    ///
+    /// For a blob, this is true if the blob is complete.
+    /// For a hash sequence, this is true if the hash sequence is complete and
+    /// all its children are complete.
+    pub fn is_complete(&self) -> bool {
+        let Some(root_requested) = self.requested_root_ranges() else {
+            // empty request is complete
+            return true;
+        };
+        if !self.bitfield.ranges.is_superset(root_requested) {
+            return false;
+        }
+        if let Some(children) = self.children.as_ref() {
+            let mut iter = self.request.ranges.iter_non_empty_infinite();
+            let max_child = self.bitfield.validated_size().map(|x| x / 32);
+            loop {
+                let Some((offset, range)) = iter.next() else {
+                    break;
+                };
+                if offset == 0 {
+                    // skip the root hash
+                    continue;
+                }
+                let child = offset - 1;
+                if let Some(hash) = children.hash_seq.get(&child) {
+                    let bitfield = &children.bitfields[hash];
+                    if !bitfield.ranges.is_superset(range) {
+                        // we don't have the requested ranges
+                        return false;
+                    }
+                } else {
+                    if let Some(max_child) = max_child {
+                        if child >= max_child {
+                            // reading after the end of the request
+                            return true;
+                        }
+                    }
+                    return false;
+                }
+            }
+        }
+        true
+    }
+
+    /// A request to get the missing data to complete this request
+    pub fn missing(&self) -> GetRequest {
+        let Some(root_requested) = self.requested_root_ranges() else {
+            // empty request is complete
+            return GetRequest::new(self.request.hash, ChunkRangesSeq::empty());
+        };
+        let mut builder = GetRequest::builder().root(root_requested - &self.bitfield.ranges);
+
+        let Some(children) = self.children.as_ref() else {
+            return builder.build(self.request.hash);
+        };
+        let mut iter = self.request.ranges.iter_non_empty_infinite();
+        let max_local = children
+            .hash_seq
+            .keys()
+            .next_back()
+            .map(|x| *x + 1)
+            .unwrap_or_default();
+        let max_offset = self.bitfield.validated_size().map(|x| x / 32);
+        loop {
+            let Some((offset, requested)) = iter.next() else {
+                break;
+            };
+            if offset == 0 {
+                // skip the root hash
+                continue;
+            }
+            let child = offset - 1;
+            let missing = match children.hash_seq.get(&child) {
+                Some(hash) => requested.difference(&children.bitfields[hash].ranges),
+                None => requested.clone(),
+            };
+            builder = builder.child(child, missing);
+            if offset >= max_local {
+                // we can't do anything clever anymore
+                break;
+            }
+        }
+        loop {
+            let Some((offset, requested)) = iter.next() else {
+                return builder.build(self.request.hash);
+            };
+            if offset == 0 {
+                // skip the root hash
+                continue;
+            }
+            let child = offset - 1;
+            if let Some(max_offset) = &max_offset {
+                if child >= *max_offset {
+                    return builder.build(self.request.hash);
+                }
+                builder = builder.child(child, requested.clone());
+            } else {
+                builder = builder.child(child, requested.clone());
+                if iter.is_at_end() {
+                    if iter.next().is_none() {
+                        return builder.build(self.request.hash);
+                    } else {
+                        return builder.build_open(self.request.hash);
+                    }
+                }
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+struct NonRawLocalInfo {
+    /// the available and relevant part of the hash sequence
+    hash_seq: BTreeMap<u64, Hash>,
+    /// For each hash relevant to the request, the local bitfield and the ranges
+    /// that were requested.
+    bitfields: BTreeMap<Hash, Bitfield>,
+}
+
+// fn iter_without_gaps<'a, T: Copy + 'a>(
+//     iter: impl IntoIterator<Item = &'a (u64, T)> + 'a,
+// ) -> impl Iterator<Item = (u64, Option<T>)> + 'a {
+//     let mut prev = 0;
+//     iter.into_iter().flat_map(move |(i, hash)| {
+//         let start = prev + 1;
+//         let curr = *i;
+//         prev = *i;
+//         (start..curr)
+//             .map(|i| (i, None))
+//             .chain(std::iter::once((curr, Some(*hash))))
+//     })
+// }
+
+impl Remote {
+    pub(crate) fn ref_from_sender(sender: &ApiClient) -> &Self {
+        Self::ref_cast(sender)
+    }
+
+    fn store(&self) -> &Store {
+        Store::ref_from_sender(&self.client)
+    }
+
+    pub async fn local_for_request(
+        &self,
+        request: impl Into<Arc<GetRequest>>,
+    ) -> anyhow::Result<LocalInfo> {
+        let request = request.into();
+        let root = request.hash;
+        let bitfield = self.store().observe(root).await?;
+        let children = if !request.ranges.is_blob() {
+            let bao = self.store().export_bao(root, bitfield.ranges.clone());
+            let mut by_index = BTreeMap::new();
+            let mut stream = bao.hashes_with_index();
+            while let Some(item) = stream.next().await {
+                let (index, hash) = item?;
+                by_index.insert(index, hash);
+            }
+            let mut bitfields = BTreeMap::new();
+            let mut hash_seq = BTreeMap::new();
+            let max = by_index.last_key_value().map(|(k, _)| *k + 1).unwrap_or(0);
+            for (index, _) in request.ranges.iter_non_empty_infinite() {
+                if index == 0 {
+                    // skip the root hash
+                    continue;
+                }
+                let child = index - 1;
+                if child > max {
+                    // we are done
+                    break;
+                }
+                let Some(hash) = by_index.get(&child) else {
+                    // we don't have the hash, so we can't store the bitfield
+                    continue;
+                };
+                let bitfield = self.store().observe(*hash).await?;
+                bitfields.insert(*hash, bitfield);
+                hash_seq.insert(child, *hash);
+            }
+            Some(NonRawLocalInfo {
+                hash_seq,
+                bitfields,
+            })
+        } else {
+            None
+        };
+        Ok(LocalInfo {
+            request: request.clone(),
+            bitfield,
+            children,
+        })
+    }
+
+    /// Get the local info for a given blob or hash sequence, at the present time.
+    pub async fn local(&self, content: impl Into<HashAndFormat>) -> anyhow::Result<LocalInfo> {
+        let request = GetRequest::from(content.into());
+        self.local_for_request(request).await
+    }
+
+    pub fn fetch(
+        &self,
+        conn: impl GetConnection + Send + 'static,
+        content: impl Into<HashAndFormat>,
+    ) -> GetProgress {
+        let content = content.into();
+        let (tx, rx) = tokio::sync::mpsc::channel(64);
+        let tx2 = tx.clone();
+        let sink = TokioMpscSenderSink(tx)
+            .with_map(GetProgressItem::Progress)
+            .with_map_err(io::Error::other);
+        let this = self.clone();
+        let fut = async move {
+            let res = this.fetch_sink(conn, content, sink).await.into();
+            tx2.send(res).await.ok();
+        };
+        GetProgress {
+            rx,
+            fut: Box::pin(fut),
+        }
+    }
+
+    /// Get a blob or hash sequence from the given connection, taking the locally available
+    /// ranges into account.
+    ///
+    /// You can provide a progress channel to get updates on the download progress. This progress
+    /// is the aggregated number of downloaded payload bytes in the request.
+    ///
+    /// This will return the stats of the download.
+    pub async fn fetch_sink(
+        &self,
+        mut conn: impl GetConnection,
+        content: impl Into<HashAndFormat>,
+        progress: impl Sink<u64, Error = io::Error>,
+    ) -> GetResult<Stats> {
+        let content = content.into();
+        let local = self
+            .local(content)
+            .await
+            .map_err(|e| LocalFailureSnafu.into_error(e.into()))?;
+        if local.is_complete() {
+            return Ok(Default::default());
+        }
+        let request = local.missing();
+        let conn = conn
+            .connection()
+            .await
+            .map_err(|e| LocalFailureSnafu.into_error(e.into()))?;
+        let stats = self.execute_get_sink(conn, request, progress).await?;
+        Ok(stats)
+    }
+
+    pub fn observe(
+        &self,
+        conn: Connection,
+        request: ObserveRequest,
+    ) -> impl Stream<Item = io::Result<Bitfield>> + 'static {
+        Gen::new(|co| async move {
+            if let Err(cause) = Self::observe_impl(conn, request, &co).await {
+                co.yield_(Err(cause)).await
+            }
+        })
+    }
+
+    async fn observe_impl(
+        conn: Connection,
+        request: ObserveRequest,
+        co: &Co<io::Result<Bitfield>>,
+    ) -> io::Result<()> {
+        let hash = request.hash;
+        debug!(%hash, "observing");
+        let (mut send, mut recv) = conn.open_bi().await?;
+        // write the request. Unlike for reading, we can just serialize it sync using postcard.
+        write_observe_request(request, &mut send).await?;
+        send.finish()?;
+        loop {
+            let msg = recv
+                .read_length_prefixed::<ObserveItem>(MAX_MESSAGE_SIZE)
+                .await?;
+            co.yield_(Ok(Bitfield::from(&msg))).await;
+        }
+    }
+
+    pub fn execute_push(&self, conn: Connection, request: PushRequest) -> PushProgress {
+        let (tx, rx) = tokio::sync::mpsc::channel(64);
+        let tx2 = tx.clone();
+        let sink = TokioMpscSenderSink(tx)
+            .with_map(PushProgressItem::Progress)
+            .with_map_err(io::Error::other);
+        let this = self.clone();
+        let fut = async move {
+            let res = this.execute_push_sink(conn, request, sink).await.into();
+            tx2.send(res).await.ok();
+        };
+        PushProgress {
+            rx,
+            fut: Box::pin(fut),
+        }
+    }
+
+    /// Push the given blob or hash sequence to a remote node.
+    ///
+    /// Note that many nodes will reject push requests. Also, this is an experimental feature for now.
+    pub async fn execute_push_sink(
+        &self,
+        conn: Connection,
+        request: PushRequest,
+        progress: impl Sink<u64, Error = io::Error>,
+    ) -> anyhow::Result<Stats> {
+        let hash = request.hash;
+        debug!(%hash, "pushing");
+        let (mut send, mut recv) = conn.open_bi().await?;
+        let mut context = StreamContext {
+            payload_bytes_sent: 0,
+            sender: progress,
+        };
+        // we are not going to need this!
+        recv.stop(0u32.into())?;
+        // write the request. Unlike for reading, we can just serialize it sync using postcard.
+        let request = write_push_request(request, &mut send).await?;
+        let mut request_ranges = request.ranges.iter_infinite();
+        let root = request.hash;
+        let root_ranges = request_ranges.next().expect("infinite iterator");
+        if !root_ranges.is_empty() {
+            self.store()
+                .export_bao(root, root_ranges.clone())
+                .write_quinn_with_progress(&mut send, &mut context, &root, 0)
+                .await?;
+        }
+        if request.ranges.is_blob() {
+            // we are done
+            send.finish()?;
+            return Ok(Default::default());
+        }
+        let hash_seq = self.store().get_bytes(root).await?;
+        let hash_seq = HashSeq::try_from(hash_seq)?;
+        for (child, (child_hash, child_ranges)) in
+            hash_seq.into_iter().zip(request_ranges).enumerate()
+        {
+            if !child_ranges.is_empty() {
+                self.store()
+                    .export_bao(child_hash, child_ranges.clone())
+                    .write_quinn_with_progress(
+                        &mut send,
+                        &mut context,
+                        &child_hash,
+                        (child + 1) as u64,
+                    )
+                    .await?;
+            }
+        }
+        send.finish()?;
+        Ok(Default::default())
+    }
+
+    pub fn execute_get(&self, conn: Connection, request: GetRequest) -> GetProgress {
+        self.execute_get_with_opts(conn, request)
+    }
+
+    pub fn execute_get_with_opts(&self, conn: Connection, request: GetRequest) -> GetProgress {
+        let (tx, rx) = tokio::sync::mpsc::channel(64);
+        let tx2 = tx.clone();
+        let sink = TokioMpscSenderSink(tx)
+            .with_map(GetProgressItem::Progress)
+            .with_map_err(io::Error::other);
+        let this = self.clone();
+        let fut = async move {
+            let res = this.execute_get_sink(conn, request, sink).await.into();
+            tx2.send(res).await.ok();
+        };
+        GetProgress {
+            rx,
+            fut: Box::pin(fut),
+        }
+    }
+
+    /// Execute a get request *without* taking the locally available ranges into account.
+    ///
+    /// You can provide a progress channel to get updates on the download progress. This progress
+    /// is the aggregated number of downloaded payload bytes in the request.
+    ///
+    /// This will download the data again even if the data is locally present.
+    ///
+    /// This will return the stats of the download.
+    pub async fn execute_get_sink(
+        &self,
+        conn: Connection,
+        request: GetRequest,
+        mut progress: impl Sink<u64, Error = io::Error>,
+    ) -> GetResult<Stats> {
+        let store = self.store();
+        let root = request.hash;
+        let start = crate::get::fsm::start(conn, request, Default::default());
+        let connected = start.next().await?;
+        trace!("Getting header");
+        // read the header
+        let next_child = match connected.next().await? {
+            ConnectedNext::StartRoot(at_start_root) => {
+                let header = at_start_root.next();
+                let end = get_blob_ranges_impl(header, root, store, &mut progress).await?;
+                match end.next() {
+                    EndBlobNext::MoreChildren(at_start_child) => Ok(at_start_child),
+                    EndBlobNext::Closing(at_closing) => Err(at_closing),
+                }
+            }
+            ConnectedNext::StartChild(at_start_child) => Ok(at_start_child),
+            ConnectedNext::Closing(at_closing) => Err(at_closing),
+        };
+        // read the rest, if any
+        let at_closing = match next_child {
+            Ok(at_start_child) => {
+                let mut next_child = Ok(at_start_child);
+                let hash_seq = HashSeq::try_from(
+                    store
+                        .get_bytes(root)
+                        .await
+                        .map_err(|e| LocalFailureSnafu.into_error(e.into()))?,
+                )
+                .map_err(|source| BadRequestSnafu.into_error(source.into()))?;
+                // let mut hash_seq = LazyHashSeq::new(store.blobs().clone(), root);
+                loop {
+                    let at_start_child = match next_child {
+                        Ok(at_start_child) => at_start_child,
+                        Err(at_closing) => break at_closing,
+                    };
+                    let offset = at_start_child.offset() - 1;
+                    let Some(hash) = hash_seq.get(offset as usize) else {
+                        break at_start_child.finish();
+                    };
+                    trace!("getting child {offset} {}", hash.fmt_short());
+                    let header = at_start_child.next(hash);
+                    let end = get_blob_ranges_impl(header, hash, store, &mut progress).await?;
+                    next_child = match end.next() {
+                        EndBlobNext::MoreChildren(at_start_child) => Ok(at_start_child),
+                        EndBlobNext::Closing(at_closing) => Err(at_closing),
+                    }
+                }
+            }
+            Err(at_closing) => at_closing,
+        };
+        // read the rest, if any
+        let stats = at_closing.next().await?;
+        trace!(?stats, "get hash seq done");
+        Ok(stats)
+    }
+
+    pub fn execute_get_many(&self, conn: Connection, request: GetManyRequest) -> GetProgress {
+        let (tx, rx) = tokio::sync::mpsc::channel(64);
+        let tx2 = tx.clone();
+        let sink = TokioMpscSenderSink(tx)
+            .with_map(GetProgressItem::Progress)
+            .with_map_err(io::Error::other);
+        let this = self.clone();
+        let fut = async move {
+            let res = this.execute_get_many_sink(conn, request, sink).await.into();
+            tx2.send(res).await.ok();
+        };
+        GetProgress {
+            rx,
+            fut: Box::pin(fut),
+        }
+    }
+
+    /// Execute a get request *without* taking the locally available ranges into account.
+    ///
+    /// You can provide a progress channel to get updates on the download progress. This progress
+    /// is the aggregated number of downloaded payload bytes in the request.
+    ///
+    /// This will download the data again even if the data is locally present.
+    ///
+    /// This will return the stats of the download.
+    pub async fn execute_get_many_sink(
+        &self,
+        conn: Connection,
+        request: GetManyRequest,
+        mut progress: impl Sink<u64, Error = io::Error>,
+    ) -> GetResult<Stats> {
+        let store = self.store();
+        let hash_seq = request.hashes.iter().copied().collect::<HashSeq>();
+        let next_child = crate::get::fsm::start_get_many(conn, request, Default::default()).await?;
+        // read all children.
+        let at_closing = match next_child {
+            Ok(at_start_child) => {
+                let mut next_child = Ok(at_start_child);
+                loop {
+                    let at_start_child = match next_child {
+                        Ok(at_start_child) => at_start_child,
+                        Err(at_closing) => break at_closing,
+                    };
+                    let offset = at_start_child.offset();
+                    println!("offset {offset}");
+                    let Some(hash) = hash_seq.get(offset as usize) else {
+                        break at_start_child.finish();
+                    };
+                    trace!("getting child {offset} {}", hash.fmt_short());
+                    let header = at_start_child.next(hash);
+                    let end = get_blob_ranges_impl(header, hash, store, &mut progress).await?;
+                    next_child = match end.next() {
+                        EndBlobNext::MoreChildren(at_start_child) => Ok(at_start_child),
+                        EndBlobNext::Closing(at_closing) => Err(at_closing),
+                    }
+                }
+            }
+            Err(at_closing) => at_closing,
+        };
+        // read the rest, if any
+        let stats = at_closing.next().await?;
+        trace!(?stats, "get hash seq done");
+        Ok(stats)
+    }
+}
+
+/// Failures for a get operation
+#[common_fields({
+    backtrace: Option<Backtrace>,
+    #[snafu(implicit)]
+    span_trace: SpanTrace,
+})]
+#[allow(missing_docs)]
+#[non_exhaustive]
+#[derive(Debug, Snafu)]
+pub enum ExecuteError {
+    /// Network or IO operation failed.
+    #[snafu(display("Unable to open bidi stream"))]
+    Connection {
+        source: iroh::endpoint::ConnectionError,
+    },
+    #[snafu(display("Unable to read from the remote"))]
+    Read { source: iroh::endpoint::ReadError },
+    #[snafu(display("Error sending the request"))]
+    Send {
+        source: crate::get::fsm::ConnectedNextError,
+    },
+    #[snafu(display("Unable to read size"))]
+    Size {
+        source: crate::get::fsm::AtBlobHeaderNextError,
+    },
+    #[snafu(display("Error while decoding the data"))]
+    Decode {
+        source: crate::get::fsm::DecodeError,
+    },
+    #[snafu(display("Internal error while reading the hash sequence"))]
+    ExportBao { source: api::ExportBaoError },
+    #[snafu(display("Hash sequence has an invalid length"))]
+    InvalidHashSeq { source: anyhow::Error },
+    #[snafu(display("Internal error importing the data"))]
+    ImportBao { source: crate::api::RequestError },
+    #[snafu(display("Error sending download progress - receiver closed"))]
+    SendDownloadProgress { source: irpc::channel::SendError },
+    #[snafu(display("Internal error importing the data"))]
+    MpscSend {
+        source: tokio::sync::mpsc::error::SendError<BaoContentItem>,
+    },
+}
+
+use std::{
+    collections::BTreeMap,
+    future::{Future, IntoFuture},
+    num::NonZeroU64,
+    sync::Arc,
+};
+
+use bao_tree::{
+    io::{BaoContentItem, Leaf},
+    ChunkNum, ChunkRanges,
+};
+use iroh::endpoint::Connection;
+use tracing::{debug, trace};
+
+use crate::{
+    api::{self, blobs::Blobs, Store},
+    get::fsm::{AtBlobHeader, AtEndBlob, BlobContentNext, ConnectedNext, EndBlobNext},
+    hashseq::{HashSeq, HashSeqIter},
+    protocol::{ChunkRangesSeq, GetRequest},
+    store::IROH_BLOCK_SIZE,
+    Hash, HashAndFormat,
+};
+
+/// Trait to lazily get a connection
+pub trait GetConnection {
+    fn connection(&mut self)
+        -> impl Future<Output = Result<Connection, anyhow::Error>> + Send + '_;
+}
+
+/// If we already have a connection, the impl is trivial
+impl GetConnection for Connection {
+    fn connection(
+        &mut self,
+    ) -> impl Future<Output = Result<Connection, anyhow::Error>> + Send + '_ {
+        let conn = self.clone();
+        async { Ok(conn) }
+    }
+}
+
+/// If we already have a connection, the impl is trivial
+impl GetConnection for &Connection {
+    fn connection(
+        &mut self,
+    ) -> impl Future<Output = Result<Connection, anyhow::Error>> + Send + '_ {
+        let conn = self.clone();
+        async { Ok(conn) }
+    }
+}
+
+fn get_buffer_size(size: NonZeroU64) -> usize {
+    (size.get() / (IROH_BLOCK_SIZE.bytes() as u64) + 2).min(64) as usize
+}
+
+async fn get_blob_ranges_impl(
+    header: AtBlobHeader,
+    hash: Hash,
+    store: &Store,
+    mut progress: impl Sink<u64, Error = io::Error>,
+) -> GetResult<AtEndBlob> {
+    let (mut content, size) = header.next().await?;
+    let Some(size) = NonZeroU64::new(size) else {
+        return if hash == Hash::EMPTY {
+            let end = content.drain().await?;
+            Ok(end)
+        } else {
+            Err(DecodeError::leaf_hash_mismatch(ChunkNum(0)).into())
+        };
+    };
+    let buffer_size = get_buffer_size(size);
+    trace!(%size, %buffer_size, "get blob");
+    let handle = store
+        .import_bao(hash, size, buffer_size)
+        .await
+        .map_err(|e| LocalFailureSnafu.into_error(e.into()))?;
+    let write = async move {
+        GetResult::Ok(loop {
+            match content.next().await {
+                BlobContentNext::More((next, res)) => {
+                    let item = res?;
+                    progress
+                        .send(next.stats().payload_bytes_read)
+                        .await
+                        .map_err(|e| LocalFailureSnafu.into_error(e.into()))?;
+                    handle.tx.send(item).await?;
+                    content = next;
+                }
+                BlobContentNext::Done(end) => {
+                    drop(handle.tx);
+                    break end;
+                }
+            }
+        })
+    };
+    let complete = async move {
+        handle.rx.await.map_err(|e| {
+            LocalFailureSnafu
+                .into_error(anyhow::anyhow!("error reading from import stream: {e}").into())
+        })
+    };
+    let (_, end) = tokio::try_join!(complete, write)?;
+    Ok(end)
+}
+
+#[derive(Debug)]
+pub(crate) struct LazyHashSeq {
+    blobs: Blobs,
+    hash: Hash,
+    current_chunk: Option<HashSeqChunk>,
+}
+
+#[derive(Debug)]
+pub(crate) struct HashSeqChunk {
+    /// the offset of the first hash in this chunk, in bytes
+    offset: u64,
+    /// the hashes in this chunk
+    chunk: HashSeq,
+}
+
+impl TryFrom<Leaf> for HashSeqChunk {
+    type Error = anyhow::Error;
+
+    fn try_from(leaf: Leaf) -> Result<Self, Self::Error> {
+        let offset = leaf.offset;
+        let chunk = HashSeq::try_from(leaf.data)?;
+        Ok(Self { offset, chunk })
+    }
+}
+
+impl IntoIterator for HashSeqChunk {
+    type Item = Hash;
+    type IntoIter = HashSeqIter;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.chunk.into_iter()
+    }
+}
+
+impl HashSeqChunk {
+    pub fn base(&self) -> u64 {
+        self.offset / 32
+    }
+
+    #[allow(dead_code)]
+    fn get(&self, offset: u64) -> Option<Hash> {
+        let start = self.offset;
+        let end = start + self.chunk.len() as u64;
+        if offset >= start && offset < end {
+            let o = (offset - start) as usize;
+            self.chunk.get(o)
+        } else {
+            None
+        }
+    }
+}
+
+impl LazyHashSeq {
+    #[allow(dead_code)]
+    pub fn new(blobs: Blobs, hash: Hash) -> Self {
+        Self {
+            blobs,
+            hash,
+            current_chunk: None,
+        }
+    }
+
+    #[allow(dead_code)]
+    pub async fn get_from_offset(&mut self, offset: u64) -> anyhow::Result<Option<Hash>> {
+        if offset == 0 {
+            Ok(Some(self.hash))
+        } else {
+            self.get(offset - 1).await
+        }
+    }
+
+    #[allow(dead_code)]
+    pub async fn get(&mut self, child_offset: u64) -> anyhow::Result<Option<Hash>> {
+        // check if we have the hash in the current chunk
+        if let Some(chunk) = &self.current_chunk {
+            if let Some(hash) = chunk.get(child_offset) {
+                return Ok(Some(hash));
+            }
+        }
+        // load the chunk covering the offset
+        let leaf = self
+            .blobs
+            .export_chunk(self.hash, child_offset * 32)
+            .await?;
+        // return the hash if it is in the chunk, otherwise we are behind the end
+        let hs = HashSeqChunk::try_from(leaf)?;
+        Ok(hs.get(child_offset).inspect(|_hash| {
+            self.current_chunk = Some(hs);
+        }))
+    }
+}
+
+async fn write_push_request(
+    request: PushRequest,
+    stream: &mut SendStream,
+) -> anyhow::Result<PushRequest> {
+    let mut request_bytes = Vec::new();
+    request_bytes.push(RequestType::Push as u8);
+    request_bytes.write_length_prefixed(&request).unwrap();
+    stream.write_all(&request_bytes).await?;
+    Ok(request)
+}
+
+async fn write_observe_request(request: ObserveRequest, stream: &mut SendStream) -> io::Result<()> {
+    let request = Request::Observe(request);
+    let request_bytes = postcard::to_allocvec(&request)
+        .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
+    stream.write_all(&request_bytes).await?;
+    Ok(())
+}
+
+struct StreamContext<S> {
+    payload_bytes_sent: u64,
+    sender: S,
+}
+
+impl<S> WriteProgress for StreamContext<S>
+where
+    S: Sink<u64, Error = io::Error>,
+{
+    async fn notify_payload_write(&mut self, _index: u64, _offset: u64, len: usize) {
+        self.payload_bytes_sent += len as u64;
+        self.sender.send(self.payload_bytes_sent).await.ok();
+    }
+
+    fn log_other_write(&mut self, _len: usize) {}
+
+    async fn send_transfer_started(&mut self, _index: u64, _hash: &Hash, _size: u64) {}
+}
+
+#[cfg(test)]
+mod tests {
+    use bao_tree::{ChunkNum, ChunkRanges};
+    use testresult::TestResult;
+
+    use crate::{
+        protocol::{ChunkRangesSeq, GetRequest},
+        store::fs::{tests::INTERESTING_SIZES, FsStore},
+        tests::{add_test_hash_seq, add_test_hash_seq_incomplete},
+        util::ChunkRangesExt,
+    };
+
+    #[tokio::test]
+    async fn test_local_info_raw() -> TestResult<()> {
+        let td = tempfile::tempdir()?;
+        let store = FsStore::load(td.path().join("blobs.db")).await?;
+        let blobs = store.blobs();
+        let tt = blobs.add_slice(b"test").temp_tag().await?;
+        let hash = *tt.hash();
+        let info = store.remote().local(hash).await?;
+        assert_eq!(info.bitfield.ranges, ChunkRanges::all());
+        assert_eq!(info.local_bytes(), 4);
+        assert!(info.is_complete());
+        assert_eq!(
+            info.missing(),
+            GetRequest::new(hash, ChunkRangesSeq::empty())
+        );
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_local_info_hash_seq_large() -> TestResult<()> {
+        let sizes = (0..1024 + 5).collect::<Vec<_>>();
+        let relevant_sizes = sizes[32 * 16..32 * 32]
+            .iter()
+            .map(|x| *x as u64)
+            .sum::<u64>();
+        let td = tempfile::tempdir()?;
+        let hash_seq_ranges = ChunkRanges::chunks(16..32);
+        let store = FsStore::load(td.path().join("blobs.db")).await?;
+        {
+            // only add the hash seq itself, and only the first chunk of the children
+            let present = |i| {
+                if i == 0 {
+                    hash_seq_ranges.clone()
+                } else {
+                    ChunkRanges::from(..ChunkNum(1))
+                }
+            };
+            let content = add_test_hash_seq_incomplete(&store, sizes, present).await?;
+            let info = store.remote().local(content).await?;
+            assert_eq!(info.bitfield.ranges, hash_seq_ranges);
+            assert!(!info.is_complete());
+            assert_eq!(info.local_bytes(), relevant_sizes + 16 * 1024);
+        }
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_local_info_hash_seq() -> TestResult<()> {
+        let sizes = INTERESTING_SIZES;
+        let total_size = sizes.iter().map(|x| *x as u64).sum::<u64>();
+        let hash_seq_size = (sizes.len() as u64) * 32;
+        let td = tempfile::tempdir()?;
+        let store = FsStore::load(td.path().join("blobs.db")).await?;
+        {
+            // only add the hash seq itself, none of the children
+            let present = |i| {
+                if i == 0 {
+                    ChunkRanges::all()
+                } else {
+                    ChunkRanges::empty()
+                }
+            };
+            let content = add_test_hash_seq_incomplete(&store, sizes, present).await?;
+            let info = store.remote().local(content).await?;
+            assert_eq!(info.bitfield.ranges, ChunkRanges::all());
+            assert_eq!(info.local_bytes(), hash_seq_size);
+            assert!(!info.is_complete());
+            assert_eq!(
+                info.missing(),
+                GetRequest::new(
+                    content.hash,
+                    ChunkRangesSeq::from_ranges([
+                        ChunkRanges::empty(), // we have the hash seq itself
+                        ChunkRanges::empty(), // we always have the empty blob
+                        ChunkRanges::all(),   // we miss all the remaining blobs (sizes.len() - 1)
+                        ChunkRanges::all(),
+                        ChunkRanges::all(),
+                        ChunkRanges::all(),
+                        ChunkRanges::all(),
+                        ChunkRanges::all(),
+                        ChunkRanges::all(),
+                    ])
+                )
+            );
+            store.tags().delete_all().await?;
+        }
+        {
+            // only add the hash seq itself, and only the first chunk of the children
+            let present = |i| {
+                if i == 0 {
+                    ChunkRanges::all()
+                } else {
+                    ChunkRanges::from(..ChunkNum(1))
+                }
+            };
+            let content = add_test_hash_seq_incomplete(&store, sizes, present).await?;
+            let info = store.remote().local(content).await?;
+            let first_chunk_size = sizes.into_iter().map(|x| x.min(1024) as u64).sum::<u64>();
+            assert_eq!(info.bitfield.ranges, ChunkRanges::all());
+            assert_eq!(info.local_bytes(), hash_seq_size + first_chunk_size);
+            assert!(!info.is_complete());
+            assert_eq!(
+                info.missing(),
+                GetRequest::new(
+                    content.hash,
+                    ChunkRangesSeq::from_ranges([
+                        ChunkRanges::empty(), // we have the hash seq itself
+                        ChunkRanges::empty(), // we always have the empty blob
+                        ChunkRanges::empty(), // size=1
+                        ChunkRanges::empty(), // size=1024
+                        ChunkRanges::chunks(1..),
+                        ChunkRanges::chunks(1..),
+                        ChunkRanges::chunks(1..),
+                        ChunkRanges::chunks(1..),
+                        ChunkRanges::chunks(1..),
+                    ])
+                )
+            );
+        }
+        {
+            let content = add_test_hash_seq(&store, sizes).await?;
+            let info = store.remote().local(content).await?;
+            assert_eq!(info.bitfield.ranges, ChunkRanges::all());
+            assert_eq!(info.local_bytes(), total_size + hash_seq_size);
+            assert!(info.is_complete());
+            assert_eq!(
+                info.missing(),
+                GetRequest::new(content.hash, ChunkRangesSeq::empty())
+            );
+        }
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_local_info_complex_request() -> TestResult<()> {
+        let sizes = INTERESTING_SIZES;
+        let hash_seq_size = (sizes.len() as u64) * 32;
+        let td = tempfile::tempdir()?;
+        let store = FsStore::load(td.path().join("blobs.db")).await?;
+        // only add the hash seq itself, and only the first chunk of the children
+        let present = |i| {
+            if i == 0 {
+                ChunkRanges::all()
+            } else {
+                ChunkRanges::chunks(..2)
+            }
+        };
+        let content = add_test_hash_seq_incomplete(&store, sizes, present).await?;
+        {
+            let request: GetRequest = GetRequest::builder()
+                .root(ChunkRanges::all())
+                .build(content.hash);
+            let info = store.remote().local_for_request(request).await?;
+            assert_eq!(info.bitfield.ranges, ChunkRanges::all());
+            assert_eq!(info.local_bytes(), hash_seq_size);
+            assert!(info.is_complete());
+        }
+        {
+            let request: GetRequest = GetRequest::builder()
+                .root(ChunkRanges::all())
+                .next(ChunkRanges::all())
+                .build(content.hash);
+            let info = store.remote().local_for_request(request).await?;
+            let expected_child_sizes = sizes
+                .into_iter()
+                .take(1)
+                .map(|x| 1024.min(x as u64))
+                .sum::<u64>();
+            assert_eq!(info.bitfield.ranges, ChunkRanges::all());
+            assert_eq!(info.local_bytes(), hash_seq_size + expected_child_sizes);
+            assert!(info.is_complete());
+        }
+        {
+            let request: GetRequest = GetRequest::builder()
+                .root(ChunkRanges::all())
+                .next(ChunkRanges::all())
+                .next(ChunkRanges::all())
+                .build(content.hash);
+            let info = store.remote().local_for_request(request).await?;
+            let expected_child_sizes = sizes
+                .into_iter()
+                .take(2)
+                .map(|x| 1024.min(x as u64))
+                .sum::<u64>();
+            assert_eq!(info.bitfield.ranges, ChunkRanges::all());
+            assert_eq!(info.local_bytes(), hash_seq_size + expected_child_sizes);
+            assert!(info.is_complete());
+        }
+        {
+            let request: GetRequest = GetRequest::builder()
+                .root(ChunkRanges::all())
+                .next(ChunkRanges::chunk(0))
+                .build_open(content.hash);
+            let info = store.remote().local_for_request(request).await?;
+            let expected_child_sizes = sizes.into_iter().map(|x| 1024.min(x as u64)).sum::<u64>();
+            assert_eq!(info.bitfield.ranges, ChunkRanges::all());
+            assert_eq!(info.local_bytes(), hash_seq_size + expected_child_sizes);
+            assert!(info.is_complete());
+        }
+        Ok(())
+    }
+}
diff --git a/src/api/tags.rs b/src/api/tags.rs
new file mode 100644
index 000000000..b235a8c6b
--- /dev/null
+++ b/src/api/tags.rs
@@ -0,0 +1,192 @@
+//! Tags API
+//!
+//! The main entry point is the [`Tags`] struct.
+use std::ops::RangeBounds;
+
+use n0_future::{Stream, StreamExt};
+use ref_cast::RefCast;
+use tracing::trace;
+
+pub use super::proto::{
+    CreateTagRequest as CreateOptions, DeleteTagsRequest as DeleteOptions,
+    ListTagsRequest as ListOptions, RenameTagRequest as RenameOptions, SetTagRequest as SetOptions,
+    TagInfo,
+};
+use super::{
+    proto::{CreateTempTagRequest, Scope},
+    ApiClient, Tag, TempTag,
+};
+use crate::{api::proto::ListTempTagsRequest, HashAndFormat};
+
+/// The API for interacting with tags and temp tags.
+#[derive(Debug, Clone, ref_cast::RefCast)]
+#[repr(transparent)]
+pub struct Tags {
+    client: ApiClient,
+}
+
+impl Tags {
+    pub(crate) fn ref_from_sender(sender: &ApiClient) -> &Self {
+        Self::ref_cast(sender)
+    }
+
+    pub async fn list_temp_tags(&self) -> irpc::Result<impl Stream<Item = HashAndFormat>> {
+        let options = ListTempTagsRequest;
+        trace!("{:?}", options);
+        let res = self.client.rpc(options).await?;
+        Ok(n0_future::stream::iter(res))
+    }
+
+    /// List all tags with options.
+    ///
+    /// This is the most flexible way to list tags. All the other list methods are just convenience
+    /// methods that call this one with the appropriate options.
+    pub async fn list_with_opts(
+        &self,
+        options: ListOptions,
+    ) -> irpc::Result<impl Stream<Item = super::Result<TagInfo>>> {
+        trace!("{:?}", options);
+        let res = self.client.rpc(options).await?;
+        Ok(n0_future::stream::iter(res))
+    }
+
+    /// Get the value of a single tag
+    pub async fn get(&self, name: impl AsRef<[u8]>) -> super::RequestResult<Option<TagInfo>> {
+        let mut stream = self
+            .list_with_opts(ListOptions::single(name.as_ref()))
+            .await?;
+        Ok(stream.next().await.transpose()?)
+    }
+
+    pub async fn set_with_opts(&self, options: SetOptions) -> super::RequestResult<()> {
+        trace!("{:?}", options);
+        self.client.rpc(options).await??;
+        Ok(())
+    }
+
+    pub async fn set(
+        &self,
+        name: impl AsRef<[u8]>,
+        value: impl Into<HashAndFormat>,
+    ) -> super::RequestResult<()> {
+        self.set_with_opts(SetOptions {
+            name: Tag::from(name.as_ref()),
+            value: value.into(),
+        })
+        .await
+    }
+
+    /// List a range of tags
+    pub async fn list_range<R, E>(
+        &self,
+        range: R,
+    ) -> irpc::Result<impl Stream<Item = super::Result<TagInfo>>>
+    where
+        R: RangeBounds<E>,
+        E: AsRef<[u8]>,
+    {
+        self.list_with_opts(ListOptions::range(range)).await
+    }
+
+    /// Lists all tags with the given prefix.
+    pub async fn list_prefix(
+        &self,
+        prefix: impl AsRef<[u8]>,
+    ) -> irpc::Result<impl Stream<Item = super::Result<TagInfo>>> {
+        self.list_with_opts(ListOptions::prefix(prefix.as_ref()))
+            .await
+    }
+
+    /// Lists all tags.
+    pub async fn list(&self) -> irpc::Result<impl Stream<Item = super::Result<TagInfo>>> {
+        self.list_with_opts(ListOptions::all()).await
+    }
+
+    /// Lists all tags with a hash_seq format.
+    pub async fn list_hash_seq(&self) -> irpc::Result<impl Stream<Item = super::Result<TagInfo>>> {
+        self.list_with_opts(ListOptions::hash_seq()).await
+    }
+
+    /// Deletes a tag.
+    pub async fn delete_with_opts(&self, options: DeleteOptions) -> super::RequestResult<()> {
+        trace!("{:?}", options);
+        self.client.rpc(options).await??;
+        Ok(())
+    }
+
+    /// Deletes a tag.
+    pub async fn delete(&self, name: impl AsRef<[u8]>) -> super::RequestResult<()> {
+        self.delete_with_opts(DeleteOptions::single(name.as_ref()))
+            .await
+    }
+
+    /// Deletes a range of tags.
+    pub async fn delete_range<R, E>(&self, range: R) -> super::RequestResult<()>
+    where
+        R: RangeBounds<E>,
+        E: AsRef<[u8]>,
+    {
+        self.delete_with_opts(DeleteOptions::range(range)).await
+    }
+
+    /// Delete all tags with the given prefix.
+    pub async fn delete_prefix(&self, prefix: impl AsRef<[u8]>) -> super::RequestResult<()> {
+        self.delete_with_opts(DeleteOptions::prefix(prefix.as_ref()))
+            .await
+    }
+
+    /// Delete all tags. Use with care. After this, all data will be garbage collected.
+    pub async fn delete_all(&self) -> super::RequestResult<()> {
+        self.delete_with_opts(DeleteOptions {
+            from: None,
+            to: None,
+        })
+        .await
+    }
+
+    /// Rename a tag atomically
+    ///
+    /// If the tag does not exist, this will return an error.
+    pub async fn rename_with_opts(&self, options: RenameOptions) -> super::RequestResult<()> {
+        trace!("{:?}", options);
+        self.client.rpc(options).await??;
+        Ok(())
+    }
+
+    /// Rename a tag atomically
+    ///
+    /// If the tag does not exist, this will return an error.
+    pub async fn rename(
+        &self,
+        from: impl AsRef<[u8]>,
+        to: impl AsRef<[u8]>,
+    ) -> super::RequestResult<()> {
+        self.rename_with_opts(RenameOptions {
+            from: Tag::from(from.as_ref()),
+            to: Tag::from(to.as_ref()),
+        })
+        .await
+    }
+
+    pub async fn create_with_opts(&self, options: CreateOptions) -> super::RequestResult<Tag> {
+        trace!("{:?}", options);
+        let rx = self.client.rpc(options);
+        Ok(rx.await??)
+    }
+
+    pub async fn create(&self, value: impl Into<HashAndFormat>) -> super::RequestResult<Tag> {
+        self.create_with_opts(CreateOptions {
+            value: value.into(),
+        })
+        .await
+    }
+
+    pub async fn temp_tag(&self, value: impl Into<HashAndFormat>) -> irpc::Result<TempTag> {
+        let value = value.into();
+        let msg = CreateTempTagRequest {
+            scope: Scope::GLOBAL,
+            value,
+        };
+        self.client.rpc(msg).await
+    }
+}
diff --git a/src/cli.rs b/src/cli.rs
deleted file mode 100644
index 5e06df84e..000000000
--- a/src/cli.rs
+++ /dev/null
@@ -1,1209 +0,0 @@
-//! Define blob-related commands.
-#![allow(missing_docs)]
-use std::{
-    collections::{BTreeMap, HashMap},
-    net::SocketAddr,
-    path::PathBuf,
-    time::Duration,
-};
-
-use anyhow::{anyhow, bail, ensure, Context, Result};
-use clap::Subcommand;
-use console::{style, Emoji};
-use futures_lite::{Stream, StreamExt};
-use indicatif::{
-    HumanBytes, HumanDuration, MultiProgress, ProgressBar, ProgressDrawTarget, ProgressState,
-    ProgressStyle,
-};
-use iroh::{NodeAddr, PublicKey, RelayUrl};
-use tokio::io::AsyncWriteExt;
-
-use crate::{
-    get::{db::DownloadProgress, progress::BlobProgress, Stats},
-    net_protocol::DownloadMode,
-    provider::AddProgress,
-    rpc::client::blobs::{
-        self, BlobInfo, BlobStatus, CollectionInfo, DownloadOptions, IncompleteBlobInfo, WrapOption,
-    },
-    store::{ConsistencyCheckProgress, ExportFormat, ExportMode, ReportLevel, ValidateProgress},
-    ticket::BlobTicket,
-    util::SetTagOption,
-    BlobFormat, Hash, HashAndFormat, Tag,
-};
-
-pub mod tags;
-
-/// Subcommands for the blob command.
-#[allow(clippy::large_enum_variant)]
-#[derive(Subcommand, Debug, Clone)]
-pub enum BlobCommands {
-    /// Add data from PATH to the running node.
-    Add {
-        /// Path to a file or folder.
-        ///
-        /// If set to `STDIN`, the data will be read from stdin.
-        source: BlobSource,
-
-        #[clap(flatten)]
-        options: BlobAddOptions,
-    },
-    /// Download data to the running node's database and provide it.
-    ///
-    /// In addition to downloading the data, you can also specify an optional output directory
-    /// where the data will be exported to after it has been downloaded.
-    Get {
-        /// Ticket or Hash to use.
-        #[clap(name = "TICKET OR HASH")]
-        ticket: TicketOrHash,
-        /// Additional socket address to use to contact the node. Can be used multiple times.
-        #[clap(long)]
-        address: Vec<SocketAddr>,
-        /// Override the relay URL to use to contact the node.
-        #[clap(long)]
-        relay_url: Option<RelayUrl>,
-        /// Override to treat the blob as a raw blob or a hash sequence.
-        #[clap(long)]
-        recursive: Option<bool>,
-        /// If set, the ticket's direct addresses will not be used.
-        #[clap(long)]
-        override_addresses: bool,
-        /// NodeId of the provider.
-        #[clap(long)]
-        node: Option<PublicKey>,
-        /// Directory or file in which to save the file(s).
-        ///
-        /// If set to `STDOUT` the output will be redirected to stdout.
-        ///
-        /// If not specified, the data will only be stored internally.
-        #[clap(long, short)]
-        out: Option<OutputTarget>,
-        /// If set, the data will be moved to the output directory, and iroh will assume that it
-        /// will not change.
-        #[clap(long, default_value_t = false)]
-        stable: bool,
-        /// Tag to tag the data with.
-        #[clap(long)]
-        tag: Option<String>,
-        /// If set, will queue the download in the download queue.
-        ///
-        /// Use this if you are doing many downloads in parallel and want to limit the number of
-        /// downloads running concurrently.
-        #[clap(long)]
-        queued: bool,
-    },
-    /// Export a blob from the internal blob store to the local filesystem.
-    Export {
-        /// The hash to export.
-        hash: Hash,
-        /// Directory or file in which to save the file(s).
-        ///
-        /// If set to `STDOUT` the output will be redirected to stdout.
-        out: OutputTarget,
-        /// Set to true if the hash refers to a collection and you want to export all children of
-        /// the collection.
-        #[clap(long, default_value_t = false)]
-        recursive: bool,
-        /// If set, the data will be moved to the output directory, and iroh will assume that it
-        /// will not change.
-        #[clap(long, default_value_t = false)]
-        stable: bool,
-    },
-    /// List available content on the node.
-    #[clap(subcommand)]
-    List(ListCommands),
-    /// Validate hashes on the running node.
-    Validate {
-        /// Verbosity level.
-        #[clap(short, long, action(clap::ArgAction::Count))]
-        verbose: u8,
-        /// Repair the store by removing invalid data
-        ///
-        /// Caution: this will remove data to make the store consistent, even
-        /// if the data might be salvageable. E.g. for an entry for which the
-        /// outboard data is missing, the entry will be removed, even if the
-        /// data is complete.
-        #[clap(long, default_value_t = false)]
-        repair: bool,
-    },
-    /// Perform a database consistency check on the running node.
-    ConsistencyCheck {
-        /// Verbosity level.
-        #[clap(short, long, action(clap::ArgAction::Count))]
-        verbose: u8,
-        /// Repair the store by removing invalid data
-        ///
-        /// Caution: this will remove data to make the store consistent, even
-        /// if the data might be salvageable. E.g. for an entry for which the
-        /// outboard data is missing, the entry will be removed, even if the
-        /// data is complete.
-        #[clap(long, default_value_t = false)]
-        repair: bool,
-    },
-    /// Delete content on the node.
-    #[clap(subcommand)]
-    Delete(DeleteCommands),
-    /// Get a ticket to share this blob.
-    Share {
-        /// Hash of the blob to share.
-        hash: Hash,
-        /// If the blob is a collection, the requester will also fetch the listed blobs.
-        #[clap(long, default_value_t = false)]
-        recursive: bool,
-        /// Display the contents of this ticket too.
-        #[clap(long, hide = true)]
-        debug: bool,
-    },
-}
-
-/// Possible outcomes of an input.
-#[derive(Debug, Clone, derive_more::Display)]
-pub enum TicketOrHash {
-    Ticket(BlobTicket),
-    Hash(Hash),
-}
-
-impl std::str::FromStr for TicketOrHash {
-    type Err = anyhow::Error;
-
-    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
-        if let Ok(ticket) = BlobTicket::from_str(s) {
-            return Ok(Self::Ticket(ticket));
-        }
-        if let Ok(hash) = Hash::from_str(s) {
-            return Ok(Self::Hash(hash));
-        }
-        Err(anyhow!("neither a valid ticket or hash"))
-    }
-}
-
-impl BlobCommands {
-    /// Runs the blob command given the iroh client.
-    pub async fn run(self, blobs: &blobs::Client, addr: NodeAddr) -> Result<()> {
-        match self {
-            Self::Get {
-                ticket,
-                mut address,
-                relay_url,
-                recursive,
-                override_addresses,
-                node,
-                out,
-                stable,
-                tag,
-                queued,
-            } => {
-                let (node_addr, hash, format) = match ticket {
-                    TicketOrHash::Ticket(ticket) => {
-                        let (node_addr, hash, blob_format) = ticket.into_parts();
-
-                        // create the node address with the appropriate overrides
-                        let node_addr = {
-                            let NodeAddr {
-                                node_id,
-                                relay_url: original_relay_url,
-                                direct_addresses,
-                            } = node_addr;
-                            let addresses = if override_addresses {
-                                // use only the cli supplied ones
-                                address
-                            } else {
-                                // use both the cli supplied ones and the ticket ones
-                                address.extend(direct_addresses);
-                                address
-                            };
-
-                            // prefer direct arg over ticket
-                            let relay_url = relay_url.or(original_relay_url);
-
-                            NodeAddr::from_parts(node_id, relay_url, addresses)
-                        };
-
-                        // check if the blob format has an override
-                        let blob_format = match recursive {
-                            Some(true) => BlobFormat::HashSeq,
-                            Some(false) => BlobFormat::Raw,
-                            None => blob_format,
-                        };
-
-                        (node_addr, hash, blob_format)
-                    }
-                    TicketOrHash::Hash(hash) => {
-                        // check if the blob format has an override
-                        let blob_format = match recursive {
-                            Some(true) => BlobFormat::HashSeq,
-                            Some(false) => BlobFormat::Raw,
-                            None => BlobFormat::Raw,
-                        };
-
-                        let Some(node) = node else {
-                            bail!("missing NodeId");
-                        };
-
-                        let node_addr = NodeAddr::from_parts(node, relay_url, address);
-                        (node_addr, hash, blob_format)
-                    }
-                };
-
-                if format != BlobFormat::Raw && out == Some(OutputTarget::Stdout) {
-                    return Err(anyhow::anyhow!("The input arguments refer to a collection of blobs and output is set to STDOUT. Only single blobs may be passed in this case."));
-                }
-
-                let tag = match tag {
-                    Some(tag) => SetTagOption::Named(Tag::from(tag)),
-                    None => SetTagOption::Auto,
-                };
-
-                let mode = match queued {
-                    true => DownloadMode::Queued,
-                    false => DownloadMode::Direct,
-                };
-
-                let mut stream = blobs
-                    .download_with_opts(
-                        hash,
-                        DownloadOptions {
-                            format,
-                            nodes: vec![node_addr],
-                            tag,
-                            mode,
-                        },
-                    )
-                    .await?;
-
-                show_download_progress(hash, &mut stream).await?;
-
-                match out {
-                    None => {}
-                    Some(OutputTarget::Stdout) => {
-                        // we asserted above that `OutputTarget::Stdout` is only permitted if getting a
-                        // single hash and not a hashseq.
-                        let mut blob_read = blobs.read(hash).await?;
-                        tokio::io::copy(&mut blob_read, &mut tokio::io::stdout()).await?;
-                    }
-                    Some(OutputTarget::Path(path)) => {
-                        let absolute = std::env::current_dir()?.join(&path);
-                        if matches!(format, BlobFormat::HashSeq) {
-                            ensure!(!absolute.is_dir(), "output must not be a directory");
-                        }
-                        let recursive = format == BlobFormat::HashSeq;
-                        let mode = match stable {
-                            true => ExportMode::TryReference,
-                            false => ExportMode::Copy,
-                        };
-                        let format = match recursive {
-                            true => ExportFormat::Collection,
-                            false => ExportFormat::Blob,
-                        };
-                        tracing::info!("exporting to {} -> {}", path.display(), absolute.display());
-                        let stream = blobs.export(hash, absolute, format, mode).await?;
-
-                        // TODO: report export progress
-                        stream.await?;
-                    }
-                };
-
-                Ok(())
-            }
-            Self::Export {
-                hash,
-                out,
-                recursive,
-                stable,
-            } => {
-                match out {
-                    OutputTarget::Stdout => {
-                        ensure!(
-                            !recursive,
-                            "Recursive option is not supported when exporting to STDOUT"
-                        );
-                        let mut blob_read = blobs.read(hash).await?;
-                        tokio::io::copy(&mut blob_read, &mut tokio::io::stdout()).await?;
-                    }
-                    OutputTarget::Path(path) => {
-                        let absolute = std::env::current_dir()?.join(&path);
-                        if !recursive {
-                            ensure!(!absolute.is_dir(), "output must not be a directory");
-                        }
-                        let mode = match stable {
-                            true => ExportMode::TryReference,
-                            false => ExportMode::Copy,
-                        };
-                        let format = match recursive {
-                            true => ExportFormat::Collection,
-                            false => ExportFormat::Blob,
-                        };
-                        tracing::info!(
-                            "exporting {hash} to {} -> {}",
-                            path.display(),
-                            absolute.display()
-                        );
-                        let stream = blobs.export(hash, absolute, format, mode).await?;
-                        // TODO: report export progress
-                        stream.await?;
-                    }
-                };
-                Ok(())
-            }
-            Self::List(cmd) => cmd.run(blobs).await,
-            Self::Delete(cmd) => cmd.run(blobs).await,
-            Self::Validate { verbose, repair } => validate(blobs, verbose, repair).await,
-            Self::ConsistencyCheck { verbose, repair } => {
-                consistency_check(blobs, verbose, repair).await
-            }
-            Self::Add {
-                source: path,
-                options,
-            } => add_with_opts(blobs, addr, path, options).await,
-            Self::Share {
-                hash,
-                recursive,
-                debug,
-            } => {
-                let format = if recursive {
-                    BlobFormat::HashSeq
-                } else {
-                    BlobFormat::Raw
-                };
-                let status = blobs.status(hash).await?;
-                let ticket = BlobTicket::new(addr, hash, format)?;
-
-                let (blob_status, size) = match (status, format) {
-                    (BlobStatus::Complete { size }, BlobFormat::Raw) => ("blob", size),
-                    (BlobStatus::Partial { size }, BlobFormat::Raw) => {
-                        ("incomplete blob", size.value())
-                    }
-                    (BlobStatus::Complete { size }, BlobFormat::HashSeq) => ("collection", size),
-                    (BlobStatus::Partial { size }, BlobFormat::HashSeq) => {
-                        ("incomplete collection", size.value())
-                    }
-                    (BlobStatus::NotFound, _) => {
-                        return Err(anyhow!("blob is missing"));
-                    }
-                };
-                println!(
-                    "Ticket for {blob_status} {hash} ({})\n{ticket}",
-                    HumanBytes(size)
-                );
-
-                if debug {
-                    println!("{ticket:#?}")
-                }
-                Ok(())
-            }
-        }
-    }
-}
-
-/// Options for the `blob add` command.
-#[derive(clap::Args, Debug, Clone)]
-pub struct BlobAddOptions {
-    /// Add in place
-    ///
-    /// Set this to true only if you are sure that the data in its current location
-    /// will not change.
-    #[clap(long, default_value_t = false)]
-    pub in_place: bool,
-
-    /// Tag to tag the data with.
-    #[clap(long)]
-    pub tag: Option<String>,
-
-    /// Wrap the added file or directory in a collection.
-    ///
-    /// When adding a single file, without `wrap` the file is added as a single blob and no
-    /// collection is created. When enabling `wrap` it also creates a collection with a
-    /// single entry, where the entry's name is the filename and the entry's content is blob.
-    ///
-    /// When adding a directory, a collection is always created.
-    /// Without `wrap`, the collection directly contains the entries from the added directory.
-    /// With `wrap`, the directory will be nested so that all names in the collection are
-    /// prefixed with the directory name, thus preserving the name of the directory.
-    ///
-    /// When adding content from STDIN and setting `wrap` you also need to set `filename` to name
-    /// the entry pointing to the content from STDIN.
-    #[clap(long, default_value_t = false)]
-    pub wrap: bool,
-
-    /// Override the filename used for the entry in the created collection.
-    ///
-    /// Only supported `wrap` is set.
-    /// Required when adding content from STDIN and setting `wrap`.
-    #[clap(long, requires = "wrap")]
-    pub filename: Option<String>,
-
-    /// Do not print the all-in-one ticket to get the added data from this node.
-    #[clap(long)]
-    pub no_ticket: bool,
-}
-
-/// Possible list subcommands.
-#[derive(Subcommand, Debug, Clone)]
-pub enum ListCommands {
-    /// List the available blobs on the running provider.
-    Blobs,
-    /// List the blobs on the running provider that are not full files.
-    IncompleteBlobs,
-    /// List the available collections on the running provider.
-    Collections,
-}
-
-impl ListCommands {
-    /// Runs a list subcommand.
-    pub async fn run(self, blobs: &blobs::Client) -> Result<()> {
-        match self {
-            Self::Blobs => {
-                let mut response = blobs.list().await?;
-                while let Some(item) = response.next().await {
-                    let BlobInfo { path, hash, size } = item?;
-                    println!("{} {} ({})", path, hash, HumanBytes(size));
-                }
-            }
-            Self::IncompleteBlobs => {
-                let mut response = blobs.list_incomplete().await?;
-                while let Some(item) = response.next().await {
-                    let IncompleteBlobInfo { hash, size, .. } = item?;
-                    println!("{} ({})", hash, HumanBytes(size));
-                }
-            }
-            Self::Collections => {
-                let mut response = blobs.list_collections()?;
-                while let Some(item) = response.next().await {
-                    let CollectionInfo {
-                        tag,
-                        hash,
-                        total_blobs_count,
-                        total_blobs_size,
-                    } = item?;
-                    let total_blobs_count = total_blobs_count.unwrap_or_default();
-                    let total_blobs_size = total_blobs_size.unwrap_or_default();
-                    println!(
-                        "{}: {} {} {} ({})",
-                        tag,
-                        hash,
-                        total_blobs_count,
-                        if total_blobs_count > 1 {
-                            "blobs"
-                        } else {
-                            "blob"
-                        },
-                        HumanBytes(total_blobs_size),
-                    );
-                }
-            }
-        }
-        Ok(())
-    }
-}
-
-/// Possible delete subcommands.
-#[derive(Subcommand, Debug, Clone)]
-pub enum DeleteCommands {
-    /// Delete the given blobs
-    Blob {
-        /// Blobs to delete
-        #[arg(required = true)]
-        hash: Hash,
-    },
-}
-
-impl DeleteCommands {
-    /// Runs the delete command.
-    pub async fn run(self, blobs: &blobs::Client) -> Result<()> {
-        match self {
-            Self::Blob { hash } => {
-                let response = blobs.delete_blob(hash).await;
-                if let Err(e) = response {
-                    eprintln!("Error: {}", e);
-                }
-            }
-        }
-        Ok(())
-    }
-}
-
-/// Returns the corresponding [`ReportLevel`] given the verbosity level.
-fn get_report_level(verbose: u8) -> ReportLevel {
-    match verbose {
-        0 => ReportLevel::Warn,
-        1 => ReportLevel::Info,
-        _ => ReportLevel::Trace,
-    }
-}
-
-/// Applies the report level to the given text.
-fn apply_report_level(text: String, level: ReportLevel) -> console::StyledObject<String> {
-    match level {
-        ReportLevel::Trace => style(text).dim(),
-        ReportLevel::Info => style(text),
-        ReportLevel::Warn => style(text).yellow(),
-        ReportLevel::Error => style(text).red(),
-    }
-}
-
-/// Checks the consistency of the blobs on the running node, and repairs inconsistencies if instructed.
-pub async fn consistency_check(blobs: &blobs::Client, verbose: u8, repair: bool) -> Result<()> {
-    let mut response = blobs.consistency_check(repair).await?;
-    let verbosity = get_report_level(verbose);
-    let print = |level: ReportLevel, entry: Option<Hash>, message: String| {
-        if level < verbosity {
-            return;
-        }
-        let level_text = level.to_string().to_lowercase();
-        let text = if let Some(hash) = entry {
-            format!("{}: {} ({})", level_text, message, hash.to_hex())
-        } else {
-            format!("{}: {}", level_text, message)
-        };
-        let styled = apply_report_level(text, level);
-        eprintln!("{}", styled);
-    };
-
-    while let Some(item) = response.next().await {
-        match item? {
-            ConsistencyCheckProgress::Start => {
-                eprintln!("Starting consistency check ...");
-            }
-            ConsistencyCheckProgress::Update {
-                message,
-                entry,
-                level,
-            } => {
-                print(level, entry, message);
-            }
-            ConsistencyCheckProgress::Done => {
-                eprintln!("Consistency check done");
-            }
-            ConsistencyCheckProgress::Abort(error) => {
-                eprintln!("Consistency check error {}", error);
-                break;
-            }
-        }
-    }
-    Ok(())
-}
-
-/// Checks the validity of the blobs on the running node, and repairs anything invalid if instructed.
-pub async fn validate(blobs: &blobs::Client, verbose: u8, repair: bool) -> Result<()> {
-    let mut state = ValidateProgressState::new();
-    let mut response = blobs.validate(repair).await?;
-    let verbosity = get_report_level(verbose);
-    let print = |level: ReportLevel, entry: Option<Hash>, message: String| {
-        if level < verbosity {
-            return;
-        }
-        let level_text = level.to_string().to_lowercase();
-        let text = if let Some(hash) = entry {
-            format!("{}: {} ({})", level_text, message, hash.to_hex())
-        } else {
-            format!("{}: {}", level_text, message)
-        };
-        let styled = apply_report_level(text, level);
-        eprintln!("{}", styled);
-    };
-
-    let mut partial = BTreeMap::new();
-
-    while let Some(item) = response.next().await {
-        match item? {
-            ValidateProgress::PartialEntry {
-                id,
-                hash,
-                path,
-                size,
-            } => {
-                partial.insert(id, hash);
-                print(
-                    ReportLevel::Trace,
-                    Some(hash),
-                    format!(
-                        "Validating partial entry {} {} {}",
-                        id,
-                        path.unwrap_or_default(),
-                        size
-                    ),
-                );
-            }
-            ValidateProgress::PartialEntryProgress { id, offset } => {
-                let entry = partial.get(&id).cloned();
-                print(
-                    ReportLevel::Trace,
-                    entry,
-                    format!("Partial entry {} at {}", id, offset),
-                );
-            }
-            ValidateProgress::PartialEntryDone { id, ranges } => {
-                let entry: Option<Hash> = partial.remove(&id);
-                print(
-                    ReportLevel::Info,
-                    entry,
-                    format!("Partial entry {} done {:?}", id, ranges.to_chunk_ranges()),
-                );
-            }
-            ValidateProgress::Starting { total } => {
-                state.starting(total);
-            }
-            ValidateProgress::Entry {
-                id,
-                hash,
-                path,
-                size,
-            } => {
-                state.add_entry(id, hash, path, size);
-            }
-            ValidateProgress::EntryProgress { id, offset } => {
-                state.progress(id, offset);
-            }
-            ValidateProgress::EntryDone { id, error } => {
-                state.done(id, error);
-            }
-            ValidateProgress::Abort(error) => {
-                state.abort(error.to_string());
-                break;
-            }
-            ValidateProgress::AllDone => {
-                break;
-            }
-        }
-    }
-    Ok(())
-}
-
-/// Collection of all the validation progress state.
-struct ValidateProgressState {
-    mp: MultiProgress,
-    pbs: HashMap<u64, ProgressBar>,
-    overall: ProgressBar,
-    total: u64,
-    errors: u64,
-    successes: u64,
-}
-
-impl ValidateProgressState {
-    /// Creates a new validation progress state collection.
-    fn new() -> Self {
-        let mp = MultiProgress::new();
-        let overall = mp.add(ProgressBar::new(0));
-        overall.enable_steady_tick(Duration::from_millis(500));
-        Self {
-            mp,
-            pbs: HashMap::new(),
-            overall,
-            total: 0,
-            errors: 0,
-            successes: 0,
-        }
-    }
-
-    /// Sets the total number to the provided value and style the progress bar to starting.
-    fn starting(&mut self, total: u64) {
-        self.total = total;
-        self.errors = 0;
-        self.successes = 0;
-        self.overall.set_position(0);
-        self.overall.set_length(total);
-        self.overall.set_style(
-            ProgressStyle::default_bar()
-                .template("{spinner:.green} [{bar:60.cyan/blue}] {msg}")
-                .unwrap()
-                .progress_chars("=>-"),
-        );
-    }
-
-    /// Adds a message to the progress bar in the given `id`.
-    fn add_entry(&mut self, id: u64, hash: Hash, path: Option<String>, size: u64) {
-        let pb = self.mp.insert_before(&self.overall, ProgressBar::new(size));
-        pb.set_style(ProgressStyle::default_bar()
-            .template("{spinner:.green} [{bar:40.cyan/blue}] {msg} {bytes}/{total_bytes} ({bytes_per_sec}, eta {eta})").unwrap()
-            .progress_chars("=>-"));
-        let msg = if let Some(path) = path {
-            format!("{} {}", hash.to_hex(), path)
-        } else {
-            hash.to_hex().to_string()
-        };
-        pb.set_message(msg);
-        pb.set_position(0);
-        pb.set_length(size);
-        pb.enable_steady_tick(Duration::from_millis(500));
-        self.pbs.insert(id, pb);
-    }
-
-    /// Progresses the progress bar with `id` by `progress` amount.
-    fn progress(&mut self, id: u64, progress: u64) {
-        if let Some(pb) = self.pbs.get_mut(&id) {
-            pb.set_position(progress);
-        }
-    }
-
-    /// Set an error in the progress bar. Consumes the [`ValidateProgressState`].
-    fn abort(self, error: String) {
-        let error_line = self.mp.add(ProgressBar::new(0));
-        error_line.set_style(ProgressStyle::default_bar().template("{msg}").unwrap());
-        error_line.set_message(error);
-    }
-
-    /// Finishes a progress bar with a given error message.
-    fn done(&mut self, id: u64, error: Option<String>) {
-        if let Some(pb) = self.pbs.remove(&id) {
-            let ok_char = style(Emoji("✔", "OK")).green();
-            let fail_char = style(Emoji("✗", "Error")).red();
-            let ok = error.is_none();
-            let msg = match error {
-                Some(error) => format!("{} {} {}", pb.message(), fail_char, error),
-                None => format!("{} {}", pb.message(), ok_char),
-            };
-            if ok {
-                self.successes += 1;
-            } else {
-                self.errors += 1;
-            }
-            self.overall.set_position(self.errors + self.successes);
-            self.overall.set_message(format!(
-                "Overall {} {}, {} {}",
-                self.errors, fail_char, self.successes, ok_char
-            ));
-            if ok {
-                pb.finish_and_clear();
-            } else {
-                pb.set_style(ProgressStyle::default_bar().template("{msg}").unwrap());
-                pb.finish_with_message(msg);
-            }
-        }
-    }
-}
-
-/// Where the data should be read from.
-#[derive(Debug, Clone, derive_more::Display, PartialEq, Eq)]
-pub enum BlobSource {
-    /// Reads from stdin
-    #[display("STDIN")]
-    Stdin,
-    /// Reads from the provided path
-    #[display("{}", _0.display())]
-    Path(PathBuf),
-}
-
-impl From<String> for BlobSource {
-    fn from(s: String) -> Self {
-        if s == "STDIN" {
-            return BlobSource::Stdin;
-        }
-
-        BlobSource::Path(s.into())
-    }
-}
-
-/// Data source for adding data to iroh.
-#[derive(Debug, Clone)]
-pub enum BlobSourceIroh {
-    /// A file or directory on the node's local file system.
-    LocalFs { path: PathBuf, in_place: bool },
-    /// Data passed via STDIN.
-    Stdin,
-}
-
-/// Whether to print an all-in-one ticket.
-#[derive(Debug, Clone)]
-pub enum TicketOption {
-    /// Do not print an all-in-one ticket
-    None,
-    /// Print an all-in-one ticket.
-    Print,
-}
-
-/// Adds a [`BlobSource`] given some [`BlobAddOptions`].
-pub async fn add_with_opts(
-    blobs: &blobs::Client,
-    addr: NodeAddr,
-    source: BlobSource,
-    opts: BlobAddOptions,
-) -> Result<()> {
-    let tag = match opts.tag {
-        Some(tag) => SetTagOption::Named(Tag::from(tag)),
-        None => SetTagOption::Auto,
-    };
-    let ticket = match opts.no_ticket {
-        true => TicketOption::None,
-        false => TicketOption::Print,
-    };
-    let source = match source {
-        BlobSource::Stdin => BlobSourceIroh::Stdin,
-        BlobSource::Path(path) => BlobSourceIroh::LocalFs {
-            path,
-            in_place: opts.in_place,
-        },
-    };
-    let wrap = match (opts.wrap, opts.filename) {
-        (true, None) => WrapOption::Wrap { name: None },
-        (true, Some(filename)) => WrapOption::Wrap {
-            name: Some(filename),
-        },
-        (false, None) => WrapOption::NoWrap,
-        (false, Some(_)) => bail!("`--filename` may not be used without `--wrap`"),
-    };
-
-    add(blobs, addr, source, tag, ticket, wrap).await
-}
-
-/// Adds data to iroh, either from a path or, if path is `None`, from STDIN.
-pub async fn add(
-    blobs: &blobs::Client,
-    addr: NodeAddr,
-    source: BlobSourceIroh,
-    tag: SetTagOption,
-    ticket: TicketOption,
-    wrap: WrapOption,
-) -> Result<()> {
-    let (hash, format, entries) = match source {
-        BlobSourceIroh::LocalFs { path, in_place } => {
-            let absolute = path.canonicalize()?;
-            println!("Adding {} as {}...", path.display(), absolute.display());
-
-            // tell the node to add the data
-            let stream = blobs.add_from_path(absolute, in_place, tag, wrap).await?;
-            aggregate_add_response(stream).await?
-        }
-        BlobSourceIroh::Stdin => {
-            println!("Adding from STDIN...");
-            // Store STDIN content into a temporary file
-            let (file, path) = tempfile::NamedTempFile::new()?.into_parts();
-            let mut file = tokio::fs::File::from_std(file);
-            let path_buf = path.to_path_buf();
-            // Copy from stdin to the file, until EOF
-            tokio::io::copy(&mut tokio::io::stdin(), &mut file).await?;
-            file.flush().await?;
-            drop(file);
-
-            // tell the node to add the data
-            let stream = blobs.add_from_path(path_buf, false, tag, wrap).await?;
-            aggregate_add_response(stream).await?
-        }
-    };
-
-    print_add_response(hash, format, entries);
-    if let TicketOption::Print = ticket {
-        let ticket = BlobTicket::new(addr, hash, format)?;
-        println!("All-in-one ticket: {ticket}");
-    }
-    Ok(())
-}
-
-/// Entry with a given name, size, and hash.
-#[derive(Debug)]
-pub struct ProvideResponseEntry {
-    pub name: String,
-    pub size: u64,
-    pub hash: Hash,
-}
-
-/// Combines the [`AddProgress`] outputs from a [`Stream`] into a single tuple.
-pub async fn aggregate_add_response(
-    mut stream: impl Stream<Item = Result<AddProgress>> + Unpin,
-) -> Result<(Hash, BlobFormat, Vec<ProvideResponseEntry>)> {
-    let mut hash_and_format = None;
-    let mut collections = BTreeMap::<u64, (String, u64, Option<Hash>)>::new();
-    let mut mp = Some(ProvideProgressState::new());
-    while let Some(item) = stream.next().await {
-        match item? {
-            AddProgress::Found { name, id, size } => {
-                tracing::trace!("Found({id},{name},{size})");
-                if let Some(mp) = mp.as_mut() {
-                    mp.found(name.clone(), id, size);
-                }
-                collections.insert(id, (name, size, None));
-            }
-            AddProgress::Progress { id, offset } => {
-                tracing::trace!("Progress({id}, {offset})");
-                if let Some(mp) = mp.as_mut() {
-                    mp.progress(id, offset);
-                }
-            }
-            AddProgress::Done { hash, id } => {
-                tracing::trace!("Done({id},{hash:?})");
-                if let Some(mp) = mp.as_mut() {
-                    mp.done(id, hash);
-                }
-                match collections.get_mut(&id) {
-                    Some((_, _, ref mut h)) => {
-                        *h = Some(hash);
-                    }
-                    None => {
-                        anyhow::bail!("Got Done for unknown collection id {id}");
-                    }
-                }
-            }
-            AddProgress::AllDone { hash, format, .. } => {
-                tracing::trace!("AllDone({hash:?})");
-                if let Some(mp) = mp.take() {
-                    mp.all_done();
-                }
-                hash_and_format = Some(HashAndFormat { hash, format });
-                break;
-            }
-            AddProgress::Abort(e) => {
-                if let Some(mp) = mp.take() {
-                    mp.error();
-                }
-                anyhow::bail!("Error while adding data: {e}");
-            }
-        }
-    }
-    let HashAndFormat { hash, format } =
-        hash_and_format.context("Missing hash for collection or blob")?;
-    let entries = collections
-        .into_iter()
-        .map(|(_, (name, size, hash))| {
-            let hash = hash.context(format!("Missing hash for {name}"))?;
-            Ok(ProvideResponseEntry { name, size, hash })
-        })
-        .collect::<Result<Vec<_>>>()?;
-    Ok((hash, format, entries))
-}
-
-/// Prints out the add response.
-pub fn print_add_response(hash: Hash, format: BlobFormat, entries: Vec<ProvideResponseEntry>) {
-    let mut total_size = 0;
-    for ProvideResponseEntry { name, size, hash } in entries {
-        total_size += size;
-        println!("- {}: {} {:#}", name, HumanBytes(size), hash);
-    }
-    println!("Total: {}", HumanBytes(total_size));
-    println!();
-    match format {
-        BlobFormat::Raw => println!("Blob: {}", hash),
-        BlobFormat::HashSeq => println!("Collection: {}", hash),
-    }
-}
-
-/// Progress state for providing.
-#[derive(Debug)]
-pub struct ProvideProgressState {
-    mp: MultiProgress,
-    pbs: HashMap<u64, ProgressBar>,
-}
-
-impl ProvideProgressState {
-    /// Creates a new provide progress state.
-    fn new() -> Self {
-        Self {
-            mp: MultiProgress::new(),
-            pbs: HashMap::new(),
-        }
-    }
-
-    /// Inserts a new progress bar with the given id, name, and size.
-    fn found(&mut self, name: String, id: u64, size: u64) {
-        let pb = self.mp.add(ProgressBar::new(size));
-        pb.set_style(ProgressStyle::default_bar()
-            .template("{spinner:.green} [{bar:40.cyan/blue}] {msg} {bytes}/{total_bytes} ({bytes_per_sec}, eta {eta})").unwrap()
-            .progress_chars("=>-"));
-        pb.set_message(name);
-        pb.set_length(size);
-        pb.set_position(0);
-        pb.enable_steady_tick(Duration::from_millis(500));
-        self.pbs.insert(id, pb);
-    }
-
-    /// Adds some progress to the progress bar with the given id.
-    fn progress(&mut self, id: u64, progress: u64) {
-        if let Some(pb) = self.pbs.get_mut(&id) {
-            pb.set_position(progress);
-        }
-    }
-
-    /// Sets the multiprogress bar with the given id as finished and clear it.
-    fn done(&mut self, id: u64, _hash: Hash) {
-        if let Some(pb) = self.pbs.remove(&id) {
-            pb.finish_and_clear();
-            self.mp.remove(&pb);
-        }
-    }
-
-    /// Sets the multiprogress bar as finished and clear them.
-    fn all_done(self) {
-        self.mp.clear().ok();
-    }
-
-    /// Clears the multiprogress bar.
-    fn error(self) {
-        self.mp.clear().ok();
-    }
-}
-
-/// Displays the download progress for a given stream.
-pub async fn show_download_progress(
-    hash: Hash,
-    mut stream: impl Stream<Item = Result<DownloadProgress>> + Unpin,
-) -> Result<()> {
-    eprintln!("Fetching: {}", hash);
-    let mp = MultiProgress::new();
-    mp.set_draw_target(ProgressDrawTarget::stderr());
-    let op = mp.add(make_overall_progress());
-    let ip = mp.add(make_individual_progress());
-    op.set_message(format!("{} Connecting ...\n", style("[1/3]").bold().dim()));
-    let mut seq = false;
-    while let Some(x) = stream.next().await {
-        match x? {
-            DownloadProgress::InitialState(state) => {
-                if state.connected {
-                    op.set_message(format!("{} Requesting ...\n", style("[2/3]").bold().dim()));
-                }
-                if let Some(count) = state.root.child_count {
-                    op.set_message(format!(
-                        "{} Downloading {} blob(s)\n",
-                        style("[3/3]").bold().dim(),
-                        count + 1,
-                    ));
-                    op.set_length(count + 1);
-                    op.reset();
-                    op.set_position(state.current.map(u64::from).unwrap_or(0));
-                    seq = true;
-                }
-                if let Some(blob) = state.get_current() {
-                    if let Some(size) = blob.size {
-                        ip.set_length(size.value());
-                        ip.reset();
-                        match blob.progress {
-                            BlobProgress::Pending => {}
-                            BlobProgress::Progressing(offset) => ip.set_position(offset),
-                            BlobProgress::Done => ip.finish_and_clear(),
-                        }
-                        if !seq {
-                            op.finish_and_clear();
-                        }
-                    }
-                }
-            }
-            DownloadProgress::FoundLocal { .. } => {}
-            DownloadProgress::Connected => {
-                op.set_message(format!("{} Requesting ...\n", style("[2/3]").bold().dim()));
-            }
-            DownloadProgress::FoundHashSeq { children, .. } => {
-                op.set_message(format!(
-                    "{} Downloading {} blob(s)\n",
-                    style("[3/3]").bold().dim(),
-                    children + 1,
-                ));
-                op.set_length(children + 1);
-                op.reset();
-                seq = true;
-            }
-            DownloadProgress::Found { size, child, .. } => {
-                if seq {
-                    op.set_position(child.into());
-                } else {
-                    op.finish_and_clear();
-                }
-                ip.set_length(size);
-                ip.reset();
-            }
-            DownloadProgress::Progress { offset, .. } => {
-                ip.set_position(offset);
-            }
-            DownloadProgress::Done { .. } => {
-                ip.finish_and_clear();
-            }
-            DownloadProgress::AllDone(Stats {
-                bytes_read,
-                elapsed,
-                ..
-            }) => {
-                op.finish_and_clear();
-                eprintln!(
-                    "Transferred {} in {}, {}/s",
-                    HumanBytes(bytes_read),
-                    HumanDuration(elapsed),
-                    HumanBytes((bytes_read as f64 / elapsed.as_secs_f64()) as u64)
-                );
-                break;
-            }
-            DownloadProgress::Abort(e) => {
-                bail!("download aborted: {}", e);
-            }
-        }
-    }
-    Ok(())
-}
-
-/// Where the data should be stored.
-#[derive(Debug, Clone, derive_more::Display, PartialEq, Eq)]
-pub enum OutputTarget {
-    /// Writes to stdout
-    #[display("STDOUT")]
-    Stdout,
-    /// Writes to the provided path
-    #[display("{}", _0.display())]
-    Path(PathBuf),
-}
-
-impl From<String> for OutputTarget {
-    fn from(s: String) -> Self {
-        if s == "STDOUT" {
-            return OutputTarget::Stdout;
-        }
-
-        OutputTarget::Path(s.into())
-    }
-}
-
-/// Creates a [`ProgressBar`] with some defaults for the overall progress.
-fn make_overall_progress() -> ProgressBar {
-    let pb = ProgressBar::hidden();
-    pb.enable_steady_tick(std::time::Duration::from_millis(100));
-    pb.set_style(
-        ProgressStyle::with_template(
-            "{msg}{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {pos}/{len}",
-        )
-        .unwrap()
-        .progress_chars("#>-"),
-    );
-    pb
-}
-
-/// Creates a [`ProgressBar`] with some defaults for the individual progress.
-fn make_individual_progress() -> ProgressBar {
-    let pb = ProgressBar::hidden();
-    pb.enable_steady_tick(std::time::Duration::from_millis(100));
-    pb.set_style(
-        ProgressStyle::with_template("{msg}{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({eta})")
-            .unwrap()
-            .with_key(
-                "eta",
-                |state: &ProgressState, w: &mut dyn std::fmt::Write| {
-                    write!(w, "{:.1}s", state.eta().as_secs_f64()).unwrap()
-                },
-            )
-            .progress_chars("#>-"),
-    );
-    pb
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_blob_source() {
-        assert_eq!(
-            BlobSource::from(BlobSource::Stdin.to_string()),
-            BlobSource::Stdin
-        );
-
-        assert_eq!(
-            BlobSource::from(BlobSource::Path("hello/world".into()).to_string()),
-            BlobSource::Path("hello/world".into()),
-        );
-    }
-
-    #[test]
-    fn test_output_target() {
-        assert_eq!(
-            OutputTarget::from(OutputTarget::Stdout.to_string()),
-            OutputTarget::Stdout
-        );
-
-        assert_eq!(
-            OutputTarget::from(OutputTarget::Path("hello/world".into()).to_string()),
-            OutputTarget::Path("hello/world".into()),
-        );
-    }
-}
diff --git a/src/cli/tags.rs b/src/cli/tags.rs
deleted file mode 100644
index 8bd723615..000000000
--- a/src/cli/tags.rs
+++ /dev/null
@@ -1,46 +0,0 @@
-//! Define the tags subcommand.
-
-use anyhow::Result;
-use bytes::Bytes;
-use clap::Subcommand;
-use futures_lite::StreamExt;
-
-use crate::{rpc::client::tags, Tag};
-
-/// Commands to manage tags.
-#[derive(Subcommand, Debug, Clone)]
-#[allow(clippy::large_enum_variant)]
-pub enum TagCommands {
-    /// List all tags
-    List,
-    /// Delete a tag
-    Delete {
-        tag: String,
-        #[clap(long, default_value_t = false)]
-        hex: bool,
-    },
-}
-
-impl TagCommands {
-    /// Runs the tag command given the iroh client.
-    pub async fn run(self, tags: &tags::Client) -> Result<()> {
-        match self {
-            Self::List => {
-                let mut response = tags.list().await?;
-                while let Some(res) = response.next().await {
-                    let res = res?;
-                    println!("{}: {} ({:?})", res.name, res.hash, res.format);
-                }
-            }
-            Self::Delete { tag, hex } => {
-                let tag = if hex {
-                    Tag::from(Bytes::from(hex::decode(tag)?))
-                } else {
-                    Tag::from(tag)
-                };
-                tags.delete(tag).await?;
-            }
-        }
-        Ok(())
-    }
-}
diff --git a/src/downloader.rs b/src/downloader.rs
deleted file mode 100644
index e6a04508b..000000000
--- a/src/downloader.rs
+++ /dev/null
@@ -1,1621 +0,0 @@
-//! Handle downloading blobs and collections concurrently and from nodes.
-//!
-//! The [`Downloader`] interacts with four main components to this end.
-//! - `ProviderMap`: Where the downloader obtains information about nodes that could be
-//!   used to perform a download.
-//! - [`Store`]: Where data is stored.
-//!
-//! Once a download request is received, the logic is as follows:
-//! 1. The `ProviderMap` is queried for nodes. From these nodes some are selected
-//!    prioritizing connected nodes with lower number of active requests. If no useful node is
-//!    connected, or useful connected nodes have no capacity to perform the request, a connection
-//!    attempt is started using the `DialerT`.
-//! 2. The download is queued for processing at a later time. Downloads are not performed right
-//!    away. Instead, they are initially delayed to allow the node to obtain the data itself, and
-//!    to wait for the new connection to be established if necessary.
-//! 3. Once a request is ready to be sent after a delay (initial or for a retry), the preferred
-//!    node is used if available. The request is now considered active.
-//!
-//! Concurrency is limited in different ways:
-//! - *Total number of active request:* This is a way to prevent a self DoS by overwhelming our own
-//!   bandwidth capacity. This is a best effort heuristic since it doesn't take into account how
-//!   much data we are actually requesting or receiving.
-//! - *Total number of connected nodes:* Peer connections are kept for a longer time than they are
-//!   strictly needed since it's likely they will be useful soon again.
-//! - *Requests per node*: to avoid overwhelming nodes with requests, the number of concurrent
-//!   requests to a single node is also limited.
-
-use std::{
-    collections::{
-        hash_map::{self, Entry},
-        HashMap, HashSet,
-    },
-    fmt,
-    future::Future,
-    num::NonZeroUsize,
-    pin::Pin,
-    sync::{
-        atomic::{AtomicU64, Ordering},
-        Arc,
-    },
-    task::Poll,
-    time::Duration,
-};
-
-use anyhow::anyhow;
-use futures_lite::{future::BoxedLocal, Stream, StreamExt};
-use hashlink::LinkedHashSet;
-use iroh::{endpoint, Endpoint, NodeAddr, NodeId};
-use tokio::{
-    sync::{mpsc, oneshot},
-    task::JoinSet,
-};
-use tokio_util::{either::Either, sync::CancellationToken, time::delay_queue};
-use tracing::{debug, error, error_span, trace, warn, Instrument};
-
-use crate::{
-    get::{db::DownloadProgress, error::GetError, Stats},
-    metrics::Metrics,
-    store::Store,
-    util::{local_pool::LocalPoolHandle, progress::ProgressSender},
-    BlobFormat, Hash, HashAndFormat,
-};
-
-mod get;
-mod invariants;
-mod progress;
-mod test;
-
-use self::progress::{BroadcastProgressSender, ProgressSubscriber, ProgressTracker};
-
-/// Duration for which we keep nodes connected after they were last useful to us.
-const IDLE_PEER_TIMEOUT: Duration = Duration::from_secs(10);
-/// Capacity of the channel used to communicate between the [`Downloader`] and the [`Service`].
-const SERVICE_CHANNEL_CAPACITY: usize = 128;
-
-/// Identifier for a download intent.
-#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, derive_more::Display)]
-pub struct IntentId(pub u64);
-
-/// Trait modeling a dialer. This allows for IO-less testing.
-trait DialerT: Stream<Item = (NodeId, anyhow::Result<Self::Connection>)> + Unpin {
-    /// Type of connections returned by the Dialer.
-    type Connection: Clone + 'static;
-    /// Dial a node.
-    fn queue_dial(&mut self, node_id: NodeId);
-    /// Get the number of dialing nodes.
-    fn pending_count(&self) -> usize;
-    /// Check if a node is being dialed.
-    fn is_pending(&self, node: NodeId) -> bool;
-    /// Get the node id of our node.
-    fn node_id(&self) -> NodeId;
-}
-
-/// Signals what should be done with the request when it fails.
-#[derive(Debug)]
-pub enum FailureAction {
-    /// The request was cancelled by us.
-    AllIntentsDropped,
-    /// An error occurred that prevents the request from being retried at all.
-    AbortRequest(GetError),
-    /// An error occurred that suggests the node should not be used in general.
-    DropPeer(anyhow::Error),
-    /// An error occurred in which neither the node nor the request are at fault.
-    RetryLater(anyhow::Error),
-}
-
-/// Future of a get request, for the checking stage.
-type GetStartFut<N> = BoxedLocal<Result<GetOutput<N>, FailureAction>>;
-/// Future of a get request, for the downloading stage.
-type GetProceedFut = BoxedLocal<InternalDownloadResult>;
-
-/// Trait modelling performing a single request over a connection. This allows for IO-less testing.
-pub trait Getter {
-    /// Type of connections the Getter requires to perform a download.
-    type Connection: 'static;
-    /// Type of the intermediary state returned from [`Self::get`] if a connection is needed.
-    type NeedsConn: NeedsConn<Self::Connection>;
-    /// Returns a future that checks the local store if the request is already complete, returning
-    /// a struct implementing [`NeedsConn`] if we need a network connection to proceed.
-    fn get(
-        &mut self,
-        kind: DownloadKind,
-        progress_sender: BroadcastProgressSender,
-    ) -> GetStartFut<Self::NeedsConn>;
-}
-
-/// Trait modelling the intermediary state when a connection is needed to proceed.
-pub trait NeedsConn<C>: std::fmt::Debug + 'static {
-    /// Proceeds the download with the given connection.
-    fn proceed(self, conn: C) -> GetProceedFut;
-}
-
-/// Output returned from [`Getter::get`].
-#[derive(Debug)]
-pub enum GetOutput<N> {
-    /// The request is already complete in the local store.
-    Complete(Stats),
-    /// The request needs a connection to continue.
-    NeedsConn(N),
-}
-
-/// Concurrency limits for the [`Downloader`].
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub struct ConcurrencyLimits {
-    /// Maximum number of requests the service performs concurrently.
-    pub max_concurrent_requests: usize,
-    /// Maximum number of requests performed by a single node concurrently.
-    pub max_concurrent_requests_per_node: usize,
-    /// Maximum number of open connections the service maintains.
-    pub max_open_connections: usize,
-    /// Maximum number of nodes to dial concurrently for a single request.
-    pub max_concurrent_dials_per_hash: usize,
-}
-
-impl Default for ConcurrencyLimits {
-    fn default() -> Self {
-        // these numbers should be checked against a running node and might depend on platform
-        ConcurrencyLimits {
-            max_concurrent_requests: 50,
-            max_concurrent_requests_per_node: 4,
-            max_open_connections: 25,
-            max_concurrent_dials_per_hash: 5,
-        }
-    }
-}
-
-impl ConcurrencyLimits {
-    /// Checks if the maximum number of concurrent requests has been reached.
-    fn at_requests_capacity(&self, active_requests: usize) -> bool {
-        active_requests >= self.max_concurrent_requests
-    }
-
-    /// Checks if the maximum number of concurrent requests per node has been reached.
-    fn node_at_request_capacity(&self, active_node_requests: usize) -> bool {
-        active_node_requests >= self.max_concurrent_requests_per_node
-    }
-
-    /// Checks if the maximum number of connections has been reached.
-    fn at_connections_capacity(&self, active_connections: usize) -> bool {
-        active_connections >= self.max_open_connections
-    }
-
-    /// Checks if the maximum number of concurrent dials per hash has been reached.
-    ///
-    /// Note that this limit is not strictly enforced, and not checked in
-    /// [`Service::check_invariants`]. A certain hash can exceed this limit in a valid way if some
-    /// of its providers are dialed for another hash. However, once the limit is reached,
-    /// no new dials will be initiated for the hash.
-    fn at_dials_per_hash_capacity(&self, concurrent_dials: usize) -> bool {
-        concurrent_dials >= self.max_concurrent_dials_per_hash
-    }
-}
-
-/// Configuration for retry behavior of the [`Downloader`].
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub struct RetryConfig {
-    /// Maximum number of retry attempts for a node that failed to dial or failed with IO errors.
-    pub max_retries_per_node: u32,
-    /// The initial delay to wait before retrying a node. On subsequent failures, the retry delay
-    /// will be multiplied with the number of failed retries.
-    pub initial_retry_delay: Duration,
-}
-
-impl Default for RetryConfig {
-    fn default() -> Self {
-        Self {
-            max_retries_per_node: 6,
-            initial_retry_delay: Duration::from_millis(500),
-        }
-    }
-}
-
-/// A download request.
-#[derive(Debug, Clone)]
-pub struct DownloadRequest {
-    kind: DownloadKind,
-    nodes: Vec<NodeAddr>,
-    progress: Option<ProgressSubscriber>,
-}
-
-impl DownloadRequest {
-    /// Create a new download request.
-    ///
-    /// It is the responsibility of the caller to ensure that the data is tagged either with a
-    /// temp tag or with a persistent tag to make sure the data is not garbage collected during
-    /// the download.
-    ///
-    /// If this is not done, there download will proceed as normal, but there is no guarantee
-    /// that the data is still available when the download is complete.
-    pub fn new(
-        resource: impl Into<DownloadKind>,
-        nodes: impl IntoIterator<Item = impl Into<NodeAddr>>,
-    ) -> Self {
-        Self {
-            kind: resource.into(),
-            nodes: nodes.into_iter().map(|n| n.into()).collect(),
-            progress: None,
-        }
-    }
-
-    /// Pass a progress sender to receive progress updates.
-    pub fn progress_sender(mut self, sender: ProgressSubscriber) -> Self {
-        self.progress = Some(sender);
-        self
-    }
-}
-
-/// The kind of resource to download.
-#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy, derive_more::From, derive_more::Into)]
-pub struct DownloadKind(HashAndFormat);
-
-impl DownloadKind {
-    /// Get the hash of this download
-    pub const fn hash(&self) -> Hash {
-        self.0.hash
-    }
-
-    /// Get the format of this download
-    pub const fn format(&self) -> BlobFormat {
-        self.0.format
-    }
-
-    /// Get the [`HashAndFormat`] pair of this download
-    pub const fn hash_and_format(&self) -> HashAndFormat {
-        self.0
-    }
-}
-
-impl fmt::Display for DownloadKind {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}:{:?}", self.0.hash.fmt_short(), self.0.format)
-    }
-}
-
-/// The result of a download request, as returned to the application code.
-type ExternalDownloadResult = Result<Stats, DownloadError>;
-
-/// The result of a download request, as used in this module.
-type InternalDownloadResult = Result<Stats, FailureAction>;
-
-/// Error returned when a download could not be completed.
-#[derive(Debug, Clone, thiserror::Error)]
-pub enum DownloadError {
-    /// Failed to download from any provider
-    #[error("Failed to complete download")]
-    DownloadFailed,
-    /// The download was cancelled by us
-    #[error("Download cancelled by us")]
-    Cancelled,
-    /// No provider nodes found
-    #[error("No provider nodes found")]
-    NoProviders,
-    /// Failed to receive response from service.
-    #[error("Failed to receive response from download service")]
-    ActorClosed,
-}
-
-/// Handle to interact with a download request.
-#[derive(Debug)]
-pub struct DownloadHandle {
-    /// Id used to identify the request in the [`Downloader`].
-    id: IntentId,
-    /// Kind of download.
-    kind: DownloadKind,
-    /// Receiver to retrieve the return value of this download.
-    receiver: oneshot::Receiver<ExternalDownloadResult>,
-}
-
-impl Future for DownloadHandle {
-    type Output = ExternalDownloadResult;
-
-    fn poll(
-        mut self: std::pin::Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-    ) -> std::task::Poll<Self::Output> {
-        use std::task::Poll::*;
-        // make it easier on holders of the handle to poll the result, removing the receiver error
-        // from the middle
-        match std::pin::Pin::new(&mut self.receiver).poll(cx) {
-            Ready(Ok(result)) => Ready(result),
-            Ready(Err(_recv_err)) => Ready(Err(DownloadError::ActorClosed)),
-            Pending => Pending,
-        }
-    }
-}
-
-/// All numerical config options for the downloader.
-#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
-pub struct Config {
-    /// Concurrency limits for the downloader.
-    pub concurrency: ConcurrencyLimits,
-    /// Retry configuration for the downloader.
-    pub retry: RetryConfig,
-}
-
-/// Handle for the download services.
-#[derive(Debug, Clone)]
-pub struct Downloader {
-    inner: Arc<Inner>,
-}
-
-#[derive(Debug)]
-struct Inner {
-    /// Next id to use for a download intent.
-    next_id: AtomicU64,
-    /// Channel to communicate with the service.
-    msg_tx: mpsc::Sender<Message>,
-    /// Configuration for the downloader.
-    config: Arc<Config>,
-    metrics: Arc<Metrics>,
-}
-
-impl Downloader {
-    /// Create a new Downloader with the default [`ConcurrencyLimits`] and [`RetryConfig`].
-    pub fn new<S>(store: S, endpoint: Endpoint, rt: LocalPoolHandle) -> Self
-    where
-        S: Store,
-    {
-        Self::with_config(store, endpoint, rt, Default::default())
-    }
-
-    /// Create a new Downloader with custom [`ConcurrencyLimits`] and [`RetryConfig`].
-    pub fn with_config<S>(store: S, endpoint: Endpoint, rt: LocalPoolHandle, config: Config) -> Self
-    where
-        S: Store,
-    {
-        let metrics = Arc::new(Metrics::default());
-        let metrics2 = metrics.clone();
-        let me = endpoint.node_id().fmt_short();
-        let (msg_tx, msg_rx) = mpsc::channel(SERVICE_CHANNEL_CAPACITY);
-        let dialer = Dialer::new(endpoint);
-        let config = Arc::new(config);
-        let config2 = config.clone();
-        let create_future = move || {
-            let getter = get::IoGetter {
-                store: store.clone(),
-            };
-            let service = Service::new(getter, dialer, config2, msg_rx, metrics2);
-            service.run().instrument(error_span!("downloader", %me))
-        };
-        rt.spawn_detached(create_future);
-        Self {
-            inner: Arc::new(Inner {
-                next_id: AtomicU64::new(0),
-                msg_tx,
-                config,
-                metrics,
-            }),
-        }
-    }
-
-    /// Get the current configuration.
-    pub fn config(&self) -> &Config {
-        &self.inner.config
-    }
-
-    /// Queue a download.
-    pub async fn queue(&self, request: DownloadRequest) -> DownloadHandle {
-        let kind = request.kind;
-        let intent_id = IntentId(self.inner.next_id.fetch_add(1, Ordering::SeqCst));
-        let (sender, receiver) = oneshot::channel();
-        let handle = DownloadHandle {
-            id: intent_id,
-            kind,
-            receiver,
-        };
-        let msg = Message::Queue {
-            on_finish: sender,
-            request,
-            intent_id,
-        };
-        // if this fails polling the handle will fail as well since the sender side of the oneshot
-        // will be dropped
-        if let Err(send_err) = self.inner.msg_tx.send(msg).await {
-            let msg = send_err.0;
-            debug!(?msg, "download not sent");
-        }
-        handle
-    }
-
-    /// Cancel a download.
-    // NOTE: receiving the handle ensures an intent can't be cancelled twice
-    pub async fn cancel(&self, handle: DownloadHandle) {
-        let DownloadHandle {
-            id,
-            kind,
-            receiver: _,
-        } = handle;
-        let msg = Message::CancelIntent { id, kind };
-        if let Err(send_err) = self.inner.msg_tx.send(msg).await {
-            let msg = send_err.0;
-            debug!(?msg, "cancel not sent");
-        }
-    }
-
-    /// Declare that certain nodes can be used to download a hash.
-    ///
-    /// Note that this does not start a download, but only provides new nodes to already queued
-    /// downloads. Use [`Self::queue`] to queue a download.
-    pub async fn nodes_have(&mut self, hash: Hash, nodes: Vec<NodeId>) {
-        let msg = Message::NodesHave { hash, nodes };
-        if let Err(send_err) = self.inner.msg_tx.send(msg).await {
-            let msg = send_err.0;
-            debug!(?msg, "nodes have not been sent")
-        }
-    }
-
-    /// Returns the metrics collected for this downloader.
-    pub fn metrics(&self) -> &Arc<Metrics> {
-        &self.inner.metrics
-    }
-}
-
-/// Messages the service can receive.
-#[derive(derive_more::Debug)]
-enum Message {
-    /// Queue a download intent.
-    Queue {
-        request: DownloadRequest,
-        #[debug(skip)]
-        on_finish: oneshot::Sender<ExternalDownloadResult>,
-        intent_id: IntentId,
-    },
-    /// Declare that nodes have a certain hash and can be used for downloading.
-    NodesHave { hash: Hash, nodes: Vec<NodeId> },
-    /// Cancel an intent. The associated request will be cancelled when the last intent is
-    /// cancelled.
-    CancelIntent { id: IntentId, kind: DownloadKind },
-}
-
-#[derive(derive_more::Debug)]
-struct IntentHandlers {
-    #[debug("oneshot::Sender<DownloadResult>")]
-    on_finish: oneshot::Sender<ExternalDownloadResult>,
-    on_progress: Option<ProgressSubscriber>,
-}
-
-/// Information about a request.
-#[derive(Debug)]
-struct RequestInfo<NC> {
-    /// Registered intents with progress senders and result callbacks.
-    intents: HashMap<IntentId, IntentHandlers>,
-    progress_sender: BroadcastProgressSender,
-    get_state: Option<NC>,
-}
-
-/// Information about a request in progress.
-#[derive(derive_more::Debug)]
-struct ActiveRequestInfo {
-    /// Token used to cancel the future doing the request.
-    #[debug(skip)]
-    cancellation: CancellationToken,
-    /// Peer doing this request attempt.
-    node: NodeId,
-}
-
-#[derive(Debug, Default)]
-struct RetryState {
-    /// How many times did we retry this node?
-    retry_count: u32,
-    /// Whether the node is currently queued for retry.
-    retry_is_queued: bool,
-}
-
-/// State of the connection to this node.
-#[derive(derive_more::Debug)]
-struct ConnectionInfo<Conn> {
-    /// Connection to this node.
-    #[debug(skip)]
-    conn: Conn,
-    /// State of this node.
-    state: ConnectedState,
-}
-
-impl<Conn> ConnectionInfo<Conn> {
-    /// Create a new idle node.
-    fn new_idle(connection: Conn, drop_key: delay_queue::Key) -> Self {
-        ConnectionInfo {
-            conn: connection,
-            state: ConnectedState::Idle { drop_key },
-        }
-    }
-
-    /// Count of active requests for the node.
-    fn active_requests(&self) -> usize {
-        match self.state {
-            ConnectedState::Busy { active_requests } => active_requests.get(),
-            ConnectedState::Idle { .. } => 0,
-        }
-    }
-
-    /// Returns `true` if the node is currently idle.
-    fn is_idle(&self) -> bool {
-        matches!(self.state, ConnectedState::Idle { .. })
-    }
-}
-
-/// State of a connected node.
-#[derive(derive_more::Debug)]
-enum ConnectedState {
-    /// Peer is handling at least one request.
-    Busy {
-        #[debug("{}", active_requests.get())]
-        active_requests: NonZeroUsize,
-    },
-    /// Peer is idle.
-    Idle {
-        #[debug(skip)]
-        drop_key: delay_queue::Key,
-    },
-}
-
-#[derive(Debug)]
-enum NodeState<'a, Conn> {
-    Connected(&'a ConnectionInfo<Conn>),
-    Dialing,
-    WaitForRetry,
-    Disconnected,
-}
-
-#[derive(Debug)]
-struct Service<G: Getter, D: DialerT> {
-    /// The getter performs individual requests.
-    getter: G,
-    /// Map to query for nodes that we believe have the data we are looking for.
-    providers: ProviderMap,
-    /// Dialer to get connections for required nodes.
-    dialer: D,
-    /// Limits to concurrent tasks handled by the service.
-    concurrency_limits: ConcurrencyLimits,
-    /// Configuration for retry behavior.
-    retry_config: RetryConfig,
-    /// Channel to receive messages from the service's handle.
-    msg_rx: mpsc::Receiver<Message>,
-    /// Nodes to which we have an active or idle connection.
-    connected_nodes: HashMap<NodeId, ConnectionInfo<D::Connection>>,
-    /// We track a retry state for nodes which failed to dial or in a transfer.
-    retry_node_state: HashMap<NodeId, RetryState>,
-    /// Delay queue for retrying failed nodes.
-    retry_nodes_queue: delay_queue::DelayQueue<NodeId>,
-    /// Delay queue for dropping idle nodes.
-    goodbye_nodes_queue: delay_queue::DelayQueue<NodeId>,
-    /// Queue of pending downloads.
-    queue: Queue,
-    /// Information about pending and active requests.
-    requests: HashMap<DownloadKind, RequestInfo<G::NeedsConn>>,
-    /// State of running downloads.
-    active_requests: HashMap<DownloadKind, ActiveRequestInfo>,
-    /// Tasks for currently running downloads.
-    in_progress_downloads: JoinSet<(DownloadKind, InternalDownloadResult)>,
-    /// Progress tracker
-    progress_tracker: ProgressTracker,
-    metrics: Arc<Metrics>,
-}
-impl<G: Getter<Connection = D::Connection>, D: DialerT> Service<G, D> {
-    fn new(
-        getter: G,
-        dialer: D,
-        config: Arc<Config>,
-        msg_rx: mpsc::Receiver<Message>,
-        metrics: Arc<Metrics>,
-    ) -> Self {
-        Service {
-            getter,
-            dialer,
-            msg_rx,
-            concurrency_limits: config.concurrency,
-            retry_config: config.retry,
-            connected_nodes: Default::default(),
-            retry_node_state: Default::default(),
-            providers: Default::default(),
-            requests: Default::default(),
-            retry_nodes_queue: delay_queue::DelayQueue::default(),
-            goodbye_nodes_queue: delay_queue::DelayQueue::default(),
-            active_requests: Default::default(),
-            in_progress_downloads: Default::default(),
-            progress_tracker: ProgressTracker::new(),
-            queue: Default::default(),
-            metrics,
-        }
-    }
-
-    /// Main loop for the service.
-    async fn run(mut self) {
-        loop {
-            trace!("wait for tick");
-            self.metrics.downloader_tick_main.inc();
-            tokio::select! {
-                Some((node, conn_result)) = self.dialer.next() => {
-                    trace!(node=%node.fmt_short(), "tick: connection ready");
-                    self.metrics.downloader_tick_connection_ready.inc();
-                    self.on_connection_ready(node, conn_result);
-                }
-                maybe_msg = self.msg_rx.recv() => {
-                    trace!(msg=?maybe_msg, "tick: message received");
-                    self.metrics.downloader_tick_message_received.inc();
-                    match maybe_msg {
-                        Some(msg) => self.handle_message(msg).await,
-                        None => return self.shutdown().await,
-                    }
-                }
-                Some(res) = self.in_progress_downloads.join_next(), if !self.in_progress_downloads.is_empty() => {
-                    match res {
-                        Ok((kind, result)) => {
-                            trace!(%kind, "tick: transfer completed");
-                            self::get::track_metrics(&result, &self.metrics);
-                            self.metrics.downloader_tick_transfer_completed.inc();
-                            self.on_download_completed(kind, result);
-                        }
-                        Err(err) => {
-                            warn!(?err, "transfer task panicked");
-                            self.metrics.downloader_tick_transfer_failed.inc();
-                        }
-                    }
-                }
-                Some(expired) = self.retry_nodes_queue.next() => {
-                    let node = expired.into_inner();
-                    trace!(node=%node.fmt_short(), "tick: retry node");
-                    self.metrics.downloader_tick_retry_node.inc();
-                    self.on_retry_wait_elapsed(node);
-                }
-                Some(expired) = self.goodbye_nodes_queue.next() => {
-                    let node = expired.into_inner();
-                    trace!(node=%node.fmt_short(), "tick: goodbye node");
-                    self.metrics.downloader_tick_goodbye_node.inc();
-                    self.disconnect_idle_node(node, "idle expired");
-                }
-            }
-
-            self.process_head();
-
-            #[cfg(any(test, debug_assertions))]
-            self.check_invariants();
-        }
-    }
-
-    /// Handle receiving a [`Message`].
-    // This is called in the actor loop, and only async because subscribing to an existing transfer
-    // sends the initial state.
-    async fn handle_message(&mut self, msg: Message) {
-        match msg {
-            Message::Queue {
-                request,
-                on_finish,
-                intent_id,
-            } => {
-                self.handle_queue_new_download(request, intent_id, on_finish)
-                    .await
-            }
-            Message::CancelIntent { id, kind } => self.handle_cancel_download(id, kind).await,
-            Message::NodesHave { hash, nodes } => {
-                let updated = self
-                    .providers
-                    .add_nodes_if_hash_exists(hash, nodes.iter().cloned());
-                if updated {
-                    self.queue.unpark_hash(hash);
-                }
-            }
-        }
-    }
-
-    /// Handle a [`Message::Queue`].
-    ///
-    /// If this intent maps to a request that already exists, it will be registered with it. If the
-    /// request is new it will be scheduled.
-    async fn handle_queue_new_download(
-        &mut self,
-        request: DownloadRequest,
-        intent_id: IntentId,
-        on_finish: oneshot::Sender<ExternalDownloadResult>,
-    ) {
-        let DownloadRequest {
-            kind,
-            nodes,
-            progress,
-        } = request;
-        debug!(%kind, nodes=?nodes.iter().map(|n| n.node_id.fmt_short()).collect::<Vec<_>>(), "queue intent");
-
-        // store the download intent
-        let intent_handlers = IntentHandlers {
-            on_finish,
-            on_progress: progress,
-        };
-
-        // add the nodes to the provider map
-        // (skip the node id of our own node - we should never attempt to download from ourselves)
-        let node_ids = nodes
-            .iter()
-            .map(|n| n.node_id)
-            .filter(|node_id| *node_id != self.dialer.node_id());
-        let updated = self.providers.add_hash_with_nodes(kind.hash(), node_ids);
-
-        // queue the transfer (if not running) or attach to transfer progress (if already running)
-        match self.requests.entry(kind) {
-            hash_map::Entry::Occupied(mut entry) => {
-                if let Some(on_progress) = &intent_handlers.on_progress {
-                    // this is async because it sends the current state over the progress channel
-                    if let Err(err) = self
-                        .progress_tracker
-                        .subscribe(kind, on_progress.clone())
-                        .await
-                    {
-                        debug!(?err, %kind, "failed to subscribe progress sender to transfer");
-                    }
-                }
-                entry.get_mut().intents.insert(intent_id, intent_handlers);
-            }
-            hash_map::Entry::Vacant(entry) => {
-                let progress_sender = self.progress_tracker.track(
-                    kind,
-                    intent_handlers
-                        .on_progress
-                        .clone()
-                        .into_iter()
-                        .collect::<Vec<_>>(),
-                );
-
-                let get_state = match self.getter.get(kind, progress_sender.clone()).await {
-                    Err(err) => {
-                        // This prints a "FailureAction" which is somewhat weird, but that's all we get here.
-                        tracing::error!(?err, "failed queuing new download");
-                        self.finalize_download(
-                            kind,
-                            [(intent_id, intent_handlers)].into(),
-                            // TODO: add better error variant? this is only triggered if the local
-                            // store failed with local IO.
-                            Err(DownloadError::DownloadFailed),
-                        );
-                        return;
-                    }
-                    Ok(GetOutput::Complete(stats)) => {
-                        self.finalize_download(
-                            kind,
-                            [(intent_id, intent_handlers)].into(),
-                            Ok(stats),
-                        );
-                        return;
-                    }
-                    Ok(GetOutput::NeedsConn(state)) => {
-                        // early exit if no providers.
-                        if self.providers.get_candidates(&kind.hash()).next().is_none() {
-                            self.finalize_download(
-                                kind,
-                                [(intent_id, intent_handlers)].into(),
-                                Err(DownloadError::NoProviders),
-                            );
-                            return;
-                        }
-                        state
-                    }
-                };
-                entry.insert(RequestInfo {
-                    intents: [(intent_id, intent_handlers)].into_iter().collect(),
-                    progress_sender,
-                    get_state: Some(get_state),
-                });
-                self.queue.insert(kind);
-            }
-        }
-
-        if updated && self.queue.is_parked(&kind) {
-            // the transfer is on hold for pending retries, and we added new nodes, so move back to queue.
-            self.queue.unpark(&kind);
-        }
-    }
-
-    /// Cancels a download intent.
-    ///
-    /// This removes the intent from the list of intents for the `kind`. If the removed intent was
-    /// the last one for the `kind`, this means that the download is no longer needed. In this
-    /// case, the `kind` will be removed from the list of pending downloads - and, if the download was
-    /// already started, the download task will be cancelled.
-    ///
-    /// The method is async because it will send a final abort event on the progress sender.
-    async fn handle_cancel_download(&mut self, intent_id: IntentId, kind: DownloadKind) {
-        let Entry::Occupied(mut occupied_entry) = self.requests.entry(kind) else {
-            warn!(%kind, %intent_id, "cancel download called for unknown download");
-            return;
-        };
-
-        let request_info = occupied_entry.get_mut();
-        if let Some(handlers) = request_info.intents.remove(&intent_id) {
-            handlers.on_finish.send(Err(DownloadError::Cancelled)).ok();
-
-            if let Some(sender) = handlers.on_progress {
-                self.progress_tracker.unsubscribe(&kind, &sender);
-                sender
-                    .send(DownloadProgress::Abort(serde_error::Error::new(
-                        &*anyhow::Error::from(DownloadError::Cancelled),
-                    )))
-                    .await
-                    .ok();
-            }
-        }
-
-        if request_info.intents.is_empty() {
-            occupied_entry.remove();
-            if let Entry::Occupied(occupied_entry) = self.active_requests.entry(kind) {
-                occupied_entry.remove().cancellation.cancel();
-            } else {
-                self.queue.remove(&kind);
-            }
-            self.remove_hash_if_not_queued(&kind.hash());
-        }
-    }
-
-    /// Handle receiving a new connection.
-    fn on_connection_ready(&mut self, node: NodeId, result: anyhow::Result<D::Connection>) {
-        debug_assert!(
-            !self.connected_nodes.contains_key(&node),
-            "newly connected node is not yet connected"
-        );
-        match result {
-            Ok(connection) => {
-                trace!(node=%node.fmt_short(), "connected to node");
-                let drop_key = self.goodbye_nodes_queue.insert(node, IDLE_PEER_TIMEOUT);
-                self.connected_nodes
-                    .insert(node, ConnectionInfo::new_idle(connection, drop_key));
-            }
-            Err(err) => {
-                debug!(%node, %err, "connection to node failed");
-                self.disconnect_and_retry(node);
-            }
-        }
-    }
-
-    fn on_download_completed(&mut self, kind: DownloadKind, result: InternalDownloadResult) {
-        // first remove the request
-        let active_request_info = self
-            .active_requests
-            .remove(&kind)
-            .expect("request was active");
-
-        // get general request info
-        let request_info = self.requests.remove(&kind).expect("request was active");
-
-        let ActiveRequestInfo { node, .. } = active_request_info;
-
-        // get node info
-        let node_info = self
-            .connected_nodes
-            .get_mut(&node)
-            .expect("node exists in the mapping");
-
-        // update node busy/idle state
-        node_info.state = match NonZeroUsize::new(node_info.active_requests() - 1) {
-            None => {
-                // last request of the node was this one, switch to idle
-                let drop_key = self.goodbye_nodes_queue.insert(node, IDLE_PEER_TIMEOUT);
-                ConnectedState::Idle { drop_key }
-            }
-            Some(active_requests) => ConnectedState::Busy { active_requests },
-        };
-
-        match &result {
-            Ok(_) => {
-                debug!(%kind, node=%node.fmt_short(), "download successful");
-                // clear retry state if operation was successful
-                self.retry_node_state.remove(&node);
-            }
-            Err(FailureAction::AllIntentsDropped) => {
-                debug!(%kind, node=%node.fmt_short(), "download cancelled");
-            }
-            Err(FailureAction::AbortRequest(reason)) => {
-                debug!(%kind, node=%node.fmt_short(), %reason, "download failed: abort request");
-                // do not try to download the hash from this node again
-                self.providers.remove_hash_from_node(&kind.hash(), &node);
-            }
-            Err(FailureAction::DropPeer(reason)) => {
-                debug!(%kind, node=%node.fmt_short(), %reason, "download failed: drop node");
-                if node_info.is_idle() {
-                    // remove the node
-                    self.remove_node(node, "explicit drop");
-                } else {
-                    // do not try to download the hash from this node again
-                    self.providers.remove_hash_from_node(&kind.hash(), &node);
-                }
-            }
-            Err(FailureAction::RetryLater(reason)) => {
-                debug!(%kind, node=%node.fmt_short(), %reason, "download failed: retry later");
-                if node_info.is_idle() {
-                    self.disconnect_and_retry(node);
-                }
-            }
-        };
-
-        // we finalize the download if either the download was successful,
-        // or if it should never proceed because all intents were dropped,
-        // or if we don't have any candidates to proceed with anymore.
-        let finalize = match &result {
-            Ok(_) | Err(FailureAction::AllIntentsDropped) => true,
-            _ => !self.providers.has_candidates(&kind.hash()),
-        };
-
-        if finalize {
-            let result = result.map_err(|_| DownloadError::DownloadFailed);
-            self.finalize_download(kind, request_info.intents, result);
-        } else {
-            // reinsert the download at the front of the queue to try from the next node
-            self.requests.insert(kind, request_info);
-            self.queue.insert_front(kind);
-        }
-    }
-
-    /// Finalize a download.
-    ///
-    /// This triggers the intent return channels, and removes the download from the progress tracker
-    /// and provider map.
-    fn finalize_download(
-        &mut self,
-        kind: DownloadKind,
-        intents: HashMap<IntentId, IntentHandlers>,
-        result: ExternalDownloadResult,
-    ) {
-        self.progress_tracker.remove(&kind);
-        self.remove_hash_if_not_queued(&kind.hash());
-        for (_id, handlers) in intents.into_iter() {
-            handlers.on_finish.send(result.clone()).ok();
-        }
-    }
-
-    fn on_retry_wait_elapsed(&mut self, node: NodeId) {
-        // check if the node is still needed
-        let Some(hashes) = self.providers.node_hash.get(&node) else {
-            self.retry_node_state.remove(&node);
-            return;
-        };
-        let Some(state) = self.retry_node_state.get_mut(&node) else {
-            warn!(node=%node.fmt_short(), "missing retry state for node ready for retry");
-            return;
-        };
-        state.retry_is_queued = false;
-        for hash in hashes {
-            self.queue.unpark_hash(*hash);
-        }
-    }
-
-    /// Start the next downloads, or dial nodes, if limits permit and the queue is non-empty.
-    ///
-    /// This is called after all actions. If there is nothing to do, it will return cheaply.
-    /// Otherwise, we will check the next hash in the queue, and:
-    /// * start the transfer if we are connected to a provider and limits are ok
-    /// * or, connect to a provider, if there is one we are not dialing yet and limits are ok
-    /// * or, disconnect an idle node if it would allow us to connect to a provider,
-    /// * or, if all providers are waiting for retry, park the download
-    /// * or, if our limits are reached, do nothing for now
-    ///
-    /// The download requests will only be popped from the queue once we either start the transfer
-    /// from a connected node [`NextStep::StartTransfer`], or if we abort the download on
-    /// [`NextStep::OutOfProviders`]. In all other cases, the request is kept at the top of the
-    /// queue, so the next call to [`Self::process_head`] will evaluate the situation again - and
-    /// so forth, until either [`NextStep::StartTransfer`] or [`NextStep::OutOfProviders`] is
-    /// reached.
-    fn process_head(&mut self) {
-        // start as many queued downloads as allowed by the request limits.
-        loop {
-            let Some(kind) = self.queue.front().cloned() else {
-                break;
-            };
-
-            let next_step = self.next_step(&kind);
-            trace!(%kind, ?next_step, "process_head");
-
-            match next_step {
-                NextStep::Wait => break,
-                NextStep::StartTransfer(node) => {
-                    let _ = self.queue.pop_front();
-                    debug!(%kind, node=%node.fmt_short(), "start transfer");
-                    self.start_download(kind, node);
-                }
-                NextStep::Dial(node) => {
-                    debug!(%kind, node=%node.fmt_short(), "dial node");
-                    self.dialer.queue_dial(node);
-                }
-                NextStep::DialQueuedDisconnect(node, key) => {
-                    let idle_node = self.goodbye_nodes_queue.remove(&key).into_inner();
-                    self.disconnect_idle_node(idle_node, "drop idle for new dial");
-                    debug!(%kind, node=%node.fmt_short(), idle_node=%idle_node.fmt_short(), "dial node, disconnect idle node)");
-                    self.dialer.queue_dial(node);
-                }
-                NextStep::Park => {
-                    debug!(%kind, "park download: all providers waiting for retry");
-                    self.queue.park_front();
-                }
-                NextStep::OutOfProviders => {
-                    debug!(%kind, "abort download: out of providers");
-                    let _ = self.queue.pop_front();
-                    let info = self.requests.remove(&kind).expect("queued downloads exist");
-                    self.finalize_download(kind, info.intents, Err(DownloadError::NoProviders));
-                }
-            }
-        }
-    }
-
-    /// Drop the connection to a node and insert it into the the retry queue.
-    fn disconnect_and_retry(&mut self, node: NodeId) {
-        self.disconnect_idle_node(node, "queue retry");
-        let retry_state = self.retry_node_state.entry(node).or_default();
-        retry_state.retry_count += 1;
-        if retry_state.retry_count <= self.retry_config.max_retries_per_node {
-            // node can be retried
-            debug!(node=%node.fmt_short(), retry_count=retry_state.retry_count, "queue retry");
-            let timeout = self.retry_config.initial_retry_delay * retry_state.retry_count;
-            self.retry_nodes_queue.insert(node, timeout);
-            retry_state.retry_is_queued = true;
-        } else {
-            // node is dead
-            self.remove_node(node, "retries exceeded");
-        }
-    }
-
-    /// Calculate the next step needed to proceed the download for `kind`.
-    ///
-    /// This is called once `kind` has reached the head of the queue, see [`Self::process_head`].
-    /// It can be called repeatedly, and does nothing on itself, only calculate what *should* be
-    /// done next.
-    ///
-    /// See [`NextStep`] for details on the potential next steps returned from this method.
-    fn next_step(&self, kind: &DownloadKind) -> NextStep {
-        // If the total requests capacity is reached, we have to wait until an active request
-        // completes.
-        if self
-            .concurrency_limits
-            .at_requests_capacity(self.active_requests.len())
-        {
-            return NextStep::Wait;
-        };
-
-        let mut candidates = self.providers.get_candidates(&kind.hash()).peekable();
-        // If we have no provider candidates for this download, there's nothing else we can do.
-        if candidates.peek().is_none() {
-            return NextStep::OutOfProviders;
-        }
-
-        // Track if there is provider node to which we are connected and which is not at its request capacity.
-        // If there are more than one, take the one with the least amount of running transfers.
-        let mut best_connected: Option<(NodeId, usize)> = None;
-        // Track if there is a disconnected provider node to which we can potentially connect.
-        let mut next_to_dial = None;
-        // Track the number of provider nodes that are currently being dialed.
-        let mut currently_dialing = 0;
-        // Track if we have at least one provider node which is currently at its request capacity.
-        // If this is the case, we will never return [`NextStep::OutOfProviders`] but [`NextStep::Wait`]
-        // instead, because we can still try that node once it has finished its work.
-        let mut has_exhausted_provider = false;
-        // Track if we have at least one provider node that is currently in the retry queue.
-        let mut has_retrying_provider = false;
-
-        for node in candidates {
-            match self.node_state(node) {
-                NodeState::Connected(info) => {
-                    let active_requests = info.active_requests();
-                    if self
-                        .concurrency_limits
-                        .node_at_request_capacity(active_requests)
-                    {
-                        has_exhausted_provider = true;
-                    } else {
-                        best_connected = Some(match best_connected.take() {
-                            Some(old) if old.1 <= active_requests => old,
-                            _ => (node, active_requests),
-                        });
-                    }
-                }
-                NodeState::Dialing => {
-                    currently_dialing += 1;
-                }
-                NodeState::WaitForRetry => {
-                    has_retrying_provider = true;
-                }
-                NodeState::Disconnected => {
-                    if next_to_dial.is_none() {
-                        next_to_dial = Some(node);
-                    }
-                }
-            }
-        }
-
-        let has_dialing = currently_dialing > 0;
-
-        // If we have a connected provider node with free slots, use it!
-        if let Some((node, _active_requests)) = best_connected {
-            NextStep::StartTransfer(node)
-        }
-        // If we have a node which could be dialed: Check capacity and act accordingly.
-        else if let Some(node) = next_to_dial {
-            // We check if the dial capacity for this hash is exceeded: We only start new dials for
-            // the hash if we are below the limit.
-            //
-            // If other requests trigger dials for providers of this hash, the limit may be
-            // exceeded, but then we just don't start further dials and wait until one completes.
-            let at_dial_capacity = has_dialing
-                && self
-                    .concurrency_limits
-                    .at_dials_per_hash_capacity(currently_dialing);
-            // Check if we reached the global connection limit.
-            let at_connections_capacity = self.at_connections_capacity();
-
-            // All slots are free: We can dial our candidate.
-            if !at_connections_capacity && !at_dial_capacity {
-                NextStep::Dial(node)
-            }
-            // The hash has free dial capacity, but the global connection capacity is reached.
-            // But if we have idle nodes, we will disconnect the longest idling node, and then dial our
-            // candidate.
-            else if at_connections_capacity
-                && !at_dial_capacity
-                && !self.goodbye_nodes_queue.is_empty()
-            {
-                let key = self.goodbye_nodes_queue.peek().expect("just checked");
-                NextStep::DialQueuedDisconnect(node, key)
-            }
-            // No dial capacity, and no idling nodes: We have to wait until capacity is freed up.
-            else {
-                NextStep::Wait
-            }
-        }
-        // If we have pending dials to candidates, or connected candidates which are busy
-        // with other work: Wait for one of these to become available.
-        else if has_exhausted_provider || has_dialing {
-            NextStep::Wait
-        }
-        // All providers are in the retry queue: Park this request until they can be tried again.
-        else if has_retrying_provider {
-            NextStep::Park
-        }
-        // We have no candidates left: Nothing more to do.
-        else {
-            NextStep::OutOfProviders
-        }
-    }
-
-    /// Start downloading from the given node.
-    ///
-    /// Panics if hash is not in self.requests or node is not in self.nodes.
-    fn start_download(&mut self, kind: DownloadKind, node: NodeId) {
-        let node_info = self.connected_nodes.get_mut(&node).expect("node exists");
-        let request_info = self.requests.get_mut(&kind).expect("request exists");
-        let progress = request_info.progress_sender.clone();
-        // .expect("queued state exists");
-
-        // create the active request state
-        let cancellation = CancellationToken::new();
-        let state = ActiveRequestInfo {
-            cancellation: cancellation.clone(),
-            node,
-        };
-        let conn = node_info.conn.clone();
-
-        // If this is the first provider node we try, we have an initial state
-        // from starting the generator in Self::handle_queue_new_download.
-        // If this not the first provider node we try, we have to recreate the generator, because
-        // we can only resume it once.
-        let get_state = match request_info.get_state.take() {
-            Some(state) => Either::Left(async move { Ok(GetOutput::NeedsConn(state)) }),
-            None => Either::Right(self.getter.get(kind, progress)),
-        };
-        let fut = async move {
-            // NOTE: it's an open question if we should do timeouts at this point. Considerations from @Frando:
-            // > at this stage we do not know the size of the download, so the timeout would have
-            // > to be so large that it won't be useful for non-huge downloads. At the same time,
-            // > this means that a super slow node would block a download from succeeding for a long
-            // > time, while faster nodes could be readily available.
-            // As a conclusion, timeouts should be added only after downloads are known to be bounded
-            let fut = async move {
-                match get_state.await? {
-                    GetOutput::Complete(stats) => Ok(stats),
-                    GetOutput::NeedsConn(state) => state.proceed(conn).await,
-                }
-            };
-            tokio::pin!(fut);
-            let res = tokio::select! {
-                _ = cancellation.cancelled() => Err(FailureAction::AllIntentsDropped),
-                res = &mut fut => res
-            };
-            trace!("transfer finished");
-
-            (kind, res)
-        }
-        .instrument(error_span!("transfer", %kind, node=%node.fmt_short()));
-        node_info.state = match &node_info.state {
-            ConnectedState::Busy { active_requests } => ConnectedState::Busy {
-                active_requests: active_requests.saturating_add(1),
-            },
-            ConnectedState::Idle { drop_key } => {
-                self.goodbye_nodes_queue.remove(drop_key);
-                ConnectedState::Busy {
-                    active_requests: NonZeroUsize::new(1).expect("clearly non zero"),
-                }
-            }
-        };
-        self.active_requests.insert(kind, state);
-        self.in_progress_downloads.spawn_local(fut);
-    }
-
-    fn disconnect_idle_node(&mut self, node: NodeId, reason: &'static str) -> bool {
-        if let Some(info) = self.connected_nodes.remove(&node) {
-            match info.state {
-                ConnectedState::Idle { drop_key } => {
-                    self.goodbye_nodes_queue.try_remove(&drop_key);
-                    true
-                }
-                ConnectedState::Busy { .. } => {
-                    warn!("expected removed node to be idle, but is busy (removal reason: {reason:?})");
-                    self.connected_nodes.insert(node, info);
-                    false
-                }
-            }
-        } else {
-            true
-        }
-    }
-
-    fn remove_node(&mut self, node: NodeId, reason: &'static str) {
-        debug!(node = %node.fmt_short(), %reason, "remove node");
-        if self.disconnect_idle_node(node, reason) {
-            self.providers.remove_node(&node);
-            self.retry_node_state.remove(&node);
-        }
-    }
-
-    fn node_state(&self, node: NodeId) -> NodeState<'_, D::Connection> {
-        if let Some(info) = self.connected_nodes.get(&node) {
-            NodeState::Connected(info)
-        } else if self.dialer.is_pending(node) {
-            NodeState::Dialing
-        } else {
-            match self.retry_node_state.get(&node) {
-                Some(state) if state.retry_is_queued => NodeState::WaitForRetry,
-                _ => NodeState::Disconnected,
-            }
-        }
-    }
-
-    /// Check if we have maxed our connection capacity.
-    fn at_connections_capacity(&self) -> bool {
-        self.concurrency_limits
-            .at_connections_capacity(self.connections_count())
-    }
-
-    /// Get the total number of connected and dialing nodes.
-    fn connections_count(&self) -> usize {
-        let connected_nodes = self.connected_nodes.values().count();
-        let dialing_nodes = self.dialer.pending_count();
-        connected_nodes + dialing_nodes
-    }
-
-    /// Remove a `hash` from the [`ProviderMap`], but only if [`Self::queue`] does not contain the
-    /// hash at all, even with the other [`BlobFormat`].
-    fn remove_hash_if_not_queued(&mut self, hash: &Hash) {
-        if !self.queue.contains_hash(*hash) {
-            self.providers.remove_hash(hash);
-        }
-    }
-
-    #[allow(clippy::unused_async)]
-    async fn shutdown(self) {
-        debug!("shutting down");
-        // TODO(@divma): how to make sure the download futures end gracefully?
-    }
-}
-
-/// The next step needed to continue a download.
-///
-/// See [`Service::next_step`] for details.
-#[derive(Debug)]
-enum NextStep {
-    /// Provider connection is ready, initiate the transfer.
-    StartTransfer(NodeId),
-    /// Start to dial `NodeId`.
-    ///
-    /// This means: We have no non-exhausted connection to a provider node, but a free connection slot
-    /// and a provider node we are not yet connected to.
-    Dial(NodeId),
-    /// Start to dial `NodeId`, but first disconnect the idle node behind [`delay_queue::Key`] in
-    /// [`Service::goodbye_nodes_queue`] to free up a connection slot.
-    DialQueuedDisconnect(NodeId, delay_queue::Key),
-    /// Resource limits are exhausted, do nothing for now and wait until a slot frees up.
-    Wait,
-    /// All providers are currently in a retry timeout. Park the download aside, and move
-    /// to the next download in the queue.
-    Park,
-    /// We have tried all available providers. There is nothing else to do.
-    OutOfProviders,
-}
-
-/// Map of potential providers for a hash.
-#[derive(Default, Debug)]
-struct ProviderMap {
-    hash_node: HashMap<Hash, HashSet<NodeId>>,
-    node_hash: HashMap<NodeId, HashSet<Hash>>,
-}
-
-impl ProviderMap {
-    /// Get candidates to download this hash.
-    pub fn get_candidates<'a>(&'a self, hash: &Hash) -> impl Iterator<Item = NodeId> + 'a {
-        self.hash_node
-            .get(hash)
-            .map(|nodes| nodes.iter())
-            .into_iter()
-            .flatten()
-            .copied()
-    }
-
-    /// Whether we have any candidates to download this hash.
-    pub fn has_candidates(&self, hash: &Hash) -> bool {
-        self.hash_node
-            .get(hash)
-            .map(|nodes| !nodes.is_empty())
-            .unwrap_or(false)
-    }
-
-    /// Register nodes for a hash. Should only be done for hashes we care to download.
-    ///
-    /// Returns `true` if new providers were added.
-    fn add_hash_with_nodes(&mut self, hash: Hash, nodes: impl Iterator<Item = NodeId>) -> bool {
-        let mut updated = false;
-        let hash_entry = self.hash_node.entry(hash).or_default();
-        for node in nodes {
-            updated |= hash_entry.insert(node);
-            let node_entry = self.node_hash.entry(node).or_default();
-            node_entry.insert(hash);
-        }
-        updated
-    }
-
-    /// Register nodes for a hash, but only if the hash is already in our queue.
-    ///
-    /// Returns `true` if a new node was added.
-    fn add_nodes_if_hash_exists(
-        &mut self,
-        hash: Hash,
-        nodes: impl Iterator<Item = NodeId>,
-    ) -> bool {
-        let mut updated = false;
-        if let Some(hash_entry) = self.hash_node.get_mut(&hash) {
-            for node in nodes {
-                updated |= hash_entry.insert(node);
-                let node_entry = self.node_hash.entry(node).or_default();
-                node_entry.insert(hash);
-            }
-        }
-        updated
-    }
-
-    /// Signal the registry that this hash is no longer of interest.
-    fn remove_hash(&mut self, hash: &Hash) {
-        if let Some(nodes) = self.hash_node.remove(hash) {
-            for node in nodes {
-                if let Some(hashes) = self.node_hash.get_mut(&node) {
-                    hashes.remove(hash);
-                    if hashes.is_empty() {
-                        self.node_hash.remove(&node);
-                    }
-                }
-            }
-        }
-    }
-
-    fn remove_node(&mut self, node: &NodeId) {
-        if let Some(hashes) = self.node_hash.remove(node) {
-            for hash in hashes {
-                if let Some(nodes) = self.hash_node.get_mut(&hash) {
-                    nodes.remove(node);
-                    if nodes.is_empty() {
-                        self.hash_node.remove(&hash);
-                    }
-                }
-            }
-        }
-    }
-
-    fn remove_hash_from_node(&mut self, hash: &Hash, node: &NodeId) {
-        if let Some(nodes) = self.hash_node.get_mut(hash) {
-            nodes.remove(node);
-            if nodes.is_empty() {
-                self.remove_hash(hash);
-            }
-        }
-        if let Some(hashes) = self.node_hash.get_mut(node) {
-            hashes.remove(hash);
-            if hashes.is_empty() {
-                self.remove_node(node);
-            }
-        }
-    }
-}
-
-/// The queue of requested downloads.
-///
-/// This manages two datastructures:
-/// * The main queue, a FIFO queue where each item can only appear once.
-///   New downloads are pushed to the back of the queue, and the next download to process is popped
-///   from the front.
-/// * The parked set, a hash set. Items can be moved from the main queue into the parked set.
-///   Parked items will not be popped unless they are moved back into the main queue.
-#[derive(Debug, Default)]
-struct Queue {
-    main: LinkedHashSet<DownloadKind>,
-    parked: HashSet<DownloadKind>,
-}
-
-impl Queue {
-    /// Peek at the front element of the main queue.
-    pub fn front(&self) -> Option<&DownloadKind> {
-        self.main.front()
-    }
-
-    #[cfg(any(test, debug_assertions))]
-    pub fn iter_parked(&self) -> impl Iterator<Item = &DownloadKind> {
-        self.parked.iter()
-    }
-
-    #[cfg(any(test, debug_assertions))]
-    pub fn iter(&self) -> impl Iterator<Item = &DownloadKind> {
-        self.main.iter().chain(self.parked.iter())
-    }
-
-    /// Returns `true` if either the main queue or the parked set contain a download.
-    pub fn contains(&self, kind: &DownloadKind) -> bool {
-        self.main.contains(kind) || self.parked.contains(kind)
-    }
-
-    /// Returns `true` if either the main queue or the parked set contain a download for a hash.
-    pub fn contains_hash(&self, hash: Hash) -> bool {
-        let as_raw = HashAndFormat::raw(hash).into();
-        let as_hash_seq = HashAndFormat::hash_seq(hash).into();
-        self.contains(&as_raw) || self.contains(&as_hash_seq)
-    }
-
-    /// Returns `true` if a download is in the parked set.
-    pub fn is_parked(&self, kind: &DownloadKind) -> bool {
-        self.parked.contains(kind)
-    }
-
-    /// Insert an element at the back of the main queue.
-    pub fn insert(&mut self, kind: DownloadKind) {
-        if !self.main.contains(&kind) {
-            self.main.insert(kind);
-        }
-    }
-
-    /// Insert an element at the front of the main queue.
-    pub fn insert_front(&mut self, kind: DownloadKind) {
-        if !self.main.contains(&kind) {
-            self.main.insert(kind);
-        }
-        self.main.to_front(&kind);
-    }
-
-    /// Dequeue the first download of the main queue.
-    pub fn pop_front(&mut self) -> Option<DownloadKind> {
-        self.main.pop_front()
-    }
-
-    /// Move the front item of the main queue into the parked set.
-    pub fn park_front(&mut self) {
-        if let Some(item) = self.pop_front() {
-            self.parked.insert(item);
-        }
-    }
-
-    /// Move a download from the parked set to the front of the main queue.
-    pub fn unpark(&mut self, kind: &DownloadKind) {
-        if self.parked.remove(kind) {
-            self.main.insert(*kind);
-            self.main.to_front(kind);
-        }
-    }
-
-    /// Move any download for a hash from the parked set to the main queue.
-    pub fn unpark_hash(&mut self, hash: Hash) {
-        let as_raw = HashAndFormat::raw(hash).into();
-        let as_hash_seq = HashAndFormat::hash_seq(hash).into();
-        self.unpark(&as_raw);
-        self.unpark(&as_hash_seq);
-    }
-
-    /// Remove a download from both the main queue and the parked set.
-    pub fn remove(&mut self, kind: &DownloadKind) -> bool {
-        self.main.remove(kind) || self.parked.remove(kind)
-    }
-}
-
-impl DialerT for Dialer {
-    type Connection = endpoint::Connection;
-
-    fn queue_dial(&mut self, node_id: NodeId) {
-        self.queue_dial(node_id, crate::protocol::ALPN)
-    }
-
-    fn pending_count(&self) -> usize {
-        self.pending_count()
-    }
-
-    fn is_pending(&self, node: NodeId) -> bool {
-        self.is_pending(node)
-    }
-
-    fn node_id(&self) -> NodeId {
-        self.endpoint().node_id()
-    }
-}
-
-/// Dials nodes and maintains a queue of pending dials.
-///
-/// The [`Dialer`] wraps an [`Endpoint`], connects to nodes through the endpoint, stores the
-/// pending connect futures and emits finished connect results.
-///
-/// The [`Dialer`] also implements [`Stream`] to retrieve the dialled connections.
-#[derive(Debug)]
-struct Dialer {
-    endpoint: Endpoint,
-    pending: JoinSet<(NodeId, anyhow::Result<endpoint::Connection>)>,
-    pending_dials: HashMap<NodeId, CancellationToken>,
-}
-
-impl Dialer {
-    /// Create a new dialer for a [`Endpoint`]
-    fn new(endpoint: Endpoint) -> Self {
-        Self {
-            endpoint,
-            pending: Default::default(),
-            pending_dials: Default::default(),
-        }
-    }
-
-    /// Starts to dial a node by [`NodeId`].
-    fn queue_dial(&mut self, node_id: NodeId, alpn: &'static [u8]) {
-        if self.is_pending(node_id) {
-            return;
-        }
-        let cancel = CancellationToken::new();
-        self.pending_dials.insert(node_id, cancel.clone());
-        let endpoint = self.endpoint.clone();
-        self.pending.spawn(async move {
-            let res = tokio::select! {
-                biased;
-                _ = cancel.cancelled() => Err(anyhow!("Cancelled")),
-                res = endpoint.connect(node_id, alpn) => res
-            };
-            (node_id, res)
-        });
-    }
-
-    /// Checks if a node is currently being dialed.
-    fn is_pending(&self, node: NodeId) -> bool {
-        self.pending_dials.contains_key(&node)
-    }
-
-    /// Number of pending connections to be opened.
-    fn pending_count(&self) -> usize {
-        self.pending_dials.len()
-    }
-
-    /// Returns a reference to the endpoint used in this dialer.
-    fn endpoint(&self) -> &Endpoint {
-        &self.endpoint
-    }
-}
-
-impl Stream for Dialer {
-    type Item = (NodeId, anyhow::Result<endpoint::Connection>);
-
-    fn poll_next(
-        mut self: Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-    ) -> Poll<Option<Self::Item>> {
-        match self.pending.poll_join_next(cx) {
-            Poll::Ready(Some(Ok((node_id, result)))) => {
-                self.pending_dials.remove(&node_id);
-                Poll::Ready(Some((node_id, result)))
-            }
-            Poll::Ready(Some(Err(e))) => {
-                error!("dialer error: {:?}", e);
-                Poll::Pending
-            }
-            _ => Poll::Pending,
-        }
-    }
-}
diff --git a/src/downloader/get.rs b/src/downloader/get.rs
deleted file mode 100644
index 99e884e34..000000000
--- a/src/downloader/get.rs
+++ /dev/null
@@ -1,100 +0,0 @@
-//! [`Getter`] implementation that performs requests over [`Connection`]s.
-//!
-//! [`Connection`]: iroh::endpoint::Connection
-
-use futures_lite::FutureExt;
-use iroh::endpoint;
-
-use super::{progress::BroadcastProgressSender, DownloadKind, FailureAction, GetStartFut, Getter};
-use crate::{
-    get::{db::get_to_db_in_steps, error::GetError},
-    store::Store,
-};
-
-impl From<GetError> for FailureAction {
-    fn from(e: GetError) -> Self {
-        match e {
-            e @ GetError::NotFound(_) => FailureAction::AbortRequest(e),
-            e @ GetError::RemoteReset(_) => FailureAction::RetryLater(e.into()),
-            e @ GetError::NoncompliantNode(_) => FailureAction::DropPeer(e.into()),
-            e @ GetError::Io(_) => FailureAction::RetryLater(e.into()),
-            e @ GetError::BadRequest(_) => FailureAction::AbortRequest(e),
-            // TODO: what do we want to do on local failures?
-            e @ GetError::LocalFailure(_) => FailureAction::AbortRequest(e),
-        }
-    }
-}
-
-/// [`Getter`] implementation that performs requests over [`Connection`]s.
-///
-/// [`Connection`]: iroh::endpoint::Connection
-pub(crate) struct IoGetter<S: Store> {
-    pub store: S,
-}
-
-impl<S: Store> Getter for IoGetter<S> {
-    type Connection = endpoint::Connection;
-    type NeedsConn = crate::get::db::GetStateNeedsConn;
-
-    fn get(
-        &mut self,
-        kind: DownloadKind,
-        progress_sender: BroadcastProgressSender,
-    ) -> GetStartFut<Self::NeedsConn> {
-        let store = self.store.clone();
-        async move {
-            match get_to_db_in_steps(store, kind.hash_and_format(), progress_sender).await {
-                Err(err) => Err(err.into()),
-                Ok(crate::get::db::GetState::Complete(stats)) => {
-                    Ok(super::GetOutput::Complete(stats))
-                }
-                Ok(crate::get::db::GetState::NeedsConn(needs_conn)) => {
-                    Ok(super::GetOutput::NeedsConn(needs_conn))
-                }
-            }
-        }
-        .boxed_local()
-    }
-}
-
-impl super::NeedsConn<endpoint::Connection> for crate::get::db::GetStateNeedsConn {
-    fn proceed(self, conn: endpoint::Connection) -> super::GetProceedFut {
-        async move {
-            let res = self.proceed(conn).await;
-            match res {
-                Ok(stats) => Ok(stats),
-                Err(err) => Err(err.into()),
-            }
-        }
-        .boxed_local()
-    }
-}
-
-pub(super) fn track_metrics(
-    res: &Result<crate::get::Stats, FailureAction>,
-    metrics: &crate::metrics::Metrics,
-) {
-    match res {
-        Ok(stats) => {
-            let crate::get::Stats {
-                bytes_written,
-                bytes_read: _,
-                elapsed,
-            } = stats;
-
-            metrics.downloads_success.inc();
-            metrics.download_bytes_total.inc_by(*bytes_written);
-            metrics
-                .download_time_total
-                .inc_by(elapsed.as_millis() as u64);
-        }
-        Err(e) => match &e {
-            FailureAction::AbortRequest(GetError::NotFound(_)) => {
-                metrics.downloads_notfound.inc();
-            }
-            _ => {
-                metrics.downloads_error.inc();
-            }
-        },
-    }
-}
diff --git a/src/downloader/invariants.rs b/src/downloader/invariants.rs
deleted file mode 100644
index 6d49b3497..000000000
--- a/src/downloader/invariants.rs
+++ /dev/null
@@ -1,163 +0,0 @@
-//! Invariants for the service.
-
-#![cfg(any(test, debug_assertions))]
-
-use super::*;
-
-/// invariants for the service.
-impl<G: Getter<Connection = D::Connection>, D: DialerT> Service<G, D> {
-    /// Checks the various invariants the service must maintain
-    #[track_caller]
-    pub(in crate::downloader) fn check_invariants(&self) {
-        self.check_active_request_count();
-        self.check_queued_requests_consistency();
-        self.check_idle_peer_consistency();
-        self.check_concurrency_limits();
-        self.check_provider_map_prunning();
-    }
-
-    /// Checks concurrency limits are maintained.
-    #[track_caller]
-    fn check_concurrency_limits(&self) {
-        let ConcurrencyLimits {
-            max_concurrent_requests,
-            max_concurrent_requests_per_node,
-            max_open_connections,
-            max_concurrent_dials_per_hash,
-        } = &self.concurrency_limits;
-
-        // check the total number of active requests to ensure it stays within the limit
-        assert!(
-            self.in_progress_downloads.len() <= *max_concurrent_requests,
-            "max_concurrent_requests exceeded"
-        );
-
-        // check that the open and dialing peers don't exceed the connection capacity
-        tracing::trace!(
-            "limits: conns: {}/{} | reqs: {}/{}",
-            self.connections_count(),
-            max_open_connections,
-            self.in_progress_downloads.len(),
-            max_concurrent_requests
-        );
-        assert!(
-            self.connections_count() <= *max_open_connections,
-            "max_open_connections exceeded"
-        );
-
-        // check the active requests per peer don't exceed the limit
-        for (node, info) in self.connected_nodes.iter() {
-            assert!(
-                info.active_requests() <= *max_concurrent_requests_per_node,
-                "max_concurrent_requests_per_node exceeded for {node}"
-            )
-        }
-
-        // check that we do not dial more nodes than allowed for the next pending hashes
-        if let Some(kind) = self.queue.front() {
-            let hash = kind.hash();
-            let nodes = self.providers.get_candidates(&hash);
-            let mut dialing = 0;
-            for node in nodes {
-                if self.dialer.is_pending(node) {
-                    dialing += 1;
-                }
-            }
-            assert!(
-                dialing <= *max_concurrent_dials_per_hash,
-                "max_concurrent_dials_per_hash exceeded for {hash}"
-            )
-        }
-    }
-
-    /// Checks that the count of active requests per peer is consistent with the active requests,
-    /// and that active request are consistent with download futures
-    #[track_caller]
-    fn check_active_request_count(&self) {
-        // check that the count of futures we are polling for downloads is consistent with the
-        // number of requests
-        assert_eq!(
-            self.active_requests.len(),
-            self.in_progress_downloads.len(),
-            "active_requests and in_progress_downloads are out of sync"
-        );
-        // check that the count of requests per peer matches the number of requests that have that
-        // peer as active
-        let mut real_count: HashMap<NodeId, usize> =
-            HashMap::with_capacity(self.connected_nodes.len());
-        for req_info in self.active_requests.values() {
-            // nothing like some classic word count
-            *real_count.entry(req_info.node).or_default() += 1;
-        }
-        for (peer, info) in self.connected_nodes.iter() {
-            assert_eq!(
-                info.active_requests(),
-                real_count.get(peer).copied().unwrap_or_default(),
-                "mismatched count of active requests for {peer}"
-            )
-        }
-    }
-
-    /// Checks that the queued requests all appear in the provider map and request map.
-    #[track_caller]
-    fn check_queued_requests_consistency(&self) {
-        // check that all hashes in the queue have candidates
-        for entry in self.queue.iter() {
-            assert!(
-                self.providers
-                    .get_candidates(&entry.hash())
-                    .next()
-                    .is_some(),
-                "all queued requests have providers"
-            );
-            assert!(
-                self.requests.contains_key(entry),
-                "all queued requests have request info"
-            );
-        }
-
-        // check that all parked hashes should be parked
-        for entry in self.queue.iter_parked() {
-            assert!(
-                matches!(self.next_step(entry), NextStep::Park),
-                "all parked downloads evaluate to the correct next step"
-            );
-            assert!(
-                self.providers
-                    .get_candidates(&entry.hash())
-                    .all(|node| matches!(self.node_state(node), NodeState::WaitForRetry)),
-                "all parked downloads have only retrying nodes"
-            );
-        }
-    }
-
-    /// Check that peers queued to be disconnected are consistent with peers considered idle.
-    #[track_caller]
-    fn check_idle_peer_consistency(&self) {
-        let idle_peers = self
-            .connected_nodes
-            .values()
-            .filter(|info| info.active_requests() == 0)
-            .count();
-        assert_eq!(
-            self.goodbye_nodes_queue.len(),
-            idle_peers,
-            "inconsistent count of idle peers"
-        );
-    }
-
-    /// Check that every hash in the provider map is needed.
-    #[track_caller]
-    fn check_provider_map_prunning(&self) {
-        for hash in self.providers.hash_node.keys() {
-            let as_raw = DownloadKind(HashAndFormat::raw(*hash));
-            let as_hash_seq = DownloadKind(HashAndFormat::hash_seq(*hash));
-            assert!(
-                self.queue.contains_hash(*hash)
-                    || self.active_requests.contains_key(&as_raw)
-                    || self.active_requests.contains_key(&as_hash_seq),
-                "all hashes in the provider map are in the queue or active"
-            )
-        }
-    }
-}
diff --git a/src/downloader/progress.rs b/src/downloader/progress.rs
deleted file mode 100644
index 9b0372976..000000000
--- a/src/downloader/progress.rs
+++ /dev/null
@@ -1,194 +0,0 @@
-use std::{
-    collections::HashMap,
-    sync::{
-        atomic::{AtomicU64, Ordering},
-        Arc,
-    },
-};
-
-use anyhow::anyhow;
-use parking_lot::Mutex;
-
-use super::DownloadKind;
-use crate::{
-    get::{db::DownloadProgress, progress::TransferState},
-    util::progress::{AsyncChannelProgressSender, IdGenerator, ProgressSendError, ProgressSender},
-};
-
-/// The channel that can be used to subscribe to progress updates.
-pub type ProgressSubscriber = AsyncChannelProgressSender<DownloadProgress>;
-
-/// Track the progress of downloads.
-///
-/// This struct allows to create [`ProgressSender`] structs to be passed to
-/// [`crate::get::db::get_to_db`]. Each progress sender can be subscribed to by any number of
-/// [`ProgressSubscriber`] channel senders, which will receive each progress update (if they have
-/// capacity). Additionally, the [`ProgressTracker`] maintains a [`TransferState`] for each
-/// transfer, applying each progress update to update this state. When subscribing to an already
-/// running transfer, the subscriber will receive a [`DownloadProgress::InitialState`] message
-/// containing the state at the time of the subscription, and then receive all further progress
-/// events directly.
-#[derive(Debug, Default)]
-pub struct ProgressTracker {
-    /// Map of shared state for each tracked download.
-    running: HashMap<DownloadKind, Shared>,
-    /// Shared [`IdGenerator`] for all progress senders created by the tracker.
-    id_gen: Arc<AtomicU64>,
-}
-
-impl ProgressTracker {
-    pub fn new() -> Self {
-        Self::default()
-    }
-
-    /// Track a new download with a list of initial subscribers.
-    ///
-    /// Note that this should only be called for *new* downloads. If a download for the `kind` is
-    /// already tracked in this [`ProgressTracker`], calling `track` will replace all existing
-    /// state and subscribers (equal to calling [`Self::remove`] first).
-    pub fn track(
-        &mut self,
-        kind: DownloadKind,
-        subscribers: impl IntoIterator<Item = ProgressSubscriber>,
-    ) -> BroadcastProgressSender {
-        let inner = Inner {
-            subscribers: subscribers.into_iter().collect(),
-            state: TransferState::new(kind.hash()),
-        };
-        let shared = Arc::new(Mutex::new(inner));
-        self.running.insert(kind, Arc::clone(&shared));
-        let id_gen = Arc::clone(&self.id_gen);
-        BroadcastProgressSender { shared, id_gen }
-    }
-
-    /// Subscribe to a tracked download.
-    ///
-    /// Will return an error if `kind` is not yet tracked.
-    pub async fn subscribe(
-        &mut self,
-        kind: DownloadKind,
-        sender: ProgressSubscriber,
-    ) -> anyhow::Result<()> {
-        let initial_msg = self
-            .running
-            .get_mut(&kind)
-            .ok_or_else(|| anyhow!("state for download {kind:?} not found"))?
-            .lock()
-            .subscribe(sender.clone());
-        sender.send(initial_msg).await?;
-        Ok(())
-    }
-
-    /// Unsubscribe `sender` from `kind`.
-    pub fn unsubscribe(&mut self, kind: &DownloadKind, sender: &ProgressSubscriber) {
-        if let Some(shared) = self.running.get_mut(kind) {
-            shared.lock().unsubscribe(sender)
-        }
-    }
-
-    /// Remove all state for a download.
-    pub fn remove(&mut self, kind: &DownloadKind) {
-        self.running.remove(kind);
-    }
-}
-
-type Shared = Arc<Mutex<Inner>>;
-
-#[derive(Debug)]
-struct Inner {
-    subscribers: Vec<ProgressSubscriber>,
-    state: TransferState,
-}
-
-impl Inner {
-    fn subscribe(&mut self, subscriber: ProgressSubscriber) -> DownloadProgress {
-        let msg = DownloadProgress::InitialState(self.state.clone());
-        self.subscribers.push(subscriber);
-        msg
-    }
-
-    fn unsubscribe(&mut self, sender: &ProgressSubscriber) {
-        self.subscribers.retain(|s| !s.same_channel(sender));
-    }
-
-    fn on_progress(&mut self, progress: DownloadProgress) {
-        self.state.on_progress(progress);
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct BroadcastProgressSender {
-    shared: Shared,
-    id_gen: Arc<AtomicU64>,
-}
-
-impl IdGenerator for BroadcastProgressSender {
-    fn new_id(&self) -> u64 {
-        self.id_gen.fetch_add(1, Ordering::SeqCst)
-    }
-}
-
-impl ProgressSender for BroadcastProgressSender {
-    type Msg = DownloadProgress;
-
-    async fn send(&self, msg: Self::Msg) -> Result<(), ProgressSendError> {
-        // making sure that the lock is not held across an await point.
-        let futs = {
-            let mut inner = self.shared.lock();
-            inner.on_progress(msg.clone());
-            let futs = inner
-                .subscribers
-                .iter_mut()
-                .map(|sender| {
-                    let sender = sender.clone();
-                    let msg = msg.clone();
-                    async move {
-                        match sender.send(msg).await {
-                            Ok(()) => None,
-                            Err(ProgressSendError::ReceiverDropped) => Some(sender),
-                        }
-                    }
-                })
-                .collect::<Vec<_>>();
-            drop(inner);
-            futs
-        };
-
-        let failed_senders = futures_buffered::join_all(futs).await;
-        // remove senders where the receiver is dropped
-        if failed_senders.iter().any(|s| s.is_some()) {
-            let mut inner = self.shared.lock();
-            for sender in failed_senders.into_iter().flatten() {
-                inner.unsubscribe(&sender);
-            }
-            drop(inner);
-        }
-        Ok(())
-    }
-
-    fn try_send(&self, msg: Self::Msg) -> Result<(), ProgressSendError> {
-        let mut inner = self.shared.lock();
-        inner.on_progress(msg.clone());
-        // remove senders where the receiver is dropped
-        inner
-            .subscribers
-            .retain_mut(|sender| match sender.try_send(msg.clone()) {
-                Err(ProgressSendError::ReceiverDropped) => false,
-                Ok(()) => true,
-            });
-        Ok(())
-    }
-
-    fn blocking_send(&self, msg: Self::Msg) -> Result<(), ProgressSendError> {
-        let mut inner = self.shared.lock();
-        inner.on_progress(msg.clone());
-        // remove senders where the receiver is dropped
-        inner
-            .subscribers
-            .retain_mut(|sender| match sender.blocking_send(msg.clone()) {
-                Err(ProgressSendError::ReceiverDropped) => false,
-                Ok(()) => true,
-            });
-        Ok(())
-    }
-}
diff --git a/src/downloader/test.rs b/src/downloader/test.rs
deleted file mode 100644
index 87ef11f26..000000000
--- a/src/downloader/test.rs
+++ /dev/null
@@ -1,544 +0,0 @@
-#![cfg(test)]
-use std::{
-    sync::atomic::AtomicUsize,
-    time::{Duration, Instant},
-};
-
-use anyhow::anyhow;
-use futures_util::future::FutureExt;
-use iroh::SecretKey;
-use tracing_test::traced_test;
-
-use super::*;
-use crate::{
-    get::{
-        db::BlobId,
-        progress::{BlobProgress, TransferState},
-    },
-    util::{
-        local_pool::LocalPool,
-        progress::{AsyncChannelProgressSender, IdGenerator},
-    },
-};
-
-mod dialer;
-mod getter;
-
-impl Downloader {
-    fn spawn_for_test(
-        dialer: dialer::TestingDialer,
-        getter: getter::TestingGetter,
-        concurrency_limits: ConcurrencyLimits,
-    ) -> (Self, LocalPool) {
-        Self::spawn_for_test_with_retry_config(
-            dialer,
-            getter,
-            concurrency_limits,
-            Default::default(),
-        )
-    }
-
-    fn spawn_for_test_with_retry_config(
-        dialer: dialer::TestingDialer,
-        getter: getter::TestingGetter,
-        concurrency_limits: ConcurrencyLimits,
-        retry_config: RetryConfig,
-    ) -> (Self, LocalPool) {
-        let (msg_tx, msg_rx) = mpsc::channel(super::SERVICE_CHANNEL_CAPACITY);
-        let metrics = Arc::new(Metrics::default());
-
-        let lp = LocalPool::default();
-        let metrics_clone = metrics.clone();
-        let config = Arc::new(Config {
-            concurrency: concurrency_limits,
-            retry: retry_config,
-        });
-        let config2 = config.clone();
-        lp.spawn_detached(move || async move {
-            // we want to see the logs of the service
-            let service = Service::new(getter, dialer, config2, msg_rx, metrics_clone);
-            service.run().await
-        });
-
-        (
-            Downloader {
-                inner: Arc::new(Inner {
-                    next_id: AtomicU64::new(0),
-                    msg_tx,
-                    config,
-                    metrics,
-                }),
-            },
-            lp,
-        )
-    }
-}
-
-/// Tests that receiving a download request and performing it doesn't explode.
-#[tokio::test]
-#[traced_test]
-async fn smoke_test() {
-    let dialer = dialer::TestingDialer::default();
-    let getter = getter::TestingGetter::default();
-    let concurrency_limits = ConcurrencyLimits::default();
-
-    let (downloader, _lp) =
-        Downloader::spawn_for_test(dialer.clone(), getter.clone(), concurrency_limits);
-
-    // send a request and make sure the peer is requested the corresponding download
-    let peer = SecretKey::generate(rand::thread_rng()).public();
-    let kind: DownloadKind = HashAndFormat::raw(Hash::new([0u8; 32])).into();
-    let req = DownloadRequest::new(kind, vec![peer]);
-    let handle = downloader.queue(req).await;
-    // wait for the download result to be reported
-    handle.await.expect("should report success");
-    // verify that the peer was dialed
-    dialer.assert_history(&[peer]);
-    // verify that the request was sent
-    getter.assert_history(&[(kind, peer)]);
-}
-
-/// Tests that multiple intents produce a single request.
-#[tokio::test]
-#[traced_test]
-async fn deduplication() {
-    let dialer = dialer::TestingDialer::default();
-    let getter = getter::TestingGetter::default();
-    // make request take some time to ensure the intents are received before completion
-    getter.set_request_duration(Duration::from_secs(1));
-    let concurrency_limits = ConcurrencyLimits::default();
-
-    let (downloader, _lp) =
-        Downloader::spawn_for_test(dialer.clone(), getter.clone(), concurrency_limits);
-
-    let peer = SecretKey::generate(rand::thread_rng()).public();
-    let kind: DownloadKind = HashAndFormat::raw(Hash::new([0u8; 32])).into();
-    let mut handles = Vec::with_capacity(10);
-    for _ in 0..10 {
-        let req = DownloadRequest::new(kind, vec![peer]);
-        let h = downloader.queue(req).await;
-        handles.push(h);
-    }
-    assert!(
-        futures_buffered::join_all(handles)
-            .await
-            .into_iter()
-            .all(|r| r.is_ok()),
-        "all downloads should succeed"
-    );
-    // verify that the request was sent just once
-    getter.assert_history(&[(kind, peer)]);
-}
-
-/// Tests that the request is cancelled only when all intents are cancelled.
-#[ignore = "flaky"]
-#[tokio::test]
-#[traced_test]
-async fn cancellation() {
-    let dialer = dialer::TestingDialer::default();
-    let getter = getter::TestingGetter::default();
-    // make request take some time to ensure cancellations are received on time
-    getter.set_request_duration(Duration::from_millis(500));
-    let concurrency_limits = ConcurrencyLimits::default();
-
-    let (downloader, _lp) =
-        Downloader::spawn_for_test(dialer.clone(), getter.clone(), concurrency_limits);
-
-    let peer = SecretKey::generate(rand::thread_rng()).public();
-    let kind_1: DownloadKind = HashAndFormat::raw(Hash::new([0u8; 32])).into();
-    let req = DownloadRequest::new(kind_1, vec![peer]);
-    let handle_a = downloader.queue(req.clone()).await;
-    let handle_b = downloader.queue(req).await;
-    downloader.cancel(handle_a).await;
-
-    // create a request with two intents and cancel them both
-    let kind_2 = HashAndFormat::raw(Hash::new([1u8; 32]));
-    let req = DownloadRequest::new(kind_2, vec![peer]);
-    let handle_c = downloader.queue(req.clone()).await;
-    let handle_d = downloader.queue(req).await;
-    downloader.cancel(handle_c).await;
-    downloader.cancel(handle_d).await;
-
-    // wait for the download result to be reported, a was cancelled but b should continue
-    handle_b.await.expect("should report success");
-    // verify that the request was sent just once, and that the second request was never sent
-    getter.assert_history(&[(kind_1, peer)]);
-}
-
-/// Test that when the downloader receives a flood of requests, they are scheduled so that the
-/// maximum number of concurrent requests is not exceed.
-/// NOTE: This is internally tested by [`Service::check_invariants`].
-#[tokio::test]
-#[traced_test]
-async fn max_concurrent_requests_total() {
-    let dialer = dialer::TestingDialer::default();
-    let getter = getter::TestingGetter::default();
-    // make request take some time to ensure concurreny limits are hit
-    getter.set_request_duration(Duration::from_millis(500));
-    // set the concurreny limit very low to ensure it's hit
-    let concurrency_limits = ConcurrencyLimits {
-        max_concurrent_requests: 2,
-        ..Default::default()
-    };
-
-    let (downloader, _lp) =
-        Downloader::spawn_for_test(dialer.clone(), getter.clone(), concurrency_limits);
-
-    // send the downloads
-    let peer = SecretKey::generate(rand::thread_rng()).public();
-    let mut handles = Vec::with_capacity(5);
-    let mut expected_history = Vec::with_capacity(5);
-    for i in 0..5 {
-        let kind: DownloadKind = HashAndFormat::raw(Hash::new([i; 32])).into();
-        let req = DownloadRequest::new(kind, vec![peer]);
-        let h = downloader.queue(req).await;
-        expected_history.push((kind, peer));
-        handles.push(h);
-    }
-
-    assert!(
-        futures_buffered::join_all(handles)
-            .await
-            .into_iter()
-            .all(|r| r.is_ok()),
-        "all downloads should succeed"
-    );
-
-    // verify that the request was sent just once
-    getter.assert_history(&expected_history);
-}
-
-/// Test that when the downloader receives a flood of requests, with only one peer to handle them,
-/// the maximum number of requests per peer is still respected.
-/// NOTE: This is internally tested by [`Service::check_invariants`].
-#[tokio::test]
-#[traced_test]
-async fn max_concurrent_requests_per_peer() {
-    let dialer = dialer::TestingDialer::default();
-    let getter = getter::TestingGetter::default();
-    // make request take some time to ensure concurreny limits are hit
-    getter.set_request_duration(Duration::from_millis(500));
-    // set the concurreny limit very low to ensure it's hit
-    let concurrency_limits = ConcurrencyLimits {
-        max_concurrent_requests_per_node: 1,
-        max_concurrent_requests: 10000, // all requests can be performed at the same time
-        ..Default::default()
-    };
-
-    let (downloader, _lp) =
-        Downloader::spawn_for_test(dialer.clone(), getter.clone(), concurrency_limits);
-
-    // send the downloads
-    let peer = SecretKey::generate(rand::thread_rng()).public();
-    let mut handles = Vec::with_capacity(5);
-    for i in 0..5 {
-        let kind = HashAndFormat::raw(Hash::new([i; 32]));
-        let req = DownloadRequest::new(kind, vec![peer]);
-        let h = downloader.queue(req).await;
-        handles.push(h);
-    }
-
-    futures_buffered::join_all(handles).await;
-}
-
-/// Tests concurrent progress reporting for multiple intents.
-///
-/// This first registers two intents for a download, and then proceeds until the `Found` event is
-/// emitted, and verifies that both intents received the event.
-/// It then registers a third intent mid-download, and makes sure it receives a correct ìnitial
-/// state. The download then finishes, and we make sure that all events are emitted properly, and
-/// the progress state of the handles converges.
-#[tokio::test]
-#[traced_test]
-async fn concurrent_progress() {
-    let dialer = dialer::TestingDialer::default();
-    let getter = getter::TestingGetter::default();
-
-    let (start_tx, start_rx) = oneshot::channel();
-    let start_rx = start_rx.shared();
-
-    let (done_tx, done_rx) = oneshot::channel();
-    let done_rx = done_rx.shared();
-
-    getter.set_handler(Arc::new(move |hash, _peer, progress, _duration| {
-        let start_rx = start_rx.clone();
-        let done_rx = done_rx.clone();
-        async move {
-            let hash = hash.hash();
-            start_rx.await.unwrap();
-            let id = progress.new_id();
-            progress
-                .send(DownloadProgress::Found {
-                    id,
-                    child: BlobId::Root,
-                    hash,
-                    size: 100,
-                })
-                .await
-                .unwrap();
-            done_rx.await.unwrap();
-            progress.send(DownloadProgress::Done { id }).await.unwrap();
-            Ok(Stats::default())
-        }
-        .boxed()
-    }));
-    let (downloader, _lp) =
-        Downloader::spawn_for_test(dialer.clone(), getter.clone(), Default::default());
-
-    let peer = SecretKey::generate(rand::thread_rng()).public();
-    let hash = Hash::new([0u8; 32]);
-    let kind_1 = HashAndFormat::raw(hash);
-
-    let (prog_a_tx, prog_a_rx) = async_channel::bounded(64);
-    let prog_a_tx = AsyncChannelProgressSender::new(prog_a_tx);
-    let req = DownloadRequest::new(kind_1, vec![peer]).progress_sender(prog_a_tx);
-    let handle_a = downloader.queue(req).await;
-
-    let (prog_b_tx, prog_b_rx) = async_channel::bounded(64);
-    let prog_b_tx = AsyncChannelProgressSender::new(prog_b_tx);
-    let req = DownloadRequest::new(kind_1, vec![peer]).progress_sender(prog_b_tx);
-    let handle_b = downloader.queue(req).await;
-
-    let mut state_a = TransferState::new(hash);
-    let mut state_b = TransferState::new(hash);
-    let mut state_c = TransferState::new(hash);
-
-    let prog0_b = prog_b_rx.recv().await.unwrap();
-    assert!(matches!(
-        prog0_b,
-        DownloadProgress::InitialState(state) if state.root.hash == hash && state.root.progress == BlobProgress::Pending,
-    ));
-
-    start_tx.send(()).unwrap();
-
-    let prog1_a = prog_a_rx.recv().await.unwrap();
-    let prog1_b = prog_b_rx.recv().await.unwrap();
-    assert!(
-        matches!(prog1_a, DownloadProgress::Found { hash: found_hash, size: 100, ..} if found_hash == hash)
-    );
-    assert!(
-        matches!(prog1_b, DownloadProgress::Found { hash: found_hash, size: 100, ..} if found_hash == hash)
-    );
-
-    state_a.on_progress(prog1_a);
-    state_b.on_progress(prog1_b);
-    assert_eq!(state_a, state_b);
-
-    let (prog_c_tx, prog_c_rx) = async_channel::bounded(64);
-    let prog_c_tx = AsyncChannelProgressSender::new(prog_c_tx);
-    let req = DownloadRequest::new(kind_1, vec![peer]).progress_sender(prog_c_tx);
-    let handle_c = downloader.queue(req).await;
-
-    let prog1_c = prog_c_rx.recv().await.unwrap();
-    assert!(matches!(&prog1_c, DownloadProgress::InitialState(state) if state == &state_a));
-    state_c.on_progress(prog1_c);
-
-    done_tx.send(()).unwrap();
-
-    let (res_a, res_b, res_c) = tokio::join!(handle_a, handle_b, handle_c);
-    res_a.unwrap();
-    res_b.unwrap();
-    res_c.unwrap();
-
-    let prog_a: Vec<_> = prog_a_rx.collect().await;
-    let prog_b: Vec<_> = prog_b_rx.collect().await;
-    let prog_c: Vec<_> = prog_c_rx.collect().await;
-
-    assert_eq!(prog_a.len(), 1);
-    assert_eq!(prog_b.len(), 1);
-    assert_eq!(prog_c.len(), 1);
-
-    assert!(matches!(prog_a[0], DownloadProgress::Done { .. }));
-    assert!(matches!(prog_b[0], DownloadProgress::Done { .. }));
-    assert!(matches!(prog_c[0], DownloadProgress::Done { .. }));
-
-    for p in prog_a {
-        state_a.on_progress(p);
-    }
-    for p in prog_b {
-        state_b.on_progress(p);
-    }
-    for p in prog_c {
-        state_c.on_progress(p);
-    }
-    assert_eq!(state_a, state_b);
-    assert_eq!(state_a, state_c);
-}
-
-#[tokio::test]
-#[traced_test]
-async fn long_queue() {
-    let dialer = dialer::TestingDialer::default();
-    let getter = getter::TestingGetter::default();
-    let concurrency_limits = ConcurrencyLimits {
-        max_open_connections: 2,
-        max_concurrent_requests_per_node: 2,
-        max_concurrent_requests: 4, // all requests can be performed at the same time
-        ..Default::default()
-    };
-
-    let (downloader, _lp) =
-        Downloader::spawn_for_test(dialer.clone(), getter.clone(), concurrency_limits);
-    let mut rng = rand::thread_rng();
-    // send the downloads
-    let nodes = [
-        SecretKey::generate(&mut rng).public(),
-        SecretKey::generate(&mut rng).public(),
-        SecretKey::generate(&mut rng).public(),
-    ];
-    let mut handles = vec![];
-    for i in 0..100usize {
-        let kind = HashAndFormat::raw(Hash::new(i.to_be_bytes()));
-        let peer = nodes[i % 3];
-        let req = DownloadRequest::new(kind, vec![peer]);
-        let h = downloader.queue(req).await;
-        handles.push(h);
-    }
-
-    let res = futures_buffered::join_all(handles).await;
-    for res in res {
-        res.expect("all downloads to succeed");
-    }
-}
-
-/// If a download errors with [`FailureAction::DropPeer`], make sure that the peer is not dropped
-/// while other transfers are still running.
-#[tokio::test]
-#[traced_test]
-async fn fail_while_running() {
-    let dialer = dialer::TestingDialer::default();
-    let getter = getter::TestingGetter::default();
-    let (downloader, _lp) =
-        Downloader::spawn_for_test(dialer.clone(), getter.clone(), Default::default());
-    let blob_fail = HashAndFormat::raw(Hash::new([1u8; 32]));
-    let blob_success = HashAndFormat::raw(Hash::new([2u8; 32]));
-
-    getter.set_handler(Arc::new(move |kind, _node, _progress_sender, _duration| {
-        async move {
-            if kind == blob_fail.into() {
-                tokio::time::sleep(Duration::from_millis(10)).await;
-                Err(FailureAction::DropPeer(anyhow!("bad!")))
-            } else if kind == blob_success.into() {
-                tokio::time::sleep(Duration::from_millis(20)).await;
-                Ok(Default::default())
-            } else {
-                unreachable!("invalid blob")
-            }
-        }
-        .boxed()
-    }));
-
-    let node = SecretKey::generate(rand::thread_rng()).public();
-    let req_success = DownloadRequest::new(blob_success, vec![node]);
-    let req_fail = DownloadRequest::new(blob_fail, vec![node]);
-    let handle_success = downloader.queue(req_success).await;
-    let handle_fail = downloader.queue(req_fail).await;
-
-    let res_fail = handle_fail.await;
-    let res_success = handle_success.await;
-
-    assert!(res_fail.is_err());
-    assert!(res_success.is_ok());
-}
-
-#[tokio::test]
-#[traced_test]
-async fn retry_nodes_simple() {
-    let dialer = dialer::TestingDialer::default();
-    let getter = getter::TestingGetter::default();
-    let (downloader, _lp) =
-        Downloader::spawn_for_test(dialer.clone(), getter.clone(), Default::default());
-    let node = SecretKey::generate(rand::thread_rng()).public();
-    let dial_attempts = Arc::new(AtomicUsize::new(0));
-    let dial_attempts2 = dial_attempts.clone();
-    // fail on first dial, then succeed
-    dialer.set_dial_outcome(move |_node| dial_attempts2.fetch_add(1, Ordering::SeqCst) != 0);
-    let kind = HashAndFormat::raw(Hash::EMPTY);
-    let req = DownloadRequest::new(kind, vec![node]);
-    let handle = downloader.queue(req).await;
-
-    assert!(handle.await.is_ok());
-    assert_eq!(dial_attempts.load(Ordering::SeqCst), 2);
-    dialer.assert_history(&[node, node]);
-}
-
-#[tokio::test]
-#[traced_test]
-async fn retry_nodes_fail() {
-    let dialer = dialer::TestingDialer::default();
-    let getter = getter::TestingGetter::default();
-    let config = RetryConfig {
-        initial_retry_delay: Duration::from_millis(10),
-        max_retries_per_node: 3,
-    };
-
-    let (downloader, _lp) = Downloader::spawn_for_test_with_retry_config(
-        dialer.clone(),
-        getter.clone(),
-        Default::default(),
-        config,
-    );
-    let node = SecretKey::generate(rand::thread_rng()).public();
-    // fail always
-    dialer.set_dial_outcome(move |_node| false);
-
-    // queue a download
-    let kind = HashAndFormat::raw(Hash::EMPTY);
-    let req = DownloadRequest::new(kind, vec![node]);
-    let now = Instant::now();
-    let handle = downloader.queue(req).await;
-
-    // assert that the download failed
-    assert!(handle.await.is_err());
-
-    // assert the dial history: we dialed 4 times
-    dialer.assert_history(&[node, node, node, node]);
-
-    // assert that the retry timeouts were uphold
-    let expected_dial_duration = Duration::from_millis(10 * 4);
-    let expected_retry_wait_duration = Duration::from_millis(10 + 2 * 10 + 3 * 10);
-    assert!(now.elapsed() >= expected_dial_duration + expected_retry_wait_duration);
-}
-
-#[tokio::test]
-#[traced_test]
-async fn retry_nodes_jump_queue() {
-    let dialer = dialer::TestingDialer::default();
-    let getter = getter::TestingGetter::default();
-    let concurrency_limits = ConcurrencyLimits {
-        max_open_connections: 2,
-        max_concurrent_requests_per_node: 2,
-        max_concurrent_requests: 4, // all requests can be performed at the same time
-        ..Default::default()
-    };
-
-    let (downloader, _lp) =
-        Downloader::spawn_for_test(dialer.clone(), getter.clone(), concurrency_limits);
-
-    let mut rng = rand::thread_rng();
-    let good_node = SecretKey::generate(&mut rng).public();
-    let bad_node = SecretKey::generate(&mut rng).public();
-
-    dialer.set_dial_outcome(move |node| node == good_node);
-    let kind1 = HashAndFormat::raw(Hash::new([0u8; 32]));
-    let kind2 = HashAndFormat::raw(Hash::new([2u8; 32]));
-
-    let req1 = DownloadRequest::new(kind1, vec![bad_node]);
-    let h1 = downloader.queue(req1).await;
-
-    let req2 = DownloadRequest::new(kind2, vec![bad_node, good_node]);
-    let h2 = downloader.queue(req2).await;
-
-    // wait for req2 to complete - this tests that the "queue is jumped" and we are not
-    // waiting for req1 to elapse all retries
-    assert!(h2.await.is_ok());
-
-    dialer.assert_history(&[bad_node, good_node]);
-
-    // now we make download1 succeed!
-    dialer.set_dial_outcome(move |_node| true);
-    assert!(h1.await.is_ok());
-
-    // assert history
-    dialer.assert_history(&[bad_node, good_node, bad_node]);
-}
diff --git a/src/downloader/test/dialer.rs b/src/downloader/test/dialer.rs
deleted file mode 100644
index 5124e2d3d..000000000
--- a/src/downloader/test/dialer.rs
+++ /dev/null
@@ -1,100 +0,0 @@
-//! Implementation of [`super::Dialer`] used for testing.
-
-use std::task::{Context, Poll};
-
-use parking_lot::RwLock;
-
-use super::*;
-
-/// Dialer for testing that keeps track of the dialing history.
-#[derive(Default, Clone)]
-pub(super) struct TestingDialer(Arc<RwLock<TestingDialerInner>>);
-
-struct TestingDialerInner {
-    /// Peers that are being dialed.
-    dialing: HashSet<NodeId>,
-    /// Queue of dials.
-    dial_futs: delay_queue::DelayQueue<NodeId>,
-    /// History of attempted dials.
-    dial_history: Vec<NodeId>,
-    /// How long does a dial last.
-    dial_duration: Duration,
-    /// Fn deciding if a dial is successful.
-    dial_outcome: Box<dyn Fn(NodeId) -> bool + Send + Sync + 'static>,
-    /// Our own node id
-    node_id: NodeId,
-}
-
-impl Default for TestingDialerInner {
-    fn default() -> Self {
-        TestingDialerInner {
-            dialing: HashSet::default(),
-            dial_futs: delay_queue::DelayQueue::default(),
-            dial_history: Vec::default(),
-            dial_duration: Duration::from_millis(10),
-            dial_outcome: Box::new(|_| true),
-            node_id: NodeId::from_bytes(&[0u8; 32]).unwrap(),
-        }
-    }
-}
-
-impl DialerT for TestingDialer {
-    type Connection = NodeId;
-
-    fn queue_dial(&mut self, node_id: NodeId) {
-        let mut inner = self.0.write();
-        inner.dial_history.push(node_id);
-        // for now assume every dial works
-        let dial_duration = inner.dial_duration;
-        if inner.dialing.insert(node_id) {
-            inner.dial_futs.insert(node_id, dial_duration);
-        }
-    }
-
-    fn pending_count(&self) -> usize {
-        self.0.read().dialing.len()
-    }
-
-    fn is_pending(&self, node: NodeId) -> bool {
-        self.0.read().dialing.contains(&node)
-    }
-
-    fn node_id(&self) -> NodeId {
-        self.0.read().node_id
-    }
-}
-
-impl Stream for TestingDialer {
-    type Item = (NodeId, anyhow::Result<NodeId>);
-
-    fn poll_next(self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        let mut inner = self.0.write();
-        match inner.dial_futs.poll_expired(cx) {
-            Poll::Ready(Some(expired)) => {
-                let node = expired.into_inner();
-                let report_ok = (inner.dial_outcome)(node);
-                let result = report_ok
-                    .then_some(node)
-                    .ok_or_else(|| anyhow::anyhow!("dialing test set to fail"));
-                inner.dialing.remove(&node);
-                Poll::Ready(Some((node, result)))
-            }
-            _ => Poll::Pending,
-        }
-    }
-}
-
-impl TestingDialer {
-    #[track_caller]
-    pub(super) fn assert_history(&self, history: &[NodeId]) {
-        assert_eq!(self.0.read().dial_history, history)
-    }
-
-    pub(super) fn set_dial_outcome(
-        &self,
-        dial_outcome: impl Fn(NodeId) -> bool + Send + Sync + 'static,
-    ) {
-        let mut inner = self.0.write();
-        inner.dial_outcome = Box::new(dial_outcome);
-    }
-}
diff --git a/src/downloader/test/getter.rs b/src/downloader/test/getter.rs
deleted file mode 100644
index 0ea200caa..000000000
--- a/src/downloader/test/getter.rs
+++ /dev/null
@@ -1,89 +0,0 @@
-//! Implementation of [`super::Getter`] used for testing.
-
-use futures_lite::{future::Boxed as BoxFuture, FutureExt};
-use parking_lot::RwLock;
-
-use super::*;
-use crate::downloader;
-
-#[derive(Default, Clone, derive_more::Debug)]
-#[debug("TestingGetter")]
-pub(super) struct TestingGetter(Arc<RwLock<TestingGetterInner>>);
-
-pub(super) type RequestHandlerFn = Arc<
-    dyn Fn(
-            DownloadKind,
-            NodeId,
-            BroadcastProgressSender,
-            Duration,
-        ) -> BoxFuture<InternalDownloadResult>
-        + Send
-        + Sync
-        + 'static,
->;
-
-#[derive(Default)]
-struct TestingGetterInner {
-    /// How long requests take.
-    request_duration: Duration,
-    /// History of requests performed by the [`Getter`] and if they were successful.
-    request_history: Vec<(DownloadKind, NodeId)>,
-    /// Set a handler function which actually handles the requests.
-    request_handler: Option<RequestHandlerFn>,
-}
-
-impl Getter for TestingGetter {
-    // since for testing we don't need a real connection, just keep track of what peer is the
-    // request being sent to
-    type Connection = NodeId;
-    type NeedsConn = GetStateNeedsConn;
-
-    fn get(
-        &mut self,
-        kind: DownloadKind,
-        progress_sender: BroadcastProgressSender,
-    ) -> GetStartFut<Self::NeedsConn> {
-        std::future::ready(Ok(downloader::GetOutput::NeedsConn(GetStateNeedsConn(
-            self.clone(),
-            kind,
-            progress_sender,
-        ))))
-        .boxed_local()
-    }
-}
-
-#[derive(Debug)]
-pub(super) struct GetStateNeedsConn(TestingGetter, DownloadKind, BroadcastProgressSender);
-
-impl downloader::NeedsConn<NodeId> for GetStateNeedsConn {
-    fn proceed(self, peer: NodeId) -> super::GetProceedFut {
-        let GetStateNeedsConn(getter, kind, progress_sender) = self;
-        let mut inner = getter.0.write();
-        inner.request_history.push((kind, peer));
-        let request_duration = inner.request_duration;
-        let handler = inner.request_handler.clone();
-        async move {
-            if let Some(f) = handler {
-                f(kind, peer, progress_sender, request_duration).await
-            } else {
-                tokio::time::sleep(request_duration).await;
-                Ok(Stats::default())
-            }
-        }
-        .boxed_local()
-    }
-}
-
-impl TestingGetter {
-    pub(super) fn set_handler(&self, handler: RequestHandlerFn) {
-        self.0.write().request_handler = Some(handler);
-    }
-    pub(super) fn set_request_duration(&self, request_duration: Duration) {
-        self.0.write().request_duration = request_duration;
-    }
-    /// Verify that the request history is as expected
-    #[track_caller]
-    pub(super) fn assert_history(&self, history: &[(DownloadKind, NodeId)]) {
-        assert_eq!(self.0.read().request_history, history);
-    }
-}
diff --git a/src/export.rs b/src/export.rs
deleted file mode 100644
index 3576edf1d..000000000
--- a/src/export.rs
+++ /dev/null
@@ -1,135 +0,0 @@
-//! Functions to export data from a store
-
-use std::path::PathBuf;
-
-use anyhow::Context;
-use bytes::Bytes;
-use serde::{Deserialize, Serialize};
-use tracing::trace;
-
-use crate::{
-    format::collection::Collection,
-    store::{BaoBlobSize, ExportFormat, ExportMode, MapEntry, Store as BaoStore},
-    util::progress::{IdGenerator, ProgressSender},
-    Hash,
-};
-
-/// Export a hash to the local file system.
-///
-/// This exports a single hash, or a collection `recursive` is true, from the `db` store to the
-/// local filesystem. Depending on `mode` the data is either copied or reflinked (if possible).
-///
-/// Progress is reported as [`ExportProgress`] through a [`ProgressSender`]. Note that the
-/// [`ExportProgress::AllDone`] event is not emitted from here, but left to an upper layer to send,
-/// if desired.
-pub async fn export<D: BaoStore>(
-    db: &D,
-    hash: Hash,
-    outpath: PathBuf,
-    format: ExportFormat,
-    mode: ExportMode,
-    progress: impl ProgressSender<Msg = ExportProgress> + IdGenerator,
-) -> anyhow::Result<()> {
-    match format {
-        ExportFormat::Blob => export_blob(db, hash, outpath, mode, progress).await,
-        ExportFormat::Collection => export_collection(db, hash, outpath, mode, progress).await,
-    }
-}
-
-/// Export all entries of a collection, recursively, to files on the local filesystem.
-pub async fn export_collection<D: BaoStore>(
-    db: &D,
-    hash: Hash,
-    outpath: PathBuf,
-    mode: ExportMode,
-    progress: impl ProgressSender<Msg = ExportProgress> + IdGenerator,
-) -> anyhow::Result<()> {
-    tokio::fs::create_dir_all(&outpath).await?;
-    let collection = Collection::load_db(db, &hash).await?;
-    for (name, hash) in collection.into_iter() {
-        #[allow(clippy::needless_borrow)]
-        let path = outpath.join(pathbuf_from_name(&name));
-        export_blob(db, hash, path, mode, progress.clone()).await?;
-    }
-    Ok(())
-}
-
-/// Export a single blob to a file on the local filesystem.
-pub async fn export_blob<D: BaoStore>(
-    db: &D,
-    hash: Hash,
-    outpath: PathBuf,
-    mode: ExportMode,
-    progress: impl ProgressSender<Msg = ExportProgress> + IdGenerator,
-) -> anyhow::Result<()> {
-    if let Some(parent) = outpath.parent() {
-        tokio::fs::create_dir_all(parent).await?;
-    }
-    trace!("exporting blob {} to {}", hash, outpath.display());
-    let id = progress.new_id();
-    let entry = db.get(&hash).await?.context("entry not there")?;
-    progress
-        .send(ExportProgress::Found {
-            id,
-            hash,
-            outpath: outpath.clone(),
-            size: entry.size(),
-            meta: None,
-        })
-        .await?;
-    let progress1 = progress.clone();
-    db.export(
-        hash,
-        outpath,
-        mode,
-        Box::new(move |offset| Ok(progress1.try_send(ExportProgress::Progress { id, offset })?)),
-    )
-    .await?;
-    progress.send(ExportProgress::Done { id }).await?;
-    Ok(())
-}
-
-/// Progress events for an export operation
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum ExportProgress {
-    /// The download part is done for this id, we are now exporting the data
-    /// to the specified out path.
-    Found {
-        /// Unique id of the entry.
-        id: u64,
-        /// The hash of the entry.
-        hash: Hash,
-        /// The size of the entry in bytes.
-        size: BaoBlobSize,
-        /// The path to the file where the data is exported.
-        outpath: PathBuf,
-        /// Operation-specific metadata.
-        meta: Option<Bytes>,
-    },
-    /// We have made progress exporting the data.
-    ///
-    /// This is only sent for large blobs.
-    Progress {
-        /// Unique id of the entry that is being exported.
-        id: u64,
-        /// The offset of the progress, in bytes.
-        offset: u64,
-    },
-    /// We finished exporting a blob
-    Done {
-        /// Unique id of the entry that is being exported.
-        id: u64,
-    },
-    /// We are done with the whole operation.
-    AllDone,
-    /// We got an error and need to abort.
-    Abort(serde_error::Error),
-}
-
-fn pathbuf_from_name(name: &str) -> PathBuf {
-    let mut path = PathBuf::new();
-    for part in name.split('/') {
-        path.push(part);
-    }
-    path
-}
diff --git a/src/format/collection.rs b/src/format/collection.rs
index ca4f95548..9716faf86 100644
--- a/src/format/collection.rs
+++ b/src/format/collection.rs
@@ -4,14 +4,13 @@ use std::{collections::BTreeMap, future::Future};
 use anyhow::Context;
 use bao_tree::blake3;
 use bytes::Bytes;
-use iroh_io::AsyncSliceReaderExt;
 use serde::{Deserialize, Serialize};
 
 use crate::{
+    api::{blobs::AddBytesOptions, Store},
     get::{fsm, Stats},
     hashseq::HashSeq,
-    store::MapEntry,
-    util::TempTag,
+    util::temp_tag::TempTag,
     BlobFormat, Hash,
 };
 
@@ -70,6 +69,12 @@ pub trait SimpleStore {
     fn load(&self, hash: Hash) -> impl Future<Output = anyhow::Result<Bytes>> + Send + '_;
 }
 
+impl SimpleStore for crate::api::Store {
+    async fn load(&self, hash: Hash) -> anyhow::Result<Bytes> {
+        Ok(self.get_bytes(hash).await?)
+    }
+}
+
 /// Metadata for a collection
 ///
 /// This is the wire format for the metadata blob.
@@ -150,7 +155,7 @@ impl Collection {
         let end = loop {
             match curr {
                 fsm::EndBlobNext::MoreChildren(more) => {
-                    let child_offset = more.child_offset();
+                    let child_offset = more.offset() - 1;
                     let Some(hash) = links.get(usize::try_from(child_offset)?) else {
                         break more.finish();
                     };
@@ -180,44 +185,21 @@ impl Collection {
         Ok(Self::from_parts(hs.into_iter().skip(1), meta))
     }
 
-    /// Load a collection from a store given a root hash
-    ///
-    /// This assumes that both the links and the metadata of the collection is stored in the store.
-    /// It does not require that all child blobs are stored in the store.
-    pub async fn load_db<D>(db: &D, root: &Hash) -> anyhow::Result<Self>
-    where
-        D: crate::store::Map,
-    {
-        let links_entry = db.get(root).await?.context("links not found")?;
-        anyhow::ensure!(links_entry.is_complete(), "links not complete");
-        let links_bytes = links_entry.data_reader().await?.read_to_end().await?;
-        let mut links = HashSeq::try_from(links_bytes)?;
-        let meta_hash = links.pop_front().context("meta hash not found")?;
-        let meta_entry = db.get(&meta_hash).await?.context("meta not found")?;
-        anyhow::ensure!(links_entry.is_complete(), "links not complete");
-        let meta_bytes = meta_entry.data_reader().await?.read_to_end().await?;
-        let meta: CollectionMeta = postcard::from_bytes(&meta_bytes)?;
-        anyhow::ensure!(
-            meta.names.len() == links.len(),
-            "names and links length mismatch"
-        );
-        Ok(Self::from_parts(links, meta))
-    }
-
     /// Store a collection in a store. returns the root hash of the collection
     /// as a TempTag.
-    pub async fn store<D>(self, db: &D) -> anyhow::Result<TempTag>
-    where
-        D: crate::store::Store,
-    {
+    pub async fn store(self, db: &Store) -> anyhow::Result<TempTag> {
         let (links, meta) = self.into_parts();
         let meta_bytes = postcard::to_stdvec(&meta)?;
-        let meta_tag = db.import_bytes(meta_bytes.into(), BlobFormat::Raw).await?;
+        let meta_tag = db.add_bytes(meta_bytes).temp_tag().await?;
         let links_bytes = std::iter::once(*meta_tag.hash())
             .chain(links)
             .collect::<HashSeq>();
         let links_tag = db
-            .import_bytes(links_bytes.into(), BlobFormat::HashSeq)
+            .add_bytes_with_opts(AddBytesOptions {
+                data: links_bytes.into(),
+                format: BlobFormat::HashSeq,
+            })
+            .temp_tag()
             .await?;
         Ok(links_tag)
     }
@@ -311,7 +293,7 @@ mod tests {
         let collection = (0..3)
             .map(|i| {
                 (
-                    format!("blob{}", i),
+                    format!("blob{i}"),
                     crate::Hash::from(blake3::hash(&[i as u8])),
                 )
             })
diff --git a/src/get.rs b/src/get.rs
index 3365e1407..049ef4855 100644
--- a/src/get.rs
+++ b/src/get.rs
@@ -1,9 +1,13 @@
-//! The client side API
+//! The low level client side API
 //!
-//! To get data, create a connection using [iroh-net] or use any quinn
-//! connection that was obtained in another way.
+//! Note that while using this API directly is fine, a simpler way to get data
+//! to a store is to use the [`crate::api::remote`] API, in particular the
+//! [`crate::api::remote::Remote::fetch`] function to download data to your
+//! local store.
 //!
-//! Create a request describing the data you want to get.
+//! To get data, create a connection using an [`iroh::Endpoint`].
+//!
+//! Create a [`crate::protocol::GetRequest`] describing the data you want to get.
 //!
 //! Then create a state machine using [fsm::start] and
 //! drive it to completion by calling next on each state.
@@ -11,7 +15,7 @@
 //! For some states you have to provide additional arguments when calling next,
 //! or you can choose to finish early.
 //!
-//! [iroh-net]: https://docs.rs/iroh-net
+//! [iroh]: https://docs.rs/iroh
 use std::{
     error::Error,
     fmt::{self, Debug},
@@ -20,28 +24,41 @@ use std::{
 
 use anyhow::Result;
 use bao_tree::{io::fsm::BaoContentItem, ChunkNum};
-use iroh::endpoint::{self, ClosedStream, RecvStream, SendStream, WriteError};
+use fsm::RequestCounters;
+use iroh::endpoint::{self, RecvStream, SendStream};
+use iroh_io::TokioStreamReader;
+use n0_snafu::SpanTrace;
+use nested_enum_utils::common_fields;
 use serde::{Deserialize, Serialize};
+use snafu::{Backtrace, IntoError, ResultExt, Snafu};
 use tracing::{debug, error};
 
-use crate::{
-    protocol::RangeSpecSeq,
-    util::io::{TrackingReader, TrackingWriter},
-    Hash, IROH_BLOCK_SIZE,
-};
+use crate::{protocol::ChunkRangesSeq, store::IROH_BLOCK_SIZE, Hash};
 
-pub mod db;
-pub mod error;
-pub mod progress;
+mod error;
 pub mod request;
+pub(crate) use error::{BadRequestSnafu, LocalFailureSnafu};
+pub use error::{GetError, GetResult};
+
+type WrappedRecvStream = TokioStreamReader<RecvStream>;
 
 /// Stats about the transfer.
-#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(
+    Debug,
+    Default,
+    Clone,
+    PartialEq,
+    Eq,
+    Serialize,
+    Deserialize,
+    derive_more::Deref,
+    derive_more::DerefMut,
+)]
 pub struct Stats {
-    /// The number of bytes written
-    pub bytes_written: u64,
-    /// The number of bytes read
-    pub bytes_read: u64,
+    /// Counters
+    #[deref]
+    #[deref_mut]
+    pub counters: RequestCounters,
     /// The time it took to transfer the data
     pub elapsed: Duration,
 }
@@ -49,9 +66,21 @@ pub struct Stats {
 impl Stats {
     /// Transfer rate in megabits per second
     pub fn mbits(&self) -> f64 {
-        let data_len_bit = self.bytes_read * 8;
+        let data_len_bit = self.total_bytes_read() * 8;
         data_len_bit as f64 / (1000. * 1000.) / self.elapsed.as_secs_f64()
     }
+
+    pub fn total_bytes_read(&self) -> u64 {
+        self.payload_bytes_read + self.other_bytes_read
+    }
+
+    pub fn combine(&mut self, that: &Stats) {
+        self.payload_bytes_written += that.payload_bytes_written;
+        self.other_bytes_written += that.other_bytes_written;
+        self.payload_bytes_read += that.payload_bytes_read;
+        self.other_bytes_read += that.other_bytes_read;
+        self.elapsed += that.elapsed;
+    }
 }
 
 /// Finite state machine for get responses.
@@ -68,27 +97,65 @@ pub mod fsm {
     use derive_more::From;
     use iroh::endpoint::Connection;
     use iroh_io::{AsyncSliceWriter, AsyncStreamReader, TokioStreamReader};
-    use tokio::io::AsyncWriteExt;
 
     use super::*;
     use crate::{
-        protocol::{GetRequest, NonEmptyRequestRangeSpecIter, Request, MAX_MESSAGE_SIZE},
-        store::BaoBatchWriter,
+        get::error::BadRequestSnafu,
+        protocol::{
+            GetManyRequest, GetRequest, NonEmptyRequestRangeSpecIter, Request, MAX_MESSAGE_SIZE,
+        },
     };
 
-    type WrappedRecvStream = TrackingReader<TokioStreamReader<RecvStream>>;
-
     self_cell::self_cell! {
         struct RangesIterInner {
-            owner: RangeSpecSeq,
-            #[covariant]
+            owner: ChunkRangesSeq,
+            #[not_covariant]
             dependent: NonEmptyRequestRangeSpecIter,
         }
     }
 
     /// The entry point of the get response machine
-    pub fn start(connection: Connection, request: GetRequest) -> AtInitial {
-        AtInitial::new(connection, request)
+    pub fn start(
+        connection: Connection,
+        request: GetRequest,
+        counters: RequestCounters,
+    ) -> AtInitial {
+        AtInitial::new(connection, request, counters)
+    }
+
+    /// Start with a get many request. Todo: turn this into distinct states.
+    pub async fn start_get_many(
+        connection: Connection,
+        request: GetManyRequest,
+        counters: RequestCounters,
+    ) -> std::result::Result<Result<AtStartChild, AtClosing>, GetError> {
+        let start = Instant::now();
+        let (mut writer, reader) = connection.open_bi().await?;
+        let request = Request::GetMany(request);
+        let request_bytes = postcard::to_stdvec(&request)
+            .map_err(|source| BadRequestSnafu.into_error(source.into()))?;
+        writer.write_all(&request_bytes).await?;
+        writer.finish()?;
+        let Request::GetMany(request) = request else {
+            unreachable!();
+        };
+        let reader = TokioStreamReader::new(reader);
+        let mut ranges_iter = RangesIter::new(request.ranges.clone());
+        let first_item = ranges_iter.next();
+        let misc = Box::new(Misc {
+            counters,
+            start,
+            ranges_iter,
+        });
+        Ok(match first_item {
+            Some((child_offset, child_ranges)) => Ok(AtStartChild {
+                ranges: child_ranges,
+                reader,
+                misc,
+                offset: child_offset,
+            }),
+            None => Err(AtClosing::new(misc, reader, true)),
+        })
     }
 
     /// Owned iterator for the ranges in a request
@@ -104,8 +171,10 @@ pub mod fsm {
     }
 
     impl RangesIter {
-        pub fn new(owner: RangeSpecSeq) -> Self {
-            Self(RangesIterInner::new(owner, |owner| owner.iter_non_empty()))
+        pub fn new(owner: ChunkRangesSeq) -> Self {
+            Self(RangesIterInner::new(owner, |owner| {
+                owner.iter_non_empty_infinite()
+            }))
         }
 
         pub fn offset(&self) -> u64 {
@@ -118,8 +187,7 @@ pub mod fsm {
 
         fn next(&mut self) -> Option<Self::Item> {
             self.0.with_dependent_mut(|_owner, iter| {
-                iter.next()
-                    .map(|(offset, ranges)| (offset, ranges.to_chunk_ranges()))
+                iter.next().map(|(offset, ranges)| (offset, ranges.clone()))
             })
         }
     }
@@ -129,6 +197,7 @@ pub mod fsm {
     pub struct AtInitial {
         connection: Connection,
         request: GetRequest,
+        counters: RequestCounters,
     }
 
     impl AtInitial {
@@ -136,10 +205,11 @@ pub mod fsm {
         ///
         /// `connection` is an existing connection
         /// `request` is the request to be sent
-        pub fn new(connection: Connection, request: GetRequest) -> Self {
+        pub fn new(connection: Connection, request: GetRequest, counters: RequestCounters) -> Self {
             Self {
                 connection,
                 request,
+                counters,
             }
         }
 
@@ -147,13 +217,13 @@ pub mod fsm {
         pub async fn next(self) -> Result<AtConnected, endpoint::ConnectionError> {
             let start = Instant::now();
             let (writer, reader) = self.connection.open_bi().await?;
-            let reader = TrackingReader::new(TokioStreamReader::new(reader));
-            let writer = TrackingWriter::new(writer);
+            let reader = TokioStreamReader::new(reader);
             Ok(AtConnected {
                 start,
                 reader,
                 writer,
                 request: self.request,
+                counters: self.counters,
             })
         }
     }
@@ -163,8 +233,9 @@ pub mod fsm {
     pub struct AtConnected {
         start: Instant,
         reader: WrappedRecvStream,
-        writer: TrackingWriter<SendStream>,
+        writer: SendStream,
         request: GetRequest,
+        counters: RequestCounters,
     }
 
     /// Possible next states after the handshake has been sent
@@ -179,50 +250,30 @@ pub mod fsm {
     }
 
     /// Error that you can get from [`AtConnected::next`]
-    #[derive(Debug, thiserror::Error)]
+    #[common_fields({
+        backtrace: Option<Backtrace>,
+        #[snafu(implicit)]
+        span_trace: SpanTrace,
+    })]
+    #[allow(missing_docs)]
+    #[derive(Debug, Snafu)]
+    #[non_exhaustive]
     pub enum ConnectedNextError {
         /// Error when serializing the request
-        #[error("postcard ser: {0}")]
-        PostcardSer(postcard::Error),
+        #[snafu(display("postcard ser: {source}"))]
+        PostcardSer { source: postcard::Error },
         /// The serialized request is too long to be sent
-        #[error("request too big")]
-        RequestTooBig,
+        #[snafu(display("request too big"))]
+        RequestTooBig {},
         /// Error when writing the request to the [`SendStream`].
-        #[error("write: {0}")]
-        Write(#[from] WriteError),
+        #[snafu(display("write: {source}"))]
+        Write { source: quinn::WriteError },
         /// Quic connection is closed.
-        #[error("closed")]
-        Closed(#[from] ClosedStream),
+        #[snafu(display("closed"))]
+        Closed { source: quinn::ClosedStream },
         /// A generic io error
-        #[error("io {0}")]
-        Io(io::Error),
-    }
-
-    impl ConnectedNextError {
-        fn from_io(cause: io::Error) -> Self {
-            if let Some(inner) = cause.get_ref() {
-                if let Some(e) = inner.downcast_ref::<endpoint::WriteError>() {
-                    Self::Write(e.clone())
-                } else {
-                    Self::Io(cause)
-                }
-            } else {
-                Self::Io(cause)
-            }
-        }
-    }
-
-    impl From<ConnectedNextError> for io::Error {
-        fn from(cause: ConnectedNextError) -> Self {
-            match cause {
-                ConnectedNextError::Write(cause) => cause.into(),
-                ConnectedNextError::Io(cause) => cause,
-                ConnectedNextError::PostcardSer(cause) => {
-                    io::Error::new(io::ErrorKind::Other, cause)
-                }
-                _ => io::Error::new(io::ErrorKind::Other, cause),
-            }
-        }
+        #[snafu(transparent)]
+        Io { source: io::Error },
     }
 
     impl AtConnected {
@@ -238,37 +289,36 @@ pub mod fsm {
                 reader,
                 mut writer,
                 mut request,
+                mut counters,
             } = self;
             // 1. Send Request
-            {
+            counters.other_bytes_written += {
                 debug!("sending request");
                 let wrapped = Request::Get(request);
-                let request_bytes =
-                    postcard::to_stdvec(&wrapped).map_err(ConnectedNextError::PostcardSer)?;
-                let Request::Get(x) = wrapped;
+                let request_bytes = postcard::to_stdvec(&wrapped).context(PostcardSerSnafu)?;
+                let Request::Get(x) = wrapped else {
+                    unreachable!();
+                };
                 request = x;
 
                 if request_bytes.len() > MAX_MESSAGE_SIZE {
-                    return Err(ConnectedNextError::RequestTooBig);
+                    return Err(RequestTooBigSnafu.build());
                 }
 
                 // write the request itself
-                writer
-                    .write_all(&request_bytes)
-                    .await
-                    .map_err(ConnectedNextError::from_io)?;
-            }
+                writer.write_all(&request_bytes).await.context(WriteSnafu)?;
+                request_bytes.len() as u64
+            };
 
             // 2. Finish writing before expecting a response
-            let (mut writer, bytes_written) = writer.into_parts();
-            writer.finish()?;
+            writer.finish().context(ClosedSnafu)?;
 
             let hash = request.hash;
             let ranges_iter = RangesIter::new(request.ranges);
             // this is in a box so we don't have to memcpy it on every state transition
             let mut misc = Box::new(Misc {
+                counters,
                 start,
-                bytes_written,
                 ranges_iter,
             });
             Ok(match misc.ranges_iter.next() {
@@ -286,7 +336,7 @@ pub mod fsm {
                             reader,
                             ranges,
                             misc,
-                            child_offset: offset - 1,
+                            offset,
                         }
                         .into()
                     }
@@ -300,7 +350,7 @@ pub mod fsm {
     #[derive(Debug)]
     pub struct AtStartRoot {
         ranges: ChunkRanges,
-        reader: TrackingReader<TokioStreamReader<RecvStream>>,
+        reader: TokioStreamReader<RecvStream>,
         misc: Box<Misc>,
         hash: Hash,
     }
@@ -309,9 +359,9 @@ pub mod fsm {
     #[derive(Debug)]
     pub struct AtStartChild {
         ranges: ChunkRanges,
-        reader: TrackingReader<TokioStreamReader<RecvStream>>,
+        reader: TokioStreamReader<RecvStream>,
         misc: Box<Misc>,
-        child_offset: u64,
+        offset: u64,
     }
 
     impl AtStartChild {
@@ -320,8 +370,8 @@ pub mod fsm {
         /// This must be used to determine the hash needed to call next.
         /// If this is larger than the number of children in the collection,
         /// you can call finish to stop reading the response.
-        pub fn child_offset(&self) -> u64 {
-            self.child_offset
+        pub fn offset(&self) -> u64 {
+            self.offset
         }
 
         /// The ranges we have requested for the child
@@ -384,35 +434,41 @@ pub mod fsm {
     #[derive(Debug)]
     pub struct AtBlobHeader {
         ranges: ChunkRanges,
-        reader: TrackingReader<TokioStreamReader<RecvStream>>,
+        reader: TokioStreamReader<RecvStream>,
         misc: Box<Misc>,
         hash: Hash,
     }
 
     /// Error that you can get from [`AtBlobHeader::next`]
-    #[derive(Debug, thiserror::Error)]
+    #[common_fields({
+        backtrace: Option<Backtrace>,
+        #[snafu(implicit)]
+        span_trace: SpanTrace,
+    })]
+    #[non_exhaustive]
+    #[derive(Debug, Snafu)]
     pub enum AtBlobHeaderNextError {
         /// Eof when reading the size header
         ///
         /// This indicates that the provider does not have the requested data.
-        #[error("not found")]
-        NotFound,
+        #[snafu(display("not found"))]
+        NotFound {},
         /// Quinn read error when reading the size header
-        #[error("read: {0}")]
-        Read(endpoint::ReadError),
+        #[snafu(display("read: {source}"))]
+        EndpointRead { source: endpoint::ReadError },
         /// Generic io error
-        #[error("io: {0}")]
-        Io(io::Error),
+        #[snafu(display("io: {source}"))]
+        Io { source: io::Error },
     }
 
     impl From<AtBlobHeaderNextError> for io::Error {
         fn from(cause: AtBlobHeaderNextError) -> Self {
             match cause {
-                AtBlobHeaderNextError::NotFound => {
+                AtBlobHeaderNextError::NotFound { .. } => {
                     io::Error::new(io::ErrorKind::UnexpectedEof, cause)
                 }
-                AtBlobHeaderNextError::Read(cause) => cause.into(),
-                AtBlobHeaderNextError::Io(cause) => cause,
+                AtBlobHeaderNextError::EndpointRead { source, .. } => source.into(),
+                AtBlobHeaderNextError::Io { source, .. } => source,
             }
         }
     }
@@ -422,16 +478,17 @@ pub mod fsm {
         pub async fn next(mut self) -> Result<(AtBlobContent, u64), AtBlobHeaderNextError> {
             let size = self.reader.read::<8>().await.map_err(|cause| {
                 if cause.kind() == io::ErrorKind::UnexpectedEof {
-                    AtBlobHeaderNextError::NotFound
+                    NotFoundSnafu.build()
                 } else if let Some(e) = cause
                     .get_ref()
                     .and_then(|x| x.downcast_ref::<endpoint::ReadError>())
                 {
-                    AtBlobHeaderNextError::Read(e.clone())
+                    EndpointReadSnafu.into_error(e.clone())
                 } else {
-                    AtBlobHeaderNextError::Io(cause)
+                    IoSnafu.into_error(cause)
                 }
             })?;
+            self.misc.other_bytes_read += 8;
             let size = u64::from_le_bytes(size);
             let stream = ResponseDecoder::new(
                 self.hash.into(),
@@ -493,16 +550,6 @@ pub mod fsm {
             Ok(res)
         }
 
-        /// Write the entire stream for this blob to a batch writer.
-        pub async fn write_all_batch<B>(self, batch: B) -> result::Result<AtEndBlob, DecodeError>
-        where
-            B: BaoBatchWriter,
-        {
-            let (content, _size) = self.next().await?;
-            let res = content.write_all_batch(batch).await?;
-            Ok(res)
-        }
-
         /// The hash of the blob we are reading.
         pub fn hash(&self) -> Hash {
             self.hash
@@ -545,41 +592,53 @@ pub mod fsm {
     /// provider should never do this, so this is an indication that the provider is
     /// not behaving correctly.
     ///
-    /// The [`DecodeError::Io`] variant is just a fallback for any other io error that
-    /// is not actually a [`ReadError`].
+    /// The [`DecodeError::DecodeIo`] variant is just a fallback for any other io error that
+    /// is not actually a [`DecodeError::Read`].
     ///
     /// [`ReadError`]: endpoint::ReadError
-    #[derive(Debug, thiserror::Error)]
+    #[common_fields({
+        backtrace: Option<Backtrace>,
+        #[snafu(implicit)]
+        span_trace: SpanTrace,
+    })]
+    #[non_exhaustive]
+    #[derive(Debug, Snafu)]
     pub enum DecodeError {
         /// A chunk was not found or invalid, so the provider stopped sending data
-        #[error("not found")]
-        NotFound,
+        #[snafu(display("not found"))]
+        ChunkNotFound {},
         /// A parent was not found or invalid, so the provider stopped sending data
-        #[error("parent not found {0:?}")]
-        ParentNotFound(TreeNode),
+        #[snafu(display("parent not found {node:?}"))]
+        ParentNotFound { node: TreeNode },
         /// A parent was not found or invalid, so the provider stopped sending data
-        #[error("chunk not found {0}")]
-        LeafNotFound(ChunkNum),
+        #[snafu(display("chunk not found {num}"))]
+        LeafNotFound { num: ChunkNum },
         /// The hash of a parent did not match the expected hash
-        #[error("parent hash mismatch: {0:?}")]
-        ParentHashMismatch(TreeNode),
+        #[snafu(display("parent hash mismatch: {node:?}"))]
+        ParentHashMismatch { node: TreeNode },
         /// The hash of a leaf did not match the expected hash
-        #[error("leaf hash mismatch: {0}")]
-        LeafHashMismatch(ChunkNum),
+        #[snafu(display("leaf hash mismatch: {num}"))]
+        LeafHashMismatch { num: ChunkNum },
         /// Error when reading from the stream
-        #[error("read: {0}")]
-        Read(endpoint::ReadError),
+        #[snafu(display("read: {source}"))]
+        Read { source: endpoint::ReadError },
         /// A generic io error
-        #[error("io: {0}")]
-        Io(#[from] io::Error),
+        #[snafu(display("io: {source}"))]
+        DecodeIo { source: io::Error },
+    }
+
+    impl DecodeError {
+        pub(crate) fn leaf_hash_mismatch(num: ChunkNum) -> Self {
+            LeafHashMismatchSnafu { num }.build()
+        }
     }
 
     impl From<AtBlobHeaderNextError> for DecodeError {
         fn from(cause: AtBlobHeaderNextError) -> Self {
             match cause {
-                AtBlobHeaderNextError::NotFound => Self::NotFound,
-                AtBlobHeaderNextError::Read(cause) => Self::Read(cause),
-                AtBlobHeaderNextError::Io(cause) => Self::Io(cause),
+                AtBlobHeaderNextError::NotFound { .. } => ChunkNotFoundSnafu.build(),
+                AtBlobHeaderNextError::EndpointRead { source, .. } => ReadSnafu.into_error(source),
+                AtBlobHeaderNextError::Io { source, .. } => DecodeIoSnafu.into_error(source),
             }
         }
     }
@@ -587,35 +646,47 @@ pub mod fsm {
     impl From<DecodeError> for io::Error {
         fn from(cause: DecodeError) -> Self {
             match cause {
-                DecodeError::ParentNotFound(_) => {
+                DecodeError::ParentNotFound { .. } => {
+                    io::Error::new(io::ErrorKind::UnexpectedEof, cause)
+                }
+                DecodeError::LeafNotFound { .. } => {
                     io::Error::new(io::ErrorKind::UnexpectedEof, cause)
                 }
-                DecodeError::LeafNotFound(_) => io::Error::new(io::ErrorKind::UnexpectedEof, cause),
-                DecodeError::Read(cause) => cause.into(),
-                DecodeError::Io(cause) => cause,
-                _ => io::Error::new(io::ErrorKind::Other, cause),
+                DecodeError::Read { source, .. } => source.into(),
+                DecodeError::DecodeIo { source, .. } => source,
+                _ => io::Error::other(cause),
             }
         }
     }
 
+    impl From<io::Error> for DecodeError {
+        fn from(value: io::Error) -> Self {
+            DecodeIoSnafu.into_error(value)
+        }
+    }
+
     impl From<bao_tree::io::DecodeError> for DecodeError {
         fn from(value: bao_tree::io::DecodeError) -> Self {
             match value {
-                bao_tree::io::DecodeError::ParentNotFound(x) => Self::ParentNotFound(x),
-                bao_tree::io::DecodeError::LeafNotFound(x) => Self::LeafNotFound(x),
+                bao_tree::io::DecodeError::ParentNotFound(x) => {
+                    ParentNotFoundSnafu { node: x }.build()
+                }
+                bao_tree::io::DecodeError::LeafNotFound(x) => LeafNotFoundSnafu { num: x }.build(),
                 bao_tree::io::DecodeError::ParentHashMismatch(node) => {
-                    Self::ParentHashMismatch(node)
+                    ParentHashMismatchSnafu { node }.build()
+                }
+                bao_tree::io::DecodeError::LeafHashMismatch(chunk) => {
+                    LeafHashMismatchSnafu { num: chunk }.build()
                 }
-                bao_tree::io::DecodeError::LeafHashMismatch(chunk) => Self::LeafHashMismatch(chunk),
                 bao_tree::io::DecodeError::Io(cause) => {
                     if let Some(inner) = cause.get_ref() {
                         if let Some(e) = inner.downcast_ref::<endpoint::ReadError>() {
-                            Self::Read(e.clone())
+                            ReadSnafu.into_error(e.clone())
                         } else {
-                            Self::Io(cause)
+                            DecodeIoSnafu.into_error(cause)
                         }
                     } else {
-                        Self::Io(cause)
+                        DecodeIoSnafu.into_error(cause)
                     }
                 }
             }
@@ -636,8 +707,17 @@ pub mod fsm {
         pub async fn next(self) -> BlobContentNext {
             match self.stream.next().await {
                 ResponseDecoderNext::More((stream, res)) => {
-                    let next = Self { stream, ..self };
+                    let mut next = Self { stream, ..self };
                     let res = res.map_err(DecodeError::from);
+                    match &res {
+                        Ok(BaoContentItem::Parent(_)) => {
+                            next.misc.other_bytes_read += 64;
+                        }
+                        Ok(BaoContentItem::Leaf(leaf)) => {
+                            next.misc.payload_bytes_read += leaf.data.len() as u64;
+                        }
+                        _ => {}
+                    }
                     BlobContentNext::More((next, res))
                 }
                 ResponseDecoderNext::Done(stream) => BlobContentNext::Done(AtEndBlob {
@@ -662,6 +742,14 @@ pub mod fsm {
             self.misc.ranges_iter.offset()
         }
 
+        /// Current stats
+        pub fn stats(&self) -> Stats {
+            Stats {
+                counters: self.misc.counters,
+                elapsed: self.misc.start.elapsed(),
+            }
+        }
+
         /// Drain the response and throw away the result
         pub async fn drain(self) -> result::Result<AtEndBlob, DecodeError> {
             let mut content = self;
@@ -701,39 +789,6 @@ pub mod fsm {
             Ok((done, res))
         }
 
-        /// Write the entire stream for this blob to a batch writer.
-        pub async fn write_all_batch<B>(self, writer: B) -> result::Result<AtEndBlob, DecodeError>
-        where
-            B: BaoBatchWriter,
-        {
-            let mut writer = writer;
-            let mut buf = Vec::new();
-            let mut content = self;
-            let size = content.tree().size();
-            loop {
-                match content.next().await {
-                    BlobContentNext::More((next, item)) => {
-                        let item = item?;
-                        match &item {
-                            BaoContentItem::Parent(_) => {
-                                buf.push(item);
-                            }
-                            BaoContentItem::Leaf(_) => {
-                                buf.push(item);
-                                let batch = std::mem::take(&mut buf);
-                                writer.write_batch(size, batch).await?;
-                            }
-                        }
-                        content = next;
-                    }
-                    BlobContentNext::Done(end) => {
-                        assert!(buf.is_empty());
-                        return Ok(end);
-                    }
-                }
-            }
-        }
-
         /// Write the entire blob to a slice writer and to an optional outboard.
         ///
         /// The outboard is only written to if the blob is larger than a single
@@ -822,7 +877,7 @@ pub mod fsm {
             if let Some((offset, ranges)) = self.misc.ranges_iter.next() {
                 AtStartChild {
                     reader: self.stream,
-                    child_offset: offset - 1,
+                    offset,
                     ranges,
                     misc: self.misc,
                 }
@@ -853,7 +908,7 @@ pub mod fsm {
         /// Finish the get response, returning statistics
         pub async fn next(self) -> result::Result<Stats, endpoint::ReadError> {
             // Shut down the stream
-            let (reader, bytes_read) = self.reader.into_parts();
+            let reader = self.reader;
             let mut reader = reader.into_inner();
             if self.check_extra_data {
                 if let Some(chunk) = reader.read_chunk(8, false).await? {
@@ -864,48 +919,68 @@ pub mod fsm {
                 reader.stop(0u8.into()).ok();
             }
             Ok(Stats {
+                counters: self.misc.counters,
                 elapsed: self.misc.start.elapsed(),
-                bytes_written: self.misc.bytes_written,
-                bytes_read,
             })
         }
     }
 
+    #[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, PartialEq, Eq)]
+    pub struct RequestCounters {
+        /// payload bytes written
+        pub payload_bytes_written: u64,
+        /// request, hash pair and size bytes written
+        pub other_bytes_written: u64,
+        /// payload bytes read
+        pub payload_bytes_read: u64,
+        /// hash pair and size bytes read
+        pub other_bytes_read: u64,
+    }
+
     /// Stuff we need to hold on to while going through the machine states
-    #[derive(Debug)]
+    #[derive(Debug, derive_more::Deref, derive_more::DerefMut)]
     struct Misc {
         /// start time for statistics
         start: Instant,
-        /// bytes written for statistics
-        bytes_written: u64,
+        /// counters
+        #[deref]
+        #[deref_mut]
+        counters: RequestCounters,
         /// iterator over the ranges of the collection and the children
         ranges_iter: RangesIter,
     }
 }
 
 /// Error when processing a response
-#[derive(thiserror::Error, Debug)]
+#[common_fields({
+    backtrace: Option<Backtrace>,
+    #[snafu(implicit)]
+    span_trace: SpanTrace,
+})]
+#[allow(missing_docs)]
+#[non_exhaustive]
+#[derive(Debug, Snafu)]
 pub enum GetResponseError {
     /// Error when opening a stream
-    #[error("connection: {0}")]
-    Connection(#[from] endpoint::ConnectionError),
+    #[snafu(display("connection: {source}"))]
+    Connection { source: endpoint::ConnectionError },
     /// Error when writing the handshake or request to the stream
-    #[error("write: {0}")]
-    Write(#[from] endpoint::WriteError),
+    #[snafu(display("write: {source}"))]
+    Write { source: endpoint::WriteError },
     /// Error when reading from the stream
-    #[error("read: {0}")]
-    Read(#[from] endpoint::ReadError),
+    #[snafu(display("read: {source}"))]
+    Read { source: endpoint::ReadError },
     /// Error when decoding, e.g. hash mismatch
-    #[error("decode: {0}")]
-    Decode(bao_tree::io::DecodeError),
+    #[snafu(display("decode: {source}"))]
+    Decode { source: bao_tree::io::DecodeError },
     /// A generic error
-    #[error("generic: {0}")]
-    Generic(anyhow::Error),
+    #[snafu(display("generic: {source}"))]
+    Generic { source: anyhow::Error },
 }
 
 impl From<postcard::Error> for GetResponseError {
     fn from(cause: postcard::Error) -> Self {
-        Self::Generic(cause.into())
+        GenericSnafu.into_error(cause.into())
     }
 }
 
@@ -916,30 +991,30 @@ impl From<bao_tree::io::DecodeError> for GetResponseError {
                 // try to downcast to specific quinn errors
                 if let Some(source) = cause.source() {
                     if let Some(error) = source.downcast_ref::<endpoint::ConnectionError>() {
-                        return Self::Connection(error.clone());
+                        return ConnectionSnafu.into_error(error.clone());
                     }
                     if let Some(error) = source.downcast_ref::<endpoint::ReadError>() {
-                        return Self::Read(error.clone());
+                        return ReadSnafu.into_error(error.clone());
                     }
                     if let Some(error) = source.downcast_ref::<endpoint::WriteError>() {
-                        return Self::Write(error.clone());
+                        return WriteSnafu.into_error(error.clone());
                     }
                 }
-                Self::Generic(cause.into())
+                GenericSnafu.into_error(cause.into())
             }
-            _ => Self::Decode(cause),
+            _ => DecodeSnafu.into_error(cause),
         }
     }
 }
 
 impl From<anyhow::Error> for GetResponseError {
     fn from(cause: anyhow::Error) -> Self {
-        Self::Generic(cause)
+        GenericSnafu.into_error(cause)
     }
 }
 
 impl From<GetResponseError> for std::io::Error {
     fn from(cause: GetResponseError) -> Self {
-        Self::new(std::io::ErrorKind::Other, cause)
+        Self::other(cause)
     }
 }
diff --git a/src/get/db.rs b/src/get/db.rs
deleted file mode 100644
index 783bbabc5..000000000
--- a/src/get/db.rs
+++ /dev/null
@@ -1,698 +0,0 @@
-//! Functions that use the iroh-blobs protocol in conjunction with a bao store.
-
-use std::{future::Future, io, num::NonZeroU64, pin::Pin};
-
-use anyhow::anyhow;
-use bao_tree::{ChunkNum, ChunkRanges};
-use futures_lite::StreamExt;
-use genawaiter::{
-    rc::{Co, Gen},
-    GeneratorState,
-};
-use iroh::endpoint::Connection;
-use iroh_io::AsyncSliceReader;
-use serde::{Deserialize, Serialize};
-use tokio::sync::oneshot;
-use tracing::trace;
-
-use crate::{
-    get::{
-        self,
-        error::GetError,
-        fsm::{AtBlobHeader, AtEndBlob, ConnectedNext, EndBlobNext},
-        progress::TransferState,
-        Stats,
-    },
-    hashseq::parse_hash_seq,
-    protocol::{GetRequest, RangeSpec, RangeSpecSeq},
-    store::{
-        BaoBatchWriter, BaoBlobSize, FallibleProgressBatchWriter, MapEntry, MapEntryMut, MapMut,
-        Store as BaoStore,
-    },
-    util::progress::{IdGenerator, ProgressSender},
-    BlobFormat, Hash, HashAndFormat,
-};
-
-type GetGenerator = Gen<Yield, (), Pin<Box<dyn Future<Output = Result<Stats, GetError>>>>>;
-type GetFuture = Pin<Box<dyn Future<Output = Result<Stats, GetError>> + 'static>>;
-
-/// Get a blob or collection into a store.
-///
-/// This considers data that is already in the store, and will only request
-/// the remaining data.
-///
-/// Progress is reported as [`DownloadProgress`] through a [`ProgressSender`]. Note that the
-/// [`DownloadProgress::AllDone`] event is not emitted from here, but left to an upper layer to send,
-/// if desired.
-pub async fn get_to_db<
-    D: BaoStore,
-    C: FnOnce() -> F,
-    F: Future<Output = anyhow::Result<Connection>>,
->(
-    db: &D,
-    get_conn: C,
-    hash_and_format: &HashAndFormat,
-    progress_sender: impl ProgressSender<Msg = DownloadProgress> + IdGenerator,
-) -> Result<Stats, GetError> {
-    match get_to_db_in_steps(db.clone(), *hash_and_format, progress_sender).await? {
-        GetState::Complete(res) => Ok(res),
-        GetState::NeedsConn(state) => {
-            let conn = get_conn().await.map_err(GetError::Io)?;
-            state.proceed(conn).await
-        }
-    }
-}
-
-/// Get a blob or collection into a store, yielding if a connection is needed.
-///
-/// This checks a get request against a local store, and returns [`GetState`],
-/// which is either `Complete` in case the requested data is fully available in the local store, or
-/// `NeedsConn`, once a connection is needed to proceed downloading the missing data.
-///
-/// In the latter case, call [`GetStateNeedsConn::proceed`] with a connection to a provider to
-/// proceed with the download.
-///
-/// Progress reporting works in the same way as documented in [`get_to_db`].
-pub async fn get_to_db_in_steps<
-    D: BaoStore,
-    P: ProgressSender<Msg = DownloadProgress> + IdGenerator,
->(
-    db: D,
-    hash_and_format: HashAndFormat,
-    progress_sender: P,
-) -> Result<GetState, GetError> {
-    let mut gen: GetGenerator = genawaiter::rc::Gen::new(move |co| {
-        let fut = async move { producer(co, &db, &hash_and_format, progress_sender).await };
-        let fut: GetFuture = Box::pin(fut);
-        fut
-    });
-    match gen.async_resume().await {
-        GeneratorState::Yielded(Yield::NeedConn(reply)) => {
-            Ok(GetState::NeedsConn(GetStateNeedsConn(gen, reply)))
-        }
-        GeneratorState::Complete(res) => res.map(GetState::Complete),
-    }
-}
-
-/// Intermediary state returned from [`get_to_db_in_steps`] for a download request that needs a
-/// connection to proceed.
-#[derive(derive_more::Debug)]
-#[debug("GetStateNeedsConn")]
-pub struct GetStateNeedsConn(GetGenerator, oneshot::Sender<Connection>);
-
-impl GetStateNeedsConn {
-    /// Proceed with the download by providing a connection to a provider.
-    pub async fn proceed(mut self, conn: Connection) -> Result<Stats, GetError> {
-        self.1.send(conn).expect("receiver is not dropped");
-        match self.0.async_resume().await {
-            GeneratorState::Yielded(y) => match y {
-                Yield::NeedConn(_) => panic!("NeedsConn may only be yielded once"),
-            },
-            GeneratorState::Complete(res) => res,
-        }
-    }
-}
-
-/// Output of [`get_to_db_in_steps`].
-#[derive(Debug)]
-pub enum GetState {
-    /// The requested data is completely available in the local store, no network requests are
-    /// needed.
-    Complete(Stats),
-    /// The requested data is not fully available in the local store, we need a connection to
-    /// proceed.
-    ///
-    /// Once a connection is available, call [`GetStateNeedsConn::proceed`] to continue.
-    NeedsConn(GetStateNeedsConn),
-}
-
-struct GetCo(Co<Yield>);
-
-impl GetCo {
-    async fn get_conn(&self) -> Connection {
-        let (tx, rx) = oneshot::channel();
-        self.0.yield_(Yield::NeedConn(tx)).await;
-        rx.await.expect("sender may not be dropped")
-    }
-}
-
-enum Yield {
-    NeedConn(oneshot::Sender<Connection>),
-}
-
-async fn producer<D: BaoStore>(
-    co: Co<Yield, ()>,
-    db: &D,
-    hash_and_format: &HashAndFormat,
-    progress: impl ProgressSender<Msg = DownloadProgress> + IdGenerator,
-) -> Result<Stats, GetError> {
-    let HashAndFormat { hash, format } = hash_and_format;
-    let co = GetCo(co);
-    match format {
-        BlobFormat::Raw => get_blob(db, co, hash, progress).await,
-        BlobFormat::HashSeq => get_hash_seq(db, co, hash, progress).await,
-    }
-}
-
-/// Get a blob that was requested completely.
-///
-/// We need to create our own files and handle the case where an outboard
-/// is not needed.
-async fn get_blob<D: BaoStore>(
-    db: &D,
-    co: GetCo,
-    hash: &Hash,
-    progress: impl ProgressSender<Msg = DownloadProgress> + IdGenerator,
-) -> Result<Stats, GetError> {
-    let end = match db.get_mut(hash).await? {
-        Some(entry) if entry.is_complete() => {
-            tracing::info!("already got entire blob");
-            progress
-                .send(DownloadProgress::FoundLocal {
-                    child: BlobId::Root,
-                    hash: *hash,
-                    size: entry.size(),
-                    valid_ranges: RangeSpec::all(),
-                })
-                .await?;
-            return Ok(Stats::default());
-        }
-        Some(entry) => {
-            trace!("got partial data for {}", hash);
-            let valid_ranges = valid_ranges::<D>(&entry)
-                .await
-                .ok()
-                .unwrap_or_else(ChunkRanges::all);
-            progress
-                .send(DownloadProgress::FoundLocal {
-                    child: BlobId::Root,
-                    hash: *hash,
-                    size: entry.size(),
-                    valid_ranges: RangeSpec::new(&valid_ranges),
-                })
-                .await?;
-            let required_ranges: ChunkRanges = ChunkRanges::all().difference(&valid_ranges);
-
-            let request = GetRequest::new(*hash, RangeSpecSeq::from_ranges([required_ranges]));
-            // full request
-            let conn = co.get_conn().await;
-            let request = get::fsm::start(conn, request);
-            // create a new bidi stream
-            let connected = request.next().await?;
-            // next step. we have requested a single hash, so this must be StartRoot
-            let ConnectedNext::StartRoot(start) = connected.next().await? else {
-                return Err(GetError::NoncompliantNode(anyhow!("expected StartRoot")));
-            };
-            // move to the header
-            let header = start.next();
-            // do the ceremony of getting the blob and adding it to the database
-
-            get_blob_inner_partial(db, header, entry, progress).await?
-        }
-        None => {
-            // full request
-            let conn = co.get_conn().await;
-            let request = get::fsm::start(conn, GetRequest::single(*hash));
-            // create a new bidi stream
-            let connected = request.next().await?;
-            // next step. we have requested a single hash, so this must be StartRoot
-            let ConnectedNext::StartRoot(start) = connected.next().await? else {
-                return Err(GetError::NoncompliantNode(anyhow!("expected StartRoot")));
-            };
-            // move to the header
-            let header = start.next();
-            // do the ceremony of getting the blob and adding it to the database
-            get_blob_inner(db, header, progress).await?
-        }
-    };
-
-    // we have requested a single hash, so we must be at closing
-    let EndBlobNext::Closing(end) = end.next() else {
-        return Err(GetError::NoncompliantNode(anyhow!("expected StartRoot")));
-    };
-    // this closes the bidi stream. Do something with the stats?
-    let stats = end.next().await?;
-    Ok(stats)
-}
-
-/// Given a partial entry, get the valid ranges.
-pub async fn valid_ranges<D: MapMut>(entry: &D::EntryMut) -> anyhow::Result<ChunkRanges> {
-    use tracing::trace as log;
-    // compute the valid range from just looking at the data file
-    let mut data_reader = entry.data_reader().await?;
-    let data_size = data_reader.size().await?;
-    let valid_from_data = ChunkRanges::from(..ChunkNum::full_chunks(data_size));
-    // compute the valid range from just looking at the outboard file
-    let mut outboard = entry.outboard().await?;
-    let all = ChunkRanges::all();
-    let mut stream = bao_tree::io::fsm::valid_outboard_ranges(&mut outboard, &all);
-    let mut valid_from_outboard = ChunkRanges::empty();
-    while let Some(range) = stream.next().await {
-        valid_from_outboard |= ChunkRanges::from(range?);
-    }
-    let valid: ChunkRanges = valid_from_data.intersection(&valid_from_outboard);
-    log!("valid_from_data: {:?}", valid_from_data);
-    log!("valid_from_outboard: {:?}", valid_from_data);
-    Ok(valid)
-}
-
-/// Get a blob that was requested completely.
-///
-/// We need to create our own files and handle the case where an outboard
-/// is not needed.
-async fn get_blob_inner<D: BaoStore>(
-    db: &D,
-    at_header: AtBlobHeader,
-    sender: impl ProgressSender<Msg = DownloadProgress> + IdGenerator,
-) -> Result<AtEndBlob, GetError> {
-    // read the size. The size we get here is not verified, but since we use
-    // it for the tree traversal we are guaranteed not to get more than size.
-    let (at_content, size) = at_header.next().await?;
-    let hash = at_content.hash();
-    let child_offset = at_content.offset();
-    // get or create the partial entry
-    let entry = db.get_or_create(hash, size).await?;
-    // open the data file in any case
-    let bw = entry.batch_writer().await?;
-    // allocate a new id for progress reports for this transfer
-    let id = sender.new_id();
-    sender
-        .send(DownloadProgress::Found {
-            id,
-            hash,
-            size,
-            child: BlobId::from_offset(child_offset),
-        })
-        .await?;
-    let sender2 = sender.clone();
-    let on_write = move |offset: u64, _length: usize| {
-        // if try send fails it means that the receiver has been dropped.
-        // in that case we want to abort the write_all_with_outboard.
-        sender2
-            .try_send(DownloadProgress::Progress { id, offset })
-            .inspect_err(|_| {
-                tracing::info!("aborting download of {}", hash);
-            })?;
-        Ok(())
-    };
-    let mut bw = FallibleProgressBatchWriter::new(bw, on_write);
-    // use the convenience method to write all to the batch writer
-    let end = at_content.write_all_batch(&mut bw).await?;
-    // sync the underlying storage, if needed
-    bw.sync().await?;
-    drop(bw);
-    db.insert_complete(entry).await?;
-    // notify that we are done
-    sender.send(DownloadProgress::Done { id }).await?;
-    Ok(end)
-}
-
-/// Get a blob that was requested partially.
-///
-/// We get passed the data and outboard ids. Partial downloads are only done
-/// for large blobs where the outboard is present.
-async fn get_blob_inner_partial<D: BaoStore>(
-    db: &D,
-    at_header: AtBlobHeader,
-    entry: D::EntryMut,
-    sender: impl ProgressSender<Msg = DownloadProgress> + IdGenerator,
-) -> Result<AtEndBlob, GetError> {
-    // read the size. The size we get here is not verified, but since we use
-    // it for the tree traversal we are guaranteed not to get more than size.
-    let (at_content, size) = at_header.next().await?;
-    // create a batch writer for the bao file
-    let bw = entry.batch_writer().await?;
-    // allocate a new id for progress reports for this transfer
-    let id = sender.new_id();
-    let hash = at_content.hash();
-    let child_offset = at_content.offset();
-    sender
-        .send(DownloadProgress::Found {
-            id,
-            hash,
-            size,
-            child: BlobId::from_offset(child_offset),
-        })
-        .await?;
-    let sender2 = sender.clone();
-    let on_write = move |offset: u64, _length: usize| {
-        // if try send fails it means that the receiver has been dropped.
-        // in that case we want to abort the write_all_with_outboard.
-        sender2
-            .try_send(DownloadProgress::Progress { id, offset })
-            .inspect_err(|_| {
-                tracing::info!("aborting download of {}", hash);
-            })?;
-        Ok(())
-    };
-    let mut bw = FallibleProgressBatchWriter::new(bw, on_write);
-    // use the convenience method to write all to the batch writer
-    let at_end = at_content.write_all_batch(&mut bw).await?;
-    // sync the underlying storage, if needed
-    bw.sync().await?;
-    drop(bw);
-    // we got to the end without error, so we can mark the entry as complete
-    //
-    // caution: this assumes that the request filled all the gaps in our local
-    // data. We can't re-check this here since that would be very expensive.
-    db.insert_complete(entry).await?;
-    // notify that we are done
-    sender.send(DownloadProgress::Done { id }).await?;
-    Ok(at_end)
-}
-
-/// Get information about a blob in a store.
-///
-/// This will compute the valid ranges for partial blobs, so it is somewhat expensive for those.
-pub async fn blob_info<D: BaoStore>(db: &D, hash: &Hash) -> io::Result<BlobInfo<D>> {
-    io::Result::Ok(match db.get_mut(hash).await? {
-        Some(entry) if entry.is_complete() => BlobInfo::Complete {
-            size: entry.size().value(),
-        },
-        Some(entry) => {
-            let valid_ranges = valid_ranges::<D>(&entry)
-                .await
-                .ok()
-                .unwrap_or_else(ChunkRanges::all);
-            BlobInfo::Partial {
-                entry,
-                valid_ranges,
-            }
-        }
-        None => BlobInfo::Missing,
-    })
-}
-
-/// Like `get_blob_info`, but for multiple hashes
-async fn blob_infos<D: BaoStore>(db: &D, hash_seq: &[Hash]) -> io::Result<Vec<BlobInfo<D>>> {
-    let items = futures_lite::stream::iter(hash_seq)
-        .then(|hash| blob_info(db, hash))
-        .collect::<Vec<_>>();
-    items.await.into_iter().collect()
-}
-
-/// Get a sequence of hashes
-async fn get_hash_seq<D: BaoStore>(
-    db: &D,
-    co: GetCo,
-    root_hash: &Hash,
-    sender: impl ProgressSender<Msg = DownloadProgress> + IdGenerator,
-) -> Result<Stats, GetError> {
-    use tracing::info as log;
-    let finishing = match db.get_mut(root_hash).await? {
-        Some(entry) if entry.is_complete() => {
-            log!("already got collection - doing partial download");
-            // send info that we have the hashseq itself entirely
-            sender
-                .send(DownloadProgress::FoundLocal {
-                    child: BlobId::Root,
-                    hash: *root_hash,
-                    size: entry.size(),
-                    valid_ranges: RangeSpec::all(),
-                })
-                .await?;
-            // got the collection
-            let reader = entry.data_reader().await?;
-            let (mut hash_seq, children) = parse_hash_seq(reader).await.map_err(|err| {
-                GetError::NoncompliantNode(anyhow!("Failed to parse downloaded HashSeq: {err}"))
-            })?;
-            sender
-                .send(DownloadProgress::FoundHashSeq {
-                    hash: *root_hash,
-                    children,
-                })
-                .await?;
-            let mut children: Vec<Hash> = vec![];
-            while let Some(hash) = hash_seq.next().await? {
-                children.push(hash);
-            }
-            let missing_info = blob_infos(db, &children).await?;
-            // send the info about what we have
-            for (i, info) in missing_info.iter().enumerate() {
-                if let Some(size) = info.size() {
-                    sender
-                        .send(DownloadProgress::FoundLocal {
-                            child: BlobId::from_offset((i as u64) + 1),
-                            hash: children[i],
-                            size,
-                            valid_ranges: RangeSpec::new(info.valid_ranges()),
-                        })
-                        .await?;
-                }
-            }
-            if missing_info
-                .iter()
-                .all(|x| matches!(x, BlobInfo::Complete { .. }))
-            {
-                log!("nothing to do");
-                return Ok(Stats::default());
-            }
-
-            let missing_iter = std::iter::once(ChunkRanges::empty())
-                .chain(missing_info.iter().map(|x| x.missing_ranges()))
-                .collect::<Vec<_>>();
-            log!("requesting chunks {:?}", missing_iter);
-            let request = GetRequest::new(*root_hash, RangeSpecSeq::from_ranges(missing_iter));
-            let conn = co.get_conn().await;
-            let request = get::fsm::start(conn, request);
-            // create a new bidi stream
-            let connected = request.next().await?;
-            log!("connected");
-            // we have not requested the root, so this must be StartChild
-            let ConnectedNext::StartChild(start) = connected.next().await? else {
-                return Err(GetError::NoncompliantNode(anyhow!("expected StartChild")));
-            };
-            let mut next = EndBlobNext::MoreChildren(start);
-            // read all the children
-            loop {
-                let start = match next {
-                    EndBlobNext::MoreChildren(start) => start,
-                    EndBlobNext::Closing(finish) => break finish,
-                };
-                let child_offset = usize::try_from(start.child_offset())
-                    .map_err(|_| GetError::NoncompliantNode(anyhow!("child offset too large")))?;
-                let (child_hash, info) =
-                    match (children.get(child_offset), missing_info.get(child_offset)) {
-                        (Some(blob), Some(info)) => (*blob, info),
-                        _ => break start.finish(),
-                    };
-                tracing::info!(
-                    "requesting child {} {:?}",
-                    child_hash,
-                    info.missing_ranges()
-                );
-                let header = start.next(child_hash);
-                let end_blob = match info {
-                    BlobInfo::Missing => get_blob_inner(db, header, sender.clone()).await?,
-                    BlobInfo::Partial { entry, .. } => {
-                        get_blob_inner_partial(db, header, entry.clone(), sender.clone()).await?
-                    }
-                    BlobInfo::Complete { .. } => {
-                        return Err(GetError::NoncompliantNode(anyhow!(
-                            "got data we have not requested"
-                        )));
-                    }
-                };
-                next = end_blob.next();
-            }
-        }
-        _ => {
-            tracing::debug!("don't have collection - doing full download");
-            // don't have the collection, so probably got nothing
-            let conn = co.get_conn().await;
-            let request = get::fsm::start(conn, GetRequest::all(*root_hash));
-            // create a new bidi stream
-            let connected = request.next().await?;
-            // next step. we have requested a single hash, so this must be StartRoot
-            let ConnectedNext::StartRoot(start) = connected.next().await? else {
-                return Err(GetError::NoncompliantNode(anyhow!("expected StartRoot")));
-            };
-            // move to the header
-            let header = start.next();
-            // read the blob and add it to the database
-            let end_root = get_blob_inner(db, header, sender.clone()).await?;
-            // read the collection fully for now
-            let entry = db
-                .get(root_hash)
-                .await?
-                .ok_or_else(|| GetError::LocalFailure(anyhow!("just downloaded but not in db")))?;
-            let reader = entry.data_reader().await?;
-            let (mut collection, count) = parse_hash_seq(reader).await.map_err(|err| {
-                GetError::NoncompliantNode(anyhow!("Failed to parse downloaded HashSeq: {err}"))
-            })?;
-            sender
-                .send(DownloadProgress::FoundHashSeq {
-                    hash: *root_hash,
-                    children: count,
-                })
-                .await?;
-            let mut children = vec![];
-            while let Some(hash) = collection.next().await? {
-                children.push(hash);
-            }
-            let mut next = end_root.next();
-            // read all the children
-            loop {
-                let start = match next {
-                    EndBlobNext::MoreChildren(start) => start,
-                    EndBlobNext::Closing(finish) => break finish,
-                };
-                let child_offset = usize::try_from(start.child_offset())
-                    .map_err(|_| GetError::NoncompliantNode(anyhow!("child offset too large")))?;
-
-                let child_hash = match children.get(child_offset) {
-                    Some(blob) => *blob,
-                    None => break start.finish(),
-                };
-                let header = start.next(child_hash);
-                let end_blob = get_blob_inner(db, header, sender.clone()).await?;
-                next = end_blob.next();
-            }
-        }
-    };
-    // this closes the bidi stream. Do something with the stats?
-    let stats = finishing.next().await?;
-    Ok(stats)
-}
-
-/// Information about a the status of a blob in a store.
-#[derive(Debug, Clone)]
-pub enum BlobInfo<D: BaoStore> {
-    /// we have the blob completely
-    Complete {
-        /// The size of the entry in bytes.
-        size: u64,
-    },
-    /// we have the blob partially
-    Partial {
-        /// The partial entry.
-        entry: D::EntryMut,
-        /// The ranges that are available locally.
-        valid_ranges: ChunkRanges,
-    },
-    /// we don't have the blob at all
-    Missing,
-}
-
-impl<D: BaoStore> BlobInfo<D> {
-    /// The size of the blob, if known.
-    pub fn size(&self) -> Option<BaoBlobSize> {
-        match self {
-            BlobInfo::Complete { size } => Some(BaoBlobSize::Verified(*size)),
-            BlobInfo::Partial { entry, .. } => Some(entry.size()),
-            BlobInfo::Missing => None,
-        }
-    }
-
-    /// Ranges that are valid locally.
-    ///
-    /// This will be all for complete blobs, empty for missing blobs,
-    /// and a set with possibly open last range for partial blobs.
-    pub fn valid_ranges(&self) -> ChunkRanges {
-        match self {
-            BlobInfo::Complete { .. } => ChunkRanges::all(),
-            BlobInfo::Partial { valid_ranges, .. } => valid_ranges.clone(),
-            BlobInfo::Missing => ChunkRanges::empty(),
-        }
-    }
-
-    /// Ranges that are missing locally and need to be requested.
-    ///
-    /// This will be empty for complete blobs, all for missing blobs, and
-    /// a set with possibly open last range for partial blobs.
-    pub fn missing_ranges(&self) -> ChunkRanges {
-        match self {
-            BlobInfo::Complete { .. } => ChunkRanges::empty(),
-            BlobInfo::Partial { valid_ranges, .. } => ChunkRanges::all().difference(valid_ranges),
-            BlobInfo::Missing => ChunkRanges::all(),
-        }
-    }
-}
-
-/// Progress updates for the get operation.
-// TODO: Move to super::progress
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum DownloadProgress {
-    /// Initial state if subscribing to a running or queued transfer.
-    InitialState(TransferState),
-    /// Data was found locally.
-    FoundLocal {
-        /// child offset
-        child: BlobId,
-        /// The hash of the entry.
-        hash: Hash,
-        /// The size of the entry in bytes.
-        size: BaoBlobSize,
-        /// The ranges that are available locally.
-        valid_ranges: RangeSpec,
-    },
-    /// A new connection was established.
-    Connected,
-    /// An item was found with hash `hash`, from now on referred to via `id`.
-    Found {
-        /// A new unique progress id for this entry.
-        id: u64,
-        /// Identifier for this blob within this download.
-        ///
-        /// Will always be [`BlobId::Root`] unless a hashseq is downloaded, in which case this
-        /// allows to identify the children by their offset in the hashseq.
-        child: BlobId,
-        /// The hash of the entry.
-        hash: Hash,
-        /// The size of the entry in bytes.
-        size: u64,
-    },
-    /// An item was found with hash `hash`, from now on referred to via `id`.
-    FoundHashSeq {
-        /// The name of the entry.
-        hash: Hash,
-        /// Number of children in the collection, if known.
-        children: u64,
-    },
-    /// We got progress ingesting item `id`.
-    Progress {
-        /// The unique id of the entry.
-        id: u64,
-        /// The offset of the progress, in bytes.
-        offset: u64,
-    },
-    /// We are done with `id`.
-    Done {
-        /// The unique id of the entry.
-        id: u64,
-    },
-    /// All operations finished.
-    ///
-    /// This will be the last message in the stream.
-    AllDone(Stats),
-    /// We got an error and need to abort.
-    ///
-    /// This will be the last message in the stream.
-    Abort(serde_error::Error),
-}
-
-/// The id of a blob in a transfer
-#[derive(
-    Debug, Copy, Clone, Ord, PartialOrd, Eq, PartialEq, std::hash::Hash, Serialize, Deserialize,
-)]
-pub enum BlobId {
-    /// The root blob (child id 0)
-    Root,
-    /// A child blob (child id > 0)
-    Child(NonZeroU64),
-}
-
-impl BlobId {
-    fn from_offset(id: u64) -> Self {
-        NonZeroU64::new(id).map(Self::Child).unwrap_or(Self::Root)
-    }
-}
-
-impl From<BlobId> for u64 {
-    fn from(value: BlobId) -> Self {
-        match value {
-            BlobId::Root => 0,
-            BlobId::Child(id) => id.into(),
-        }
-    }
-}
diff --git a/src/get/error.rs b/src/get/error.rs
index 03bec51e0..1c3ea9465 100644
--- a/src/get/error.rs
+++ b/src/get/error.rs
@@ -1,37 +1,161 @@
 //! Error returned from get operations
+use std::io;
 
 use iroh::endpoint::{self, ClosedStream};
+use n0_snafu::SpanTrace;
+use nested_enum_utils::common_fields;
+use quinn::{ConnectionError, ReadError, WriteError};
+use snafu::{Backtrace, IntoError, Snafu};
 
-use crate::util::progress::ProgressSendError;
+use crate::{
+    api::ExportBaoError,
+    get::fsm::{AtBlobHeaderNextError, ConnectedNextError, DecodeError},
+};
+
+#[derive(Debug, Snafu)]
+pub enum NotFoundCases {
+    #[snafu(transparent)]
+    AtBlobHeaderNext { source: AtBlobHeaderNextError },
+    #[snafu(transparent)]
+    Decode { source: DecodeError },
+}
+
+#[derive(Debug, Snafu)]
+pub enum NoncompliantNodeCases {
+    #[snafu(transparent)]
+    Connection { source: ConnectionError },
+    #[snafu(transparent)]
+    Decode { source: DecodeError },
+}
+
+#[derive(Debug, Snafu)]
+pub enum RemoteResetCases {
+    #[snafu(transparent)]
+    Read { source: ReadError },
+    #[snafu(transparent)]
+    Write { source: WriteError },
+    #[snafu(transparent)]
+    Connection { source: ConnectionError },
+}
+
+#[derive(Debug, Snafu)]
+pub enum BadRequestCases {
+    #[snafu(transparent)]
+    Anyhow { source: anyhow::Error },
+    #[snafu(transparent)]
+    Postcard { source: postcard::Error },
+    #[snafu(transparent)]
+    ConnectedNext { source: ConnectedNextError },
+}
+
+#[derive(Debug, Snafu)]
+pub enum LocalFailureCases {
+    #[snafu(transparent)]
+    Io {
+        source: io::Error,
+    },
+    #[snafu(transparent)]
+    Anyhow {
+        source: anyhow::Error,
+    },
+    #[snafu(transparent)]
+    IrpcSend {
+        source: irpc::channel::SendError,
+    },
+    #[snafu(transparent)]
+    Irpc {
+        source: irpc::Error,
+    },
+    #[snafu(transparent)]
+    ExportBao {
+        source: ExportBaoError,
+    },
+    TokioSend {},
+}
+
+impl<T> From<tokio::sync::mpsc::error::SendError<T>> for LocalFailureCases {
+    fn from(_: tokio::sync::mpsc::error::SendError<T>) -> Self {
+        LocalFailureCases::TokioSend {}
+    }
+}
+
+#[derive(Debug, Snafu)]
+pub enum IoCases {
+    #[snafu(transparent)]
+    Io { source: io::Error },
+    #[snafu(transparent)]
+    ConnectionError { source: endpoint::ConnectionError },
+    #[snafu(transparent)]
+    ReadError { source: endpoint::ReadError },
+    #[snafu(transparent)]
+    WriteError { source: endpoint::WriteError },
+    #[snafu(transparent)]
+    ClosedStream { source: endpoint::ClosedStream },
+    #[snafu(transparent)]
+    ConnectedNextError { source: ConnectedNextError },
+    #[snafu(transparent)]
+    AtBlobHeaderNextError { source: AtBlobHeaderNextError },
+}
 
 /// Failures for a get operation
-#[derive(Debug, thiserror::Error)]
+#[common_fields({
+    backtrace: Option<Backtrace>,
+    #[snafu(implicit)]
+    span_trace: SpanTrace,
+})]
+#[derive(Debug, Snafu)]
+#[snafu(visibility(pub(crate)))]
 pub enum GetError {
-    /// Hash not found.
-    #[error("Hash not found")]
-    NotFound(#[source] anyhow::Error),
+    /// Hash not found, or a requested chunk for the hash not found.
+    #[snafu(display("Data for hash not found"))]
+    NotFound {
+        #[snafu(source(from(NotFoundCases, Box::new)))]
+        source: Box<NotFoundCases>,
+    },
     /// Remote has reset the connection.
-    #[error("Remote has reset the connection")]
-    RemoteReset(#[source] anyhow::Error),
+    #[snafu(display("Remote has reset the connection"))]
+    RemoteReset {
+        #[snafu(source(from(RemoteResetCases, Box::new)))]
+        source: Box<RemoteResetCases>,
+    },
     /// Remote behaved in a non-compliant way.
-    #[error("Remote behaved in a non-compliant way")]
-    NoncompliantNode(#[source] anyhow::Error),
+    #[snafu(display("Remote behaved in a non-compliant way"))]
+    NoncompliantNode {
+        #[snafu(source(from(NoncompliantNodeCases, Box::new)))]
+        source: Box<NoncompliantNodeCases>,
+    },
 
     /// Network or IO operation failed.
-    #[error("A network or IO operation failed")]
-    Io(#[source] anyhow::Error),
-
+    #[snafu(display("A network or IO operation failed"))]
+    Io {
+        #[snafu(source(from(IoCases, Box::new)))]
+        source: Box<IoCases>,
+    },
     /// Our download request is invalid.
-    #[error("Our download request is invalid")]
-    BadRequest(#[source] anyhow::Error),
+    #[snafu(display("Our download request is invalid"))]
+    BadRequest {
+        #[snafu(source(from(BadRequestCases, Box::new)))]
+        source: Box<BadRequestCases>,
+    },
     /// Operation failed on the local node.
-    #[error("Operation failed on the local node")]
-    LocalFailure(#[source] anyhow::Error),
+    #[snafu(display("Operation failed on the local node"))]
+    LocalFailure {
+        #[snafu(source(from(LocalFailureCases, Box::new)))]
+        source: Box<LocalFailureCases>,
+    },
+}
+
+pub type GetResult<T> = std::result::Result<T, GetError>;
+
+impl From<irpc::channel::SendError> for GetError {
+    fn from(value: irpc::channel::SendError) -> Self {
+        LocalFailureSnafu.into_error(value.into())
+    }
 }
 
-impl From<ProgressSendError> for GetError {
-    fn from(value: ProgressSendError) -> Self {
-        Self::LocalFailure(value.into())
+impl<T: Send + Sync + 'static> From<tokio::sync::mpsc::error::SendError<T>> for GetError {
+    fn from(value: tokio::sync::mpsc::error::SendError<T>) -> Self {
+        LocalFailureSnafu.into_error(value.into())
     }
 }
 
@@ -43,40 +167,40 @@ impl From<endpoint::ConnectionError> for GetError {
             e @ ConnectionError::VersionMismatch => {
                 // > The peer doesn't implement any supported version
                 // unsupported version is likely a long time error, so this peer is not usable
-                GetError::NoncompliantNode(e.into())
+                NoncompliantNodeSnafu.into_error(e.into())
             }
             e @ ConnectionError::TransportError(_) => {
                 // > The peer violated the QUIC specification as understood by this implementation
                 // bad peer we don't want to keep around
-                GetError::NoncompliantNode(e.into())
+                NoncompliantNodeSnafu.into_error(e.into())
             }
             e @ ConnectionError::ConnectionClosed(_) => {
                 // > The peer's QUIC stack aborted the connection automatically
                 // peer might be disconnecting or otherwise unavailable, drop it
-                GetError::Io(e.into())
+                IoSnafu.into_error(e.into())
             }
             e @ ConnectionError::ApplicationClosed(_) => {
                 // > The peer closed the connection
                 // peer might be disconnecting or otherwise unavailable, drop it
-                GetError::Io(e.into())
+                IoSnafu.into_error(e.into())
             }
             e @ ConnectionError::Reset => {
                 // > The peer is unable to continue processing this connection, usually due to having restarted
-                GetError::RemoteReset(e.into())
+                RemoteResetSnafu.into_error(e.into())
             }
             e @ ConnectionError::TimedOut => {
                 // > Communication with the peer has lapsed for longer than the negotiated idle timeout
-                GetError::Io(e.into())
+                IoSnafu.into_error(e.into())
             }
             e @ ConnectionError::LocallyClosed => {
                 // > The local application closed the connection
                 // TODO(@divma): don't see how this is reachable but let's just not use the peer
-                GetError::Io(e.into())
+                IoSnafu.into_error(e.into())
             }
             e @ ConnectionError::CidsExhausted => {
                 // > The connection could not be created because not enough of the CID space
                 // > is available
-                GetError::Io(e.into())
+                IoSnafu.into_error(e.into())
             }
         }
     }
@@ -86,32 +210,32 @@ impl From<endpoint::ReadError> for GetError {
     fn from(value: endpoint::ReadError) -> Self {
         use endpoint::ReadError;
         match value {
-            e @ ReadError::Reset(_) => GetError::RemoteReset(e.into()),
+            e @ ReadError::Reset(_) => RemoteResetSnafu.into_error(e.into()),
             ReadError::ConnectionLost(conn_error) => conn_error.into(),
             ReadError::ClosedStream
             | ReadError::IllegalOrderedRead
             | ReadError::ZeroRttRejected => {
                 // all these errors indicate the peer is not usable at this moment
-                GetError::Io(value.into())
+                IoSnafu.into_error(value.into())
             }
         }
     }
 }
 impl From<ClosedStream> for GetError {
     fn from(value: ClosedStream) -> Self {
-        GetError::Io(value.into())
+        IoSnafu.into_error(value.into())
     }
 }
 
-impl From<endpoint::WriteError> for GetError {
-    fn from(value: endpoint::WriteError) -> Self {
-        use endpoint::WriteError;
+impl From<quinn::WriteError> for GetError {
+    fn from(value: quinn::WriteError) -> Self {
+        use quinn::WriteError;
         match value {
-            e @ WriteError::Stopped(_) => GetError::RemoteReset(e.into()),
+            e @ WriteError::Stopped(_) => RemoteResetSnafu.into_error(e.into()),
             WriteError::ConnectionLost(conn_error) => conn_error.into(),
             WriteError::ClosedStream | WriteError::ZeroRttRejected => {
                 // all these errors indicate the peer is not usable at this moment
-                GetError::Io(value.into())
+                IoSnafu.into_error(value.into())
             }
         }
     }
@@ -121,19 +245,19 @@ impl From<crate::get::fsm::ConnectedNextError> for GetError {
     fn from(value: crate::get::fsm::ConnectedNextError) -> Self {
         use crate::get::fsm::ConnectedNextError::*;
         match value {
-            e @ PostcardSer(_) => {
+            e @ PostcardSer { .. } => {
                 // serialization errors indicate something wrong with the request itself
-                GetError::BadRequest(e.into())
+                BadRequestSnafu.into_error(e.into())
             }
-            e @ RequestTooBig => {
+            e @ RequestTooBig { .. } => {
                 // request will never be sent, drop it
-                GetError::BadRequest(e.into())
+                BadRequestSnafu.into_error(e.into())
             }
-            Write(e) => e.into(),
-            Closed(e) => e.into(),
-            e @ Io(_) => {
+            Write { source, .. } => source.into(),
+            Closed { source, .. } => source.into(),
+            e @ Io { .. } => {
                 // io errors are likely recoverable
-                GetError::Io(e.into())
+                IoSnafu.into_error(e.into())
             }
         }
     }
@@ -143,15 +267,15 @@ impl From<crate::get::fsm::AtBlobHeaderNextError> for GetError {
     fn from(value: crate::get::fsm::AtBlobHeaderNextError) -> Self {
         use crate::get::fsm::AtBlobHeaderNextError::*;
         match value {
-            e @ NotFound => {
+            e @ NotFound { .. } => {
                 // > This indicates that the provider does not have the requested data.
                 // peer might have the data later, simply retry it
-                GetError::NotFound(e.into())
+                NotFoundSnafu.into_error(e.into())
             }
-            Read(e) => e.into(),
-            e @ Io(_) => {
+            EndpointRead { source, .. } => source.into(),
+            e @ Io { .. } => {
                 // io errors are likely recoverable
-                GetError::Io(e.into())
+                IoSnafu.into_error(e.into())
             }
         }
     }
@@ -162,21 +286,21 @@ impl From<crate::get::fsm::DecodeError> for GetError {
         use crate::get::fsm::DecodeError::*;
 
         match value {
-            e @ NotFound => GetError::NotFound(e.into()),
-            e @ ParentNotFound(_) => GetError::NotFound(e.into()),
-            e @ LeafNotFound(_) => GetError::NotFound(e.into()),
-            e @ ParentHashMismatch(_) => {
+            e @ ChunkNotFound { .. } => NotFoundSnafu.into_error(e.into()),
+            e @ ParentNotFound { .. } => NotFoundSnafu.into_error(e.into()),
+            e @ LeafNotFound { .. } => NotFoundSnafu.into_error(e.into()),
+            e @ ParentHashMismatch { .. } => {
                 // TODO(@divma): did the peer sent wrong data? is it corrupted? did we sent a wrong
                 // request?
-                GetError::NoncompliantNode(e.into())
+                NoncompliantNodeSnafu.into_error(e.into())
             }
-            e @ LeafHashMismatch(_) => {
+            e @ LeafHashMismatch { .. } => {
                 // TODO(@divma): did the peer sent wrong data? is it corrupted? did we sent a wrong
                 // request?
-                GetError::NoncompliantNode(e.into())
+                NoncompliantNodeSnafu.into_error(e.into())
             }
-            Read(e) => e.into(),
-            Io(e) => e.into(),
+            Read { source, .. } => source.into(),
+            DecodeIo { source, .. } => source.into(),
         }
     }
 }
@@ -185,6 +309,6 @@ impl From<std::io::Error> for GetError {
     fn from(value: std::io::Error) -> Self {
         // generally consider io errors recoverable
         // we might want to revisit this at some point
-        GetError::Io(value.into())
+        IoSnafu.into_error(value.into())
     }
 }
diff --git a/src/get/progress.rs b/src/get/progress.rs
deleted file mode 100644
index d4025e5c3..000000000
--- a/src/get/progress.rs
+++ /dev/null
@@ -1,185 +0,0 @@
-//! Types for get progress state management.
-
-use std::{collections::HashMap, num::NonZeroU64};
-
-use serde::{Deserialize, Serialize};
-use tracing::warn;
-
-use super::db::{BlobId, DownloadProgress};
-use crate::{protocol::RangeSpec, store::BaoBlobSize, Hash};
-
-/// The identifier for progress events.
-pub type ProgressId = u64;
-
-/// Accumulated progress state of a transfer.
-#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
-pub struct TransferState {
-    /// The root blob of this transfer (may be a hash seq),
-    pub root: BlobState,
-    /// Whether we are connected to a node
-    pub connected: bool,
-    /// Children if the root blob is a hash seq, empty for raw blobs
-    pub children: HashMap<NonZeroU64, BlobState>,
-    /// Child being transferred at the moment.
-    pub current: Option<BlobId>,
-    /// Progress ids for individual blobs.
-    pub progress_id_to_blob: HashMap<ProgressId, BlobId>,
-}
-
-impl TransferState {
-    /// Create a new, empty transfer state.
-    pub fn new(root_hash: Hash) -> Self {
-        Self {
-            root: BlobState::new(root_hash),
-            connected: false,
-            children: Default::default(),
-            current: None,
-            progress_id_to_blob: Default::default(),
-        }
-    }
-}
-
-/// State of a single blob in transfer
-#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
-pub struct BlobState {
-    /// The hash of this blob.
-    pub hash: Hash,
-    /// The size of this blob. Only known if the blob is partially present locally, or after having
-    /// received the size from the remote.
-    pub size: Option<BaoBlobSize>,
-    /// The current state of the blob transfer.
-    pub progress: BlobProgress,
-    /// Ranges already available locally at the time of starting the transfer.
-    pub local_ranges: Option<RangeSpec>,
-    /// Number of children (only applies to hashseqs, None for raw blobs).
-    pub child_count: Option<u64>,
-}
-
-/// Progress state for a single blob
-#[derive(Debug, Default, Clone, Serialize, Deserialize, Eq, PartialEq)]
-pub enum BlobProgress {
-    /// Download is pending
-    #[default]
-    Pending,
-    /// Download is in progress
-    Progressing(u64),
-    /// Download has finished
-    Done,
-}
-
-impl BlobState {
-    /// Create a new [`BlobState`].
-    pub fn new(hash: Hash) -> Self {
-        Self {
-            hash,
-            size: None,
-            local_ranges: None,
-            child_count: None,
-            progress: BlobProgress::default(),
-        }
-    }
-}
-
-impl TransferState {
-    /// Get state of the root blob of this transfer.
-    pub fn root(&self) -> &BlobState {
-        &self.root
-    }
-
-    /// Get a blob state by its [`BlobId`] in this transfer.
-    pub fn get_blob(&self, blob_id: &BlobId) -> Option<&BlobState> {
-        match blob_id {
-            BlobId::Root => Some(&self.root),
-            BlobId::Child(id) => self.children.get(id),
-        }
-    }
-
-    /// Get the blob state currently being transferred.
-    pub fn get_current(&self) -> Option<&BlobState> {
-        self.current.as_ref().and_then(|id| self.get_blob(id))
-    }
-
-    fn get_or_insert_blob(&mut self, blob_id: BlobId, hash: Hash) -> &mut BlobState {
-        match blob_id {
-            BlobId::Root => &mut self.root,
-            BlobId::Child(id) => self
-                .children
-                .entry(id)
-                .or_insert_with(|| BlobState::new(hash)),
-        }
-    }
-    fn get_blob_mut(&mut self, blob_id: &BlobId) -> Option<&mut BlobState> {
-        match blob_id {
-            BlobId::Root => Some(&mut self.root),
-            BlobId::Child(id) => self.children.get_mut(id),
-        }
-    }
-
-    fn get_by_progress_id(&mut self, progress_id: ProgressId) -> Option<&mut BlobState> {
-        let blob_id = *self.progress_id_to_blob.get(&progress_id)?;
-        self.get_blob_mut(&blob_id)
-    }
-
-    /// Update the state with a new [`DownloadProgress`] event for this transfer.
-    pub fn on_progress(&mut self, event: DownloadProgress) {
-        match event {
-            DownloadProgress::InitialState(s) => {
-                *self = s;
-            }
-            DownloadProgress::FoundLocal {
-                child,
-                hash,
-                size,
-                valid_ranges,
-            } => {
-                let blob = self.get_or_insert_blob(child, hash);
-                blob.size = Some(size);
-                blob.local_ranges = Some(valid_ranges);
-            }
-            DownloadProgress::Connected => self.connected = true,
-            DownloadProgress::Found {
-                id: progress_id,
-                child: blob_id,
-                hash,
-                size,
-            } => {
-                let blob = self.get_or_insert_blob(blob_id, hash);
-                blob.size = match blob.size {
-                    // If we don't have a verified size for this blob yet: Use the size as reported
-                    // by the remote.
-                    None | Some(BaoBlobSize::Unverified(_)) => Some(BaoBlobSize::Unverified(size)),
-                    // Otherwise, keep the existing verified size.
-                    value @ Some(BaoBlobSize::Verified(_)) => value,
-                };
-                blob.progress = BlobProgress::Progressing(0);
-                self.progress_id_to_blob.insert(progress_id, blob_id);
-                self.current = Some(blob_id);
-            }
-            DownloadProgress::FoundHashSeq { hash, children } => {
-                if hash == self.root.hash {
-                    self.root.child_count = Some(children);
-                } else {
-                    // I think it is an invariant of the protocol that `FoundHashSeq` is only
-                    // triggered for the root hash.
-                    warn!("Received `FoundHashSeq` event for a hash which is not the download's root hash.")
-                }
-            }
-            DownloadProgress::Progress { id, offset } => {
-                if let Some(blob) = self.get_by_progress_id(id) {
-                    blob.progress = BlobProgress::Progressing(offset);
-                } else {
-                    warn!(%id, "Received `Progress` event for unknown progress id.")
-                }
-            }
-            DownloadProgress::Done { id } => {
-                if let Some(blob) = self.get_by_progress_id(id) {
-                    blob.progress = BlobProgress::Done;
-                    self.progress_id_to_blob.remove(&id);
-                } else {
-                    warn!(%id, "Received `Done` event for unknown progress id.")
-                }
-            }
-            DownloadProgress::AllDone(_) | DownloadProgress::Abort(_) => {}
-        }
-    }
-}
diff --git a/src/get/request.rs b/src/get/request.rs
index 54a52f2b8..86ffcabb2 100644
--- a/src/get/request.rs
+++ b/src/get/request.rs
@@ -1,31 +1,166 @@
-//! Utilities for complex get requests.
-use std::sync::Arc;
+//! Utilities to generate or execute complex get requests without persisting to a store.
+//!
+//! Any complex request can be executed with downloading to a store, using the
+//! [`crate::api::remote::Remote::execute_get`] method. But for some requests it
+//! is useful to just get the data without persisting it to a store.
+//!
+//! In addition to these utilities, there are also constructors in [`crate::protocol::ChunkRangesSeq`]
+//! to construct complex requests.
+use std::{
+    future::{Future, IntoFuture},
+    pin::Pin,
+    sync::Arc,
+    task::{Context, Poll},
+};
 
-use bao_tree::{ChunkNum, ChunkRanges};
+use bao_tree::{io::BaoContentItem, ChunkNum, ChunkRanges};
 use bytes::Bytes;
+use genawaiter::sync::{Co, Gen};
 use iroh::endpoint::Connection;
+use n0_future::{Stream, StreamExt};
+use nested_enum_utils::enum_conversions;
 use rand::Rng;
+use snafu::IntoError;
+use tokio::sync::mpsc;
 
-use super::{fsm, Stats};
+use super::{fsm, GetError, GetResult, Stats};
 use crate::{
+    get::error::{BadRequestSnafu, LocalFailureSnafu},
     hashseq::HashSeq,
-    protocol::{GetRequest, RangeSpecSeq},
+    protocol::{ChunkRangesSeq, GetRequest},
+    util::ChunkRangesExt,
     Hash, HashAndFormat,
 };
 
+/// Result of a [`get_blob`] request.
+///
+/// This is a stream of [`GetBlobItem`]s. You can also await it to get just
+/// the bytes of the blob.
+pub struct GetBlobResult {
+    rx: n0_future::stream::Boxed<GetBlobItem>,
+}
+
+impl IntoFuture for GetBlobResult {
+    type Output = GetResult<Bytes>;
+    type IntoFuture = Pin<Box<dyn Future<Output = Self::Output> + Send>>;
+
+    fn into_future(self) -> Self::IntoFuture {
+        Box::pin(self.bytes())
+    }
+}
+
+impl GetBlobResult {
+    pub async fn bytes(self) -> GetResult<Bytes> {
+        let (bytes, _) = self.bytes_and_stats().await?;
+        Ok(bytes)
+    }
+
+    pub async fn bytes_and_stats(mut self) -> GetResult<(Bytes, Stats)> {
+        let mut parts = Vec::new();
+        let stats = loop {
+            let Some(item) = self.next().await else {
+                return Err(LocalFailureSnafu.into_error(anyhow::anyhow!("unexpected end").into()));
+            };
+            match item {
+                GetBlobItem::Item(item) => {
+                    if let BaoContentItem::Leaf(leaf) = item {
+                        parts.push(leaf.data);
+                    }
+                }
+                GetBlobItem::Done(stats) => {
+                    break stats;
+                }
+                GetBlobItem::Error(cause) => {
+                    return Err(cause);
+                }
+            }
+        };
+        let bytes = if parts.len() == 1 {
+            parts.pop().unwrap()
+        } else {
+            let mut bytes = Vec::new();
+            for part in parts {
+                bytes.extend_from_slice(&part);
+            }
+            bytes.into()
+        };
+        Ok((bytes, stats))
+    }
+}
+
+impl Stream for GetBlobResult {
+    type Item = GetBlobItem;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
+        self.rx.poll_next(cx)
+    }
+}
+
+/// A single item in a [`GetBlobResult`].
+#[derive(Debug)]
+#[enum_conversions()]
+pub enum GetBlobItem {
+    /// Content
+    Item(BaoContentItem),
+    /// Request completed successfully
+    Done(Stats),
+    /// Request failed
+    Error(GetError),
+}
+
+pub fn get_blob(connection: Connection, hash: Hash) -> GetBlobResult {
+    let generator = Gen::new(|co| async move {
+        if let Err(cause) = get_blob_impl(&connection, &hash, &co).await {
+            co.yield_(GetBlobItem::Error(cause)).await;
+        }
+    });
+    GetBlobResult {
+        rx: Box::pin(generator),
+    }
+}
+
+async fn get_blob_impl(
+    connection: &Connection,
+    hash: &Hash,
+    co: &Co<GetBlobItem>,
+) -> GetResult<()> {
+    let request = GetRequest::blob(*hash);
+    let request = fsm::start(connection.clone(), request, Default::default());
+    let connected = request.next().await?;
+    let fsm::ConnectedNext::StartRoot(start) = connected.next().await? else {
+        unreachable!("expected start root");
+    };
+    let header = start.next();
+    let (mut curr, _size) = header.next().await?;
+    let end = loop {
+        match curr.next().await {
+            fsm::BlobContentNext::More((next, res)) => {
+                co.yield_(res?.into()).await;
+                curr = next;
+            }
+            fsm::BlobContentNext::Done(end) => {
+                break end;
+            }
+        }
+    };
+    let fsm::EndBlobNext::Closing(closing) = end.next() else {
+        unreachable!("expected closing");
+    };
+    let stats = closing.next().await?;
+    co.yield_(stats.into()).await;
+    Ok(())
+}
+
 /// Get the claimed size of a blob from a peer.
 ///
 /// This is just reading the size header and then immediately closing the connection.
 /// It can be used to check if a peer has any data at all.
-pub async fn get_unverified_size(
-    connection: &Connection,
-    hash: &Hash,
-) -> anyhow::Result<(u64, Stats)> {
+pub async fn get_unverified_size(connection: &Connection, hash: &Hash) -> GetResult<(u64, Stats)> {
     let request = GetRequest::new(
         *hash,
-        RangeSpecSeq::from_ranges(vec![ChunkRanges::from(ChunkNum(u64::MAX)..)]),
+        ChunkRangesSeq::from_ranges(vec![ChunkRanges::last_chunk()]),
     );
-    let request = fsm::start(connection.clone(), request);
+    let request = fsm::start(connection.clone(), request, Default::default());
     let connected = request.next().await?;
     let fsm::ConnectedNext::StartRoot(start) = connected.next().await? else {
         unreachable!("expected start root");
@@ -40,16 +175,13 @@ pub async fn get_unverified_size(
 ///
 /// This asks for the last chunk of the blob and validates the response.
 /// Note that this does not validate that the peer has all the data.
-pub async fn get_verified_size(
-    connection: &Connection,
-    hash: &Hash,
-) -> anyhow::Result<(u64, Stats)> {
+pub async fn get_verified_size(connection: &Connection, hash: &Hash) -> GetResult<(u64, Stats)> {
     tracing::trace!("Getting verified size of {}", hash.to_hex());
     let request = GetRequest::new(
         *hash,
-        RangeSpecSeq::from_ranges(vec![ChunkRanges::from(ChunkNum(u64::MAX)..)]),
+        ChunkRangesSeq::from_ranges(vec![ChunkRanges::last_chunk()]),
     );
-    let request = fsm::start(connection.clone(), request);
+    let request = fsm::start(connection.clone(), request, Default::default());
     let connected = request.next().await?;
     let fsm::ConnectedNext::StartRoot(start) = connected.next().await? else {
         unreachable!("expected start root");
@@ -82,22 +214,23 @@ pub async fn get_verified_size(
 /// Given a hash of a hash seq, get the hash seq and the verified sizes of its
 /// children.
 ///
+/// If this operation succeeds we have a strong indication that the peer has
+/// the hash seq and the last chunk of each child.
+///
 /// This can be used to compute the total size when requesting a hash seq.
 pub async fn get_hash_seq_and_sizes(
     connection: &Connection,
     hash: &Hash,
     max_size: u64,
-) -> anyhow::Result<(HashSeq, Arc<[u64]>)> {
+    _progress: Option<mpsc::Sender<u64>>,
+) -> GetResult<(HashSeq, Arc<[u64]>)> {
     let content = HashAndFormat::hash_seq(*hash);
     tracing::debug!("Getting hash seq and children sizes of {}", content);
     let request = GetRequest::new(
         *hash,
-        RangeSpecSeq::from_ranges_infinite([
-            ChunkRanges::all(),
-            ChunkRanges::from(ChunkNum(u64::MAX)..),
-        ]),
+        ChunkRangesSeq::from_ranges_infinite([ChunkRanges::all(), ChunkRanges::last_chunk()]),
     );
-    let at_start = fsm::start(connection.clone(), request);
+    let at_start = fsm::start(connection.clone(), request, Default::default());
     let at_connected = at_start.next().await?;
     let fsm::ConnectedNext::StartRoot(start) = at_connected.next().await? else {
         unreachable!("query includes root");
@@ -106,10 +239,11 @@ pub async fn get_hash_seq_and_sizes(
     let (at_blob_content, size) = at_start_root.next().await?;
     // check the size to avoid parsing a maliciously large hash seq
     if size > max_size {
-        anyhow::bail!("size too large");
+        return Err(BadRequestSnafu.into_error(anyhow::anyhow!("size too large").into()));
     }
     let (mut curr, hash_seq) = at_blob_content.concatenate_into_vec().await?;
-    let hash_seq = HashSeq::try_from(Bytes::from(hash_seq))?;
+    let hash_seq = HashSeq::try_from(Bytes::from(hash_seq))
+        .map_err(|e| BadRequestSnafu.into_error(e.into()))?;
     let mut sizes = Vec::with_capacity(hash_seq.len());
     let closing = loop {
         match curr.next() {
@@ -138,16 +272,24 @@ pub async fn get_hash_seq_and_sizes(
 
 /// Probe for a single chunk of a blob.
 ///
-/// This is used to check if a peer has a specific chunk.
+/// This is used to check if a peer has a specific chunk. If the operation
+/// is successful, we have a strong indication that the peer had the chunk at
+/// the time of the request.
+///
+/// If the operation fails, either the connection failed or the peer did not
+/// have the chunk.
+///
+/// It is usually not very helpful to try to distinguish between these two
+/// cases.
 pub async fn get_chunk_probe(
     connection: &Connection,
     hash: &Hash,
     chunk: ChunkNum,
-) -> anyhow::Result<Stats> {
+) -> GetResult<Stats> {
     let ranges = ChunkRanges::from(chunk..chunk + 1);
-    let ranges = RangeSpecSeq::from_ranges([ranges]);
+    let ranges = ChunkRangesSeq::from_ranges([ranges]);
     let request = GetRequest::new(*hash, ranges);
-    let request = fsm::start(connection.clone(), request);
+    let request = fsm::start(connection.clone(), request, Default::default());
     let connected = request.next().await?;
     let fsm::ConnectedNext::StartRoot(start) = connected.next().await? else {
         unreachable!("query includes root");
@@ -177,7 +319,7 @@ pub async fn get_chunk_probe(
 ///
 /// The random chunk is chosen uniformly from the chunks of the children, so
 /// larger children are more likely to be selected.
-pub fn random_hash_seq_ranges(sizes: &[u64], mut rng: impl Rng) -> RangeSpecSeq {
+pub fn random_hash_seq_ranges(sizes: &[u64], mut rng: impl Rng) -> ChunkRangesSeq {
     let total_chunks = sizes
         .iter()
         .map(|size| ChunkNum::full_chunks(*size).0)
@@ -198,5 +340,5 @@ pub fn random_hash_seq_ranges(sizes: &[u64], mut rng: impl Rng) -> RangeSpecSeq
             ranges.push(ChunkRanges::empty());
         }
     }
-    RangeSpecSeq::from_ranges(ranges)
+    ChunkRangesSeq::from_ranges(ranges)
 }
diff --git a/src/hash.rs b/src/hash.rs
index dda748348..8190009aa 100644
--- a/src/hash.rs
+++ b/src/hash.rs
@@ -2,8 +2,15 @@
 
 use std::{borrow::Borrow, fmt, str::FromStr};
 
+use arrayvec::ArrayString;
+use bao_tree::blake3;
+use n0_snafu::SpanTrace;
+use nested_enum_utils::common_fields;
 use postcard::experimental::max_size::MaxSize;
 use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
+use snafu::{Backtrace, ResultExt, Snafu};
+
+use crate::store::util::DD;
 
 /// Hash type used throughout.
 #[derive(PartialEq, Eq, Copy, Clone, Hash)]
@@ -15,14 +22,6 @@ impl fmt::Debug for Hash {
     }
 }
 
-struct DD<T: fmt::Display>(T);
-
-impl<T: fmt::Display> fmt::Debug for DD<T> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        fmt::Display::fmt(&self.0, f)
-    }
-}
-
 impl Hash {
     /// The hash for the empty byte range (`b""`).
     pub const EMPTY: Hash = Hash::from_bytes([
@@ -53,8 +52,12 @@ impl Hash {
 
     /// Convert to a hex string limited to the first 5bytes for a friendly string
     /// representation of the hash.
-    pub fn fmt_short(&self) -> String {
-        data_encoding::HEXLOWER.encode(&self.as_bytes()[..5])
+    pub fn fmt_short(&self) -> ArrayString<10> {
+        let mut res = ArrayString::new();
+        data_encoding::HEXLOWER
+            .encode_write(&self.as_bytes()[..5], &mut res)
+            .unwrap();
+        res
     }
 }
 
@@ -120,25 +123,33 @@ impl Ord for Hash {
 
 impl fmt::Display for Hash {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        // result will be 52 bytes
-        let mut res = [b'b'; 52];
-        // write the encoded bytes
-        data_encoding::BASE32_NOPAD.encode_mut(self.as_bytes(), &mut res);
-        // convert to string, this is guaranteed to succeed
-        let t = std::str::from_utf8_mut(res.as_mut()).unwrap();
-        // hack since data_encoding doesn't have BASE32LOWER_NOPAD as a const
-        t.make_ascii_lowercase();
-        // write the str, no allocations
-        f.write_str(t)
+        f.write_str(&self.to_hex())
+        // // result will be 52 bytes
+        // let mut res = [b'b'; 52];
+        // // write the encoded bytes
+        // data_encoding::BASE32_NOPAD.encode_mut(self.as_bytes(), &mut res);
+        // // convert to string, this is guaranteed to succeed
+        // let t = std::str::from_utf8_mut(res.as_mut()).unwrap();
+        // // hack since data_encoding doesn't have BASE32LOWER_NOPAD as a const
+        // t.make_ascii_lowercase();
+        // // write the str, no allocations
+        // f.write_str(t)
     }
 }
 
-#[derive(Debug, thiserror::Error)]
+#[common_fields({
+    backtrace: Option<Backtrace>,
+    #[snafu(implicit)]
+    span_trace: SpanTrace,
+})]
+#[allow(missing_docs)]
+#[non_exhaustive]
+#[derive(Debug, Snafu)]
 pub enum HexOrBase32ParseError {
-    #[error("Invalid length")]
-    DecodeInvalidLength,
-    #[error("Failed to decode {0}")]
-    Decode(#[from] data_encoding::DecodeError),
+    #[snafu(display("Invalid length"))]
+    DecodeInvalidLength {},
+    #[snafu(display("Failed to decode {source}"))]
+    Decode { source: data_encoding::DecodeError },
 }
 
 impl FromStr for Hash {
@@ -151,20 +162,15 @@ impl FromStr for Hash {
             // hex
             data_encoding::HEXLOWER.decode_mut(s.as_bytes(), &mut bytes)
         } else {
-            let input = s.to_ascii_uppercase();
-            let input = input.as_bytes();
-            if data_encoding::BASE32_NOPAD.decode_len(input.len())? != bytes.len() {
-                return Err(HexOrBase32ParseError::DecodeInvalidLength);
-            }
-            data_encoding::BASE32_NOPAD.decode_mut(input, &mut bytes)
+            data_encoding::BASE32_NOPAD.decode_mut(s.to_ascii_uppercase().as_bytes(), &mut bytes)
         };
         match res {
             Ok(len) => {
                 if len != 32 {
-                    return Err(HexOrBase32ParseError::DecodeInvalidLength);
+                    return Err(DecodeInvalidLengthSnafu.build());
                 }
             }
-            Err(partial) => return Err(partial.error.into()),
+            Err(partial) => return Err(partial.error).context(DecodeSnafu),
         }
         Ok(Self(blake3::Hash::from_bytes(bytes)))
     }
@@ -248,7 +254,7 @@ impl BlobFormat {
 }
 
 /// A hash and format pair
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, MaxSize, Hash)]
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, MaxSize, Hash)]
 pub struct HashAndFormat {
     /// The hash
     pub hash: Hash,
@@ -256,13 +262,28 @@ pub struct HashAndFormat {
     pub format: BlobFormat,
 }
 
+impl std::fmt::Debug for HashAndFormat {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        std::fmt::Debug::fmt(&(DD(self.hash.to_hex()), self.format), f)
+    }
+}
+
+impl From<(Hash, BlobFormat)> for HashAndFormat {
+    fn from((hash, format): (Hash, BlobFormat)) -> Self {
+        Self { hash, format }
+    }
+}
+
 impl From<Hash> for HashAndFormat {
     fn from(hash: Hash) -> Self {
-        Self::raw(hash)
+        Self {
+            hash,
+            format: BlobFormat::Raw,
+        }
     }
 }
 
-#[cfg(feature = "redb")]
+// #[cfg(feature = "redb")]
 mod redb_support {
     use postcard::experimental::max_size::MaxSize;
     use redb::{Key as RedbKey, Value as RedbValue};
@@ -427,10 +448,11 @@ impl<'de> Deserialize<'de> for HashAndFormat {
 
 #[cfg(test)]
 mod tests {
+
+    use iroh_test::{assert_eq_hex, hexdump::parse_hexdump};
     use serde_test::{assert_tokens, Configure, Token};
 
     use super::*;
-    use crate::{assert_eq_hex, util::hexdump::parse_hexdump};
 
     #[test]
     fn test_display_parse_roundtrip() {
@@ -471,7 +493,7 @@ mod tests {
         assert_eq_hex!(serialized, expected);
     }
 
-    #[cfg(feature = "redb")]
+    // #[cfg(feature = "redb")]
     #[test]
     fn hash_redb() {
         use redb::Value as RedbValue;
@@ -496,7 +518,7 @@ mod tests {
         assert_eq_hex!(serialized, expected);
     }
 
-    #[cfg(feature = "redb")]
+    // #[cfg(feature = "redb")]
     #[test]
     fn hash_and_format_redb() {
         use redb::Value as RedbValue;
@@ -541,7 +563,7 @@ mod tests {
         assert_tokens(&hash.compact(), &tokens);
 
         let tokens = vec![Token::String(
-            "5khrmpntq2bjexseshc6ldklwnig56gbj23yvbxjbdcwestheahq",
+            "ea8f163db38682925e4491c5e58d4bb3506ef8c14eb78a86e908c5624a67200f",
         )];
         assert_tokens(&hash.readable(), &tokens);
     }
@@ -563,7 +585,7 @@ mod tests {
         let de = serde_json::from_str(&ser).unwrap();
         assert_eq!(hash, de);
         // 52 bytes of base32 + 2 quotes
-        assert_eq!(ser.len(), 54);
+        assert_eq!(ser.len(), 66);
     }
 
     #[test]
@@ -594,9 +616,4 @@ mod tests {
         let de = serde_json::from_str(&ser).unwrap();
         assert_eq!(haf, de);
     }
-
-    #[test]
-    fn test_hash_invalid() {
-        let _ = Hash::from_str("invalid").unwrap_err();
-    }
 }
diff --git a/src/hashseq.rs b/src/hashseq.rs
index f77f8b81d..bcbebf98a 100644
--- a/src/hashseq.rs
+++ b/src/hashseq.rs
@@ -1,4 +1,4 @@
-//! traits related to collections of blobs
+//! Helpers for blobs that contain a sequence of hashes.
 use std::{fmt::Debug, io};
 
 use bytes::Bytes;
@@ -7,9 +7,21 @@ use iroh_io::{AsyncSliceReader, AsyncSliceReaderExt};
 use crate::Hash;
 
 /// A sequence of links, backed by a [`Bytes`] object.
-#[derive(Debug, Clone, derive_more::Into)]
+#[derive(Clone, derive_more::Into)]
 pub struct HashSeq(Bytes);
 
+impl Debug for HashSeq {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_list().entries(self.iter()).finish()
+    }
+}
+
+impl<'a> FromIterator<&'a Hash> for HashSeq {
+    fn from_iter<T: IntoIterator<Item = &'a Hash>>(iter: T) -> Self {
+        iter.into_iter().copied().collect()
+    }
+}
+
 impl FromIterator<Hash> for HashSeq {
     fn from_iter<T: IntoIterator<Item = Hash>>(iter: T) -> Self {
         let iter = iter.into_iter();
diff --git a/src/lib.rs b/src/lib.rs
index fc9e397bb..3c4b38018 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,73 +1,20 @@
-#![doc = include_str!("../README.md")]
-//! Blobs layer for iroh.
-//!
-//! The crate is designed to be used from the [iroh] crate, which provides a
-//! [high level interface](https://docs.rs/iroh/latest/iroh/client/blobs/index.html),
-//! but can also be used standalone.
-//!
-//! It implements a [protocol] for streaming content-addressed data transfer using
-//! [BLAKE3] verified streaming.
-//!
-//! It also provides a [store] interface for storage of blobs and outboards,
-//! as well as a [persistent](crate::store::fs) and a [memory](crate::store::mem)
-//! store implementation.
-//!
-//! To implement a server, the [provider] module provides helpers for handling
-//! connections and individual requests given a store.
-//!
-//! To perform get requests, the [get] module provides utilities to perform
-//! requests and store the result in a store, as well as a low level state
-//! machine for executing requests.
-//!
-//! The [downloader] module provides a component to download blobs from
-//! multiple sources and store them in a store.
-//!
-//! # Feature flags
-//!
-//! - rpc: Enable the rpc server and client. Enabled by default.
-//! - net_protocol: Enable the network protocol. Enabled by default.
-//! - downloader: Enable the downloader. Enabled by default.
-//! - fs-store: Enable the filesystem store. Enabled by default.
-//!
-//! - cli: Enable the cli. Disabled by default.
-//! - example-iroh: dependencies for examples in this crate. Disabled by default.
-//! - test: test utilities. Disabled by default.
-//!
-//! [BLAKE3]: https://github.com/BLAKE3-team/BLAKE3-specs/blob/master/blake3.pdf
-//! [iroh]: https://docs.rs/iroh
-#![deny(missing_docs, rustdoc::broken_intra_doc_links)]
-#![recursion_limit = "256"]
-#![cfg_attr(iroh_docsrs, feature(doc_auto_cfg))]
+mod hash;
+pub mod store;
+pub use hash::{BlobFormat, Hash, HashAndFormat};
+pub mod api;
 
-#[cfg(feature = "cli")]
-pub mod cli;
-#[cfg(feature = "downloader")]
-pub mod downloader;
-pub mod export;
 pub mod format;
 pub mod get;
 pub mod hashseq;
-pub mod metrics;
-#[cfg(feature = "net_protocol")]
+mod metrics;
 pub mod net_protocol;
 pub mod protocol;
 pub mod provider;
-#[cfg(feature = "rpc")]
-pub mod rpc;
-pub mod store;
+pub mod test;
 pub mod ticket;
 pub mod util;
 
-mod hash;
-
-use bao_tree::BlockSize;
-
-#[doc(inline)]
-pub use crate::protocol::ALPN;
-pub use crate::{
-    hash::{BlobFormat, Hash, HashAndFormat},
-    util::{Tag, TempTag},
-};
+#[cfg(test)]
+mod tests;
 
-/// Block size used by iroh, 2^4*1024 = 16KiB
-pub const IROH_BLOCK_SIZE: BlockSize = BlockSize::from_chunk_log(4);
+pub use protocol::ALPN;
diff --git a/src/metrics.rs b/src/metrics.rs
index b4d25de53..c47fb6eae 100644
--- a/src/metrics.rs
+++ b/src/metrics.rs
@@ -3,7 +3,8 @@
 use iroh_metrics::{Counter, MetricsGroup};
 
 /// Enum of metrics for the module
-#[derive(Debug, MetricsGroup, Default)]
+#[allow(missing_docs)]
+#[derive(Debug, Default, MetricsGroup)]
 #[metrics(name = "iroh-blobs")]
 pub struct Metrics {
     /// Total number of content bytes downloaded
@@ -16,25 +17,18 @@ pub struct Metrics {
     pub downloads_error: Counter,
     /// Total number of downloads failed with not found
     pub downloads_notfound: Counter,
-
-    /// Number of times the main pub downloader actor loop ticked
+    /// Number of times the main downloader actor loop ticked
     pub downloader_tick_main: Counter,
-
-    /// Number of times the pub downloader actor ticked for a connection ready
+    /// Number of times the downloader actor ticked for a connection ready
     pub downloader_tick_connection_ready: Counter,
-
-    /// Number of times the pub downloader actor ticked for a message received
+    /// Number of times the downloader actor ticked for a message received
     pub downloader_tick_message_received: Counter,
-
-    /// Number of times the pub downloader actor ticked for a transfer completed
+    /// Number of times the downloader actor ticked for a transfer completed
     pub downloader_tick_transfer_completed: Counter,
-
-    /// Number of times the pub downloader actor ticked for a transfer failed
+    /// Number of times the downloader actor ticked for a transfer failed
     pub downloader_tick_transfer_failed: Counter,
-
-    /// Number of times the pub downloader actor ticked for a retry node
+    /// Number of times the downloader actor ticked for a retry node
     pub downloader_tick_retry_node: Counter,
-
-    /// Number of times the pub downloader actor ticked for a goodbye node
+    /// Number of times the downloader actor ticked for a goodbye node
     pub downloader_tick_goodbye_node: Counter,
 }
diff --git a/src/net_protocol.rs b/src/net_protocol.rs
index 8017248da..63b50bdb1 100644
--- a/src/net_protocol.rs
+++ b/src/net_protocol.rs
@@ -1,388 +1,122 @@
-//! Adaptation of `iroh-blobs` as an `iroh` protocol.
-
-// TODO: reduce API surface and add documentation
-#![allow(missing_docs)]
-
-use std::{
-    collections::BTreeSet,
-    fmt::Debug,
-    ops::{Deref, DerefMut},
-    sync::Arc,
+//! Adaptation of `iroh-blobs` as an [`iroh`] [`ProtocolHandler`].
+//!
+//! This is the easiest way to share data from a [`crate::api::Store`] over iroh connections.
+//!
+//! # Example
+//!
+//! ```rust
+//! # async fn example() -> anyhow::Result<()> {
+//! use iroh::{protocol::Router, Endpoint};
+//! use iroh_blobs::{net_protocol::Blobs, store};
+//!
+//! // create a store
+//! let store = store::fs::FsStore::load("blobs").await?;
+//!
+//! // add some data
+//! let t = store.add_slice(b"hello world").await?;
+//!
+//! // create an iroh endpoint
+//! let endpoint = Endpoint::builder().discovery_n0().bind().await?;
+//!
+//! // create a blobs protocol handler
+//! let blobs = Blobs::new(&store, endpoint.clone(), None);
+//!
+//! // create a router and add the blobs protocol handler
+//! let router = Router::builder(endpoint)
+//!     .accept(iroh_blobs::ALPN, blobs.clone())
+//!     .spawn();
+//!
+//! // this data is now globally available using the ticket
+//! let ticket = blobs.ticket(t).await?;
+//! println!("ticket: {}", ticket);
+//!
+//! // wait for control-c to exit
+//! tokio::signal::ctrl_c().await?;
+//! #   Ok(())
+//! # }
+//! ```
+
+use std::{fmt::Debug, future::Future, sync::Arc};
+
+use iroh::{
+    endpoint::Connection,
+    protocol::{AcceptError, ProtocolHandler},
+    Endpoint, Watcher,
 };
-
-use anyhow::{bail, Result};
-use futures_lite::future::Boxed as BoxedFuture;
-use futures_util::future::BoxFuture;
-use iroh::{endpoint::Connection, protocol::ProtocolHandler, Endpoint, NodeAddr};
-use serde::{Deserialize, Serialize};
-use tracing::debug;
+use tokio::sync::mpsc;
+use tracing::error;
 
 use crate::{
-    downloader::{self, ConcurrencyLimits, Downloader, RetryConfig},
-    metrics::Metrics,
-    provider::EventSender,
-    store::GcConfig,
-    util::{
-        local_pool::{self, LocalPool, LocalPoolHandle},
-        SetTagOption,
-    },
-    BlobFormat, Hash,
+    api::Store,
+    provider::{Event, EventSender},
+    ticket::BlobTicket,
+    HashAndFormat,
 };
 
-/// A callback that blobs can ask about a set of hashes that should not be garbage collected.
-pub type ProtectCb = Box<dyn Fn(&mut BTreeSet<Hash>) -> BoxFuture<()> + Send + Sync>;
-
-/// The state of the gc loop.
-#[derive(derive_more::Debug)]
-enum GcState {
-    // Gc loop is not yet running. Other protocols can add protect callbacks
-    Initial(#[debug(skip)] Vec<ProtectCb>),
-    // Gc loop is running. No more protect callbacks can be added.
-    Started(#[allow(dead_code)] Option<local_pool::Run<()>>),
-}
-
-impl Default for GcState {
-    fn default() -> Self {
-        Self::Initial(Vec::new())
-    }
-}
-
-#[derive(Debug)]
-enum Rt {
-    Handle(LocalPoolHandle),
-    Owned(LocalPool),
-}
-
-impl Deref for Rt {
-    type Target = LocalPoolHandle;
-
-    fn deref(&self) -> &Self::Target {
-        match self {
-            Self::Handle(ref handle) => handle,
-            Self::Owned(ref pool) => pool.handle(),
-        }
-    }
-}
-
 #[derive(Debug)]
-pub(crate) struct BlobsInner<S> {
-    rt: Rt,
-    pub(crate) store: S,
-    events: EventSender,
-    pub(crate) downloader: Downloader,
+pub(crate) struct BlobsInner {
+    pub(crate) store: Store,
     pub(crate) endpoint: Endpoint,
-    gc_state: std::sync::Mutex<GcState>,
-    #[cfg(feature = "rpc")]
-    pub(crate) batches: tokio::sync::Mutex<BlobBatches>,
-}
-
-impl<S> BlobsInner<S> {
-    pub(crate) fn rt(&self) -> &LocalPoolHandle {
-        &self.rt
-    }
+    pub(crate) events: EventSender,
 }
 
+/// A protocol handler for the blobs protocol.
 #[derive(Debug, Clone)]
-pub struct Blobs<S> {
-    pub(crate) inner: Arc<BlobsInner<S>>,
-    #[cfg(feature = "rpc")]
-    pub(crate) rpc_handler: Arc<std::sync::OnceLock<crate::rpc::RpcHandler>>,
-}
-
-/// Keeps track of all the currently active batch operations of the blobs api.
-#[cfg(feature = "rpc")]
-#[derive(Debug, Default)]
-pub(crate) struct BlobBatches {
-    /// Currently active batches
-    batches: std::collections::BTreeMap<BatchId, BlobBatch>,
-    /// Used to generate new batch ids.
-    max: u64,
-}
-
-/// A single batch of blob operations
-#[cfg(feature = "rpc")]
-#[derive(Debug, Default)]
-struct BlobBatch {
-    /// The tags in this batch.
-    tags: std::collections::BTreeMap<crate::HashAndFormat, Vec<crate::TempTag>>,
-}
-
-#[cfg(feature = "rpc")]
-impl BlobBatches {
-    /// Create a new unique batch id.
-    pub fn create(&mut self) -> BatchId {
-        let id = self.max;
-        self.max += 1;
-        BatchId(id)
-    }
-
-    /// Store a temp tag in a batch identified by a batch id.
-    pub fn store(&mut self, batch: BatchId, tt: crate::TempTag) {
-        let entry = self.batches.entry(batch).or_default();
-        entry.tags.entry(tt.hash_and_format()).or_default().push(tt);
-    }
-
-    /// Remove a tag from a batch.
-    pub fn remove_one(&mut self, batch: BatchId, content: &crate::HashAndFormat) -> Result<()> {
-        if let Some(batch) = self.batches.get_mut(&batch) {
-            if let Some(tags) = batch.tags.get_mut(content) {
-                tags.pop();
-                if tags.is_empty() {
-                    batch.tags.remove(content);
-                }
-                return Ok(());
-            }
-        }
-        // this can happen if we try to upgrade a tag from an expired batch
-        anyhow::bail!("tag not found in batch");
-    }
-
-    /// Remove an entire batch.
-    pub fn remove(&mut self, batch: BatchId) {
-        self.batches.remove(&batch);
-    }
-}
-
-/// Builder for the Blobs protocol handler
-#[derive(Debug)]
-pub struct Builder<S> {
-    store: S,
-    events: Option<EventSender>,
-    downloader_config: Option<crate::downloader::Config>,
-    rt: Option<LocalPoolHandle>,
-}
-
-impl<S: crate::store::Store> Builder<S> {
-    /// Set the event sender for the blobs protocol.
-    pub fn events(mut self, value: EventSender) -> Self {
-        self.events = Some(value);
-        self
-    }
-
-    /// Set a custom [`LocalPoolHandle`] to use.
-    pub fn local_pool(mut self, rt: LocalPoolHandle) -> Self {
-        self.rt = Some(rt);
-        self
-    }
-
-    /// Set custom downloader config
-    pub fn downloader_config(mut self, downloader_config: downloader::Config) -> Self {
-        self.downloader_config = Some(downloader_config);
-        self
-    }
-
-    /// Set custom [`ConcurrencyLimits`] to use.
-    pub fn concurrency_limits(mut self, concurrency_limits: ConcurrencyLimits) -> Self {
-        let downloader_config = self.downloader_config.get_or_insert_with(Default::default);
-        downloader_config.concurrency = concurrency_limits;
-        self
-    }
-
-    /// Set a custom [`RetryConfig`] to use.
-    pub fn retry_config(mut self, retry_config: RetryConfig) -> Self {
-        let downloader_config = self.downloader_config.get_or_insert_with(Default::default);
-        downloader_config.retry = retry_config;
-        self
-    }
-
-    /// Build the Blobs protocol handler.
-    /// You need to provide a the endpoint.
-    pub fn build(self, endpoint: &Endpoint) -> Blobs<S> {
-        let rt = self
-            .rt
-            .map(Rt::Handle)
-            .unwrap_or_else(|| Rt::Owned(LocalPool::default()));
-        let downloader_config = self.downloader_config.unwrap_or_default();
-        let downloader = Downloader::with_config(
-            self.store.clone(),
-            endpoint.clone(),
-            rt.clone(),
-            downloader_config,
-        );
-        Blobs::new(
-            self.store,
-            rt,
-            self.events.unwrap_or_default(),
-            downloader,
-            endpoint.clone(),
-        )
-    }
-}
-
-impl<S> Blobs<S> {
-    /// Create a new Blobs protocol handler builder, given a store.
-    pub fn builder(store: S) -> Builder<S> {
-        Builder {
-            store,
-            events: None,
-            downloader_config: None,
-            rt: None,
-        }
-    }
-}
-
-impl Blobs<crate::store::mem::Store> {
-    /// Create a new memory-backed Blobs protocol handler.
-    pub fn memory() -> Builder<crate::store::mem::Store> {
-        Self::builder(crate::store::mem::Store::new())
-    }
+pub struct Blobs {
+    pub(crate) inner: Arc<BlobsInner>,
 }
 
-impl Blobs<crate::store::fs::Store> {
-    /// Load a persistent Blobs protocol handler from a path.
-    pub async fn persistent(
-        path: impl AsRef<std::path::Path>,
-    ) -> anyhow::Result<Builder<crate::store::fs::Store>> {
-        Ok(Self::builder(crate::store::fs::Store::load(path).await?))
-    }
-}
-
-impl<S: crate::store::Store> Blobs<S> {
-    fn new(
-        store: S,
-        rt: Rt,
-        events: EventSender,
-        downloader: Downloader,
-        endpoint: Endpoint,
-    ) -> Self {
+impl Blobs {
+    pub fn new(store: &Store, endpoint: Endpoint, events: Option<mpsc::Sender<Event>>) -> Self {
         Self {
             inner: Arc::new(BlobsInner {
-                rt,
-                store,
-                events,
-                downloader,
+                store: store.clone(),
                 endpoint,
-                #[cfg(feature = "rpc")]
-                batches: Default::default(),
-                gc_state: Default::default(),
+                events: EventSender::new(events),
             }),
-            #[cfg(feature = "rpc")]
-            rpc_handler: Default::default(),
         }
     }
 
-    pub fn store(&self) -> &S {
+    pub fn store(&self) -> &Store {
         &self.inner.store
     }
 
-    pub fn metrics(&self) -> &Arc<Metrics> {
-        self.downloader().metrics()
-    }
-
-    pub fn events(&self) -> &EventSender {
-        &self.inner.events
-    }
-
-    pub fn rt(&self) -> &LocalPoolHandle {
-        self.inner.rt()
-    }
-
-    pub fn downloader(&self) -> &Downloader {
-        &self.inner.downloader
-    }
-
     pub fn endpoint(&self) -> &Endpoint {
         &self.inner.endpoint
     }
 
-    /// Add a callback that will be called before the garbage collector runs.
+    /// Create a ticket for content on this node.
     ///
-    /// This can only be called before the garbage collector has started, otherwise it will return an error.
-    pub fn add_protected(&self, cb: ProtectCb) -> Result<()> {
-        let mut state = self.inner.gc_state.lock().unwrap();
-        match &mut *state {
-            GcState::Initial(cbs) => {
-                cbs.push(cb);
-            }
-            GcState::Started(_) => {
-                anyhow::bail!("cannot add protected blobs after gc has started");
-            }
-        }
-        Ok(())
-    }
-
-    /// Start garbage collection with the given settings.
-    pub fn start_gc(&self, config: GcConfig) -> Result<()> {
-        let mut state = self.inner.gc_state.lock().unwrap();
-        let protected = match state.deref_mut() {
-            GcState::Initial(items) => std::mem::take(items),
-            GcState::Started(_) => bail!("gc already started"),
-        };
-        let protected = Arc::new(protected);
-        let protected_cb = move || {
-            let protected = protected.clone();
-            async move {
-                let mut set = BTreeSet::new();
-                for cb in protected.iter() {
-                    cb(&mut set).await;
-                }
-                set
-            }
-        };
-        let store = self.store().clone();
-        let run = self
-            .rt()
-            .spawn(move || async move { store.gc_run(config, protected_cb).await });
-        *state = GcState::Started(Some(run));
-        Ok(())
+    /// Note that this does not check whether the content is partially or fully available. It is
+    /// just a convenience method to create a ticket from content and the address of this node.
+    pub async fn ticket(&self, content: impl Into<HashAndFormat>) -> anyhow::Result<BlobTicket> {
+        let content = content.into();
+        let addr = self.inner.endpoint.node_addr().initialized().await?;
+        let ticket = BlobTicket::new(addr, content.hash, content.format);
+        Ok(ticket)
     }
 }
 
-impl<S: crate::store::Store> ProtocolHandler for Blobs<S> {
-    fn accept(&self, conn: Connection) -> BoxedFuture<Result<()>> {
-        let db = self.store().clone();
-        let events = self.events().clone();
-        let rt = self.rt().clone();
+impl ProtocolHandler for Blobs {
+    fn accept(
+        &self,
+        conn: Connection,
+    ) -> impl Future<Output = std::result::Result<(), AcceptError>> + Send {
+        let store = self.store().clone();
+        let events = self.inner.events.clone();
 
         Box::pin(async move {
-            crate::provider::handle_connection(conn, db, events, rt).await;
+            crate::provider::handle_connection(conn, store, events).await;
             Ok(())
         })
     }
 
-    fn shutdown(&self) -> BoxedFuture<()> {
+    fn shutdown(&self) -> impl Future<Output = ()> + Send {
         let store = self.store().clone();
         Box::pin(async move {
-            store.shutdown().await;
+            if let Err(cause) = store.shutdown().await {
+                error!("error shutting down store: {:?}", cause);
+            }
         })
     }
 }
-
-/// A request to the node to download and share the data specified by the hash.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct BlobDownloadRequest {
-    /// This mandatory field contains the hash of the data to download and share.
-    pub hash: Hash,
-    /// If the format is [`BlobFormat::HashSeq`], all children are downloaded and shared as
-    /// well.
-    pub format: BlobFormat,
-    /// This mandatory field specifies the nodes to download the data from.
-    ///
-    /// If set to more than a single node, they will all be tried. If `mode` is set to
-    /// [`DownloadMode::Direct`], they will be tried sequentially until a download succeeds.
-    /// If `mode` is set to [`DownloadMode::Queued`], the nodes may be dialed in parallel,
-    /// if the concurrency limits permit.
-    pub nodes: Vec<NodeAddr>,
-    /// Optional tag to tag the data with.
-    pub tag: SetTagOption,
-    /// Whether to directly start the download or add it to the download queue.
-    pub mode: DownloadMode,
-}
-
-/// Set the mode for whether to directly start the download or add it to the download queue.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum DownloadMode {
-    /// Start the download right away.
-    ///
-    /// No concurrency limits or queuing will be applied. It is up to the user to manage download
-    /// concurrency.
-    Direct,
-    /// Queue the download.
-    ///
-    /// The download queue will be processed in-order, while respecting the downloader concurrency limits.
-    Queued,
-}
-
-/// Newtype for a batch id
-#[derive(Debug, PartialEq, Eq, PartialOrd, Serialize, Deserialize, Ord, Clone, Copy, Hash)]
-pub struct BatchId(pub u64);
diff --git a/src/protocol.rs b/src/protocol.rs
index 5f9d2a1d6..850431996 100644
--- a/src/protocol.rs
+++ b/src/protocol.rs
@@ -1,7 +1,4 @@
-//! Protocol for transferring content-addressed blobs and collections over quic
-//! connections. This can be used either with normal quic connections when using
-//! the [quinn](https://crates.io/crates/quinn) crate or with magicsock connections
-//! when using the [iroh-net](https://crates.io/crates/iroh-net) crate.
+//! Protocol for transferring content-addressed blobs over [`iroh`] p2p QUIC connections.
 //!
 //! # Participants
 //!
@@ -88,33 +85,36 @@
 //! ## Specifying the required data
 //!
 //! A [`GetRequest`] contains a hash and a specification of what data related to
-//! that hash is required. The specification is using a [`RangeSpecSeq`] which
+//! that hash is required. The specification is using a [`ChunkRangesSeq`] which
 //! has a compact representation on the wire but is otherwise identical to a
 //! sequence of sets of ranges.
 //!
-//! In the following, we describe how the [`RangeSpecSeq`] is to be created for
+//! In the following, we describe how the [`GetRequest`] is to be created for
 //! different common scenarios.
 //!
+//! Under the hood, this is using the [`ChunkRangesSeq`] type, but the most
+//! convenient way to create a [`GetRequest`] is to use the builder API.
+//!
 //! Ranges are always given in terms of 1024 byte blake3 chunks, *not* in terms
 //! of bytes or chunk groups. The reason for this is that chunks are the fundamental
-//! unit of hashing in blake3. Addressing anything smaller than a chunk is not
+//! unit of hashing in BLAKE3. Addressing anything smaller than a chunk is not
 //! possible, and combining multiple chunks is merely an optimization to reduce
 //! metadata overhead.
 //!
 //! ### Individual blobs
 //!
 //! In the easiest case, the getter just wants to retrieve a single blob. In this
-//! case, the getter specifies [`RangeSpecSeq`] that contains a single element.
+//! case, the getter specifies [`ChunkRangesSeq`] that contains a single element.
 //! This element is the set of all chunks to indicate that we
 //! want the entire blob, no matter how many chunks it has.
 //!
 //! Since this is a very common case, there is a convenience method
-//! [`GetRequest::single`] that only requires the hash of the blob.
+//! [`GetRequest::blob`] that only requires the hash of the blob.
 //!
 //! ```rust
 //! # use iroh_blobs::protocol::GetRequest;
 //! # let hash: iroh_blobs::Hash = [0; 32].into();
-//! let request = GetRequest::single(hash);
+//! let request = GetRequest::blob(hash);
 //! ```
 //!
 //! ### Ranges of blobs
@@ -122,36 +122,55 @@
 //! In this case, we have a (possibly large) blob and we want to retrieve only
 //! some ranges of chunks. This is useful in similar cases as HTTP range requests.
 //!
-//! We still need just a single element in the [`RangeSpecSeq`], since we are
+//! We still need just a single element in the [`ChunkRangesSeq`], since we are
 //! still only interested in a single blob. However, this element contains all
 //! the chunk ranges we want to retrieve.
 //!
 //! For example, if we want to retrieve chunks 0-10 of a blob, we would
-//! create a [`RangeSpecSeq`] like this:
+//! create a [`ChunkRangesSeq`] like this:
 //!
 //! ```rust
-//! # use bao_tree::{ChunkNum, ChunkRanges};
-//! # use iroh_blobs::protocol::{GetRequest, RangeSpecSeq};
+//! # use iroh_blobs::protocol::{GetRequest, ChunkRanges, ChunkRangesExt};
 //! # let hash: iroh_blobs::Hash = [0; 32].into();
-//! let spec = RangeSpecSeq::from_ranges([ChunkRanges::from(..ChunkNum(10))]);
-//! let request = GetRequest::new(hash, spec);
+//! let request = GetRequest::builder()
+//!     .root(ChunkRanges::chunks(..10))
+//!     .build(hash);
 //! ```
 //!
-//! Here `ChunkNum` is a newtype wrapper around `u64` that is used to indicate
-//! that we are talking about chunk numbers, not bytes.
-//!
 //! While not that common, it is also possible to request multiple ranges of a
 //! single blob. For example, if we want to retrieve chunks `0-10` and `100-110`
-//! of a large file, we would create a [`RangeSpecSeq`] like this:
+//! of a large file, we would create a [`GetRequest`] like this:
+//!
+//! ```rust
+//! # use iroh_blobs::protocol::{GetRequest, ChunkRanges, ChunkRangesExt, ChunkRangesSeq};
+//! # let hash: iroh_blobs::Hash = [0; 32].into();
+//! let request = GetRequest::builder()
+//!     .root(ChunkRanges::chunks(..10) | ChunkRanges::chunks(100..110))
+//!     .build(hash);
+//! ```
+//!
+//! This is all great, but in most cases we are not interested in chunks but
+//! in bytes. The [`ChunkRanges`] type has a constructor that allows providing
+//! byte ranges instead of chunk ranges. These will be rounded up to the
+//! nearest chunk.
 //!
 //! ```rust
-//! # use bao_tree::{ChunkNum, ChunkRanges};
-//! # use iroh_blobs::protocol::{GetRequest, RangeSpecSeq};
+//! # use iroh_blobs::protocol::{GetRequest, ChunkRanges, ChunkRangesExt, ChunkRangesSeq};
 //! # let hash: iroh_blobs::Hash = [0; 32].into();
-//! let ranges =
-//!     &ChunkRanges::from(..ChunkNum(10)) | &ChunkRanges::from(ChunkNum(100)..ChunkNum(110));
-//! let spec = RangeSpecSeq::from_ranges([ranges]);
-//! let request = GetRequest::new(hash, spec);
+//! let request = GetRequest::builder()
+//!     .root(ChunkRanges::bytes(..1000) | ChunkRanges::bytes(10000..11000))
+//!     .build(hash);
+//! ```
+//!
+//! There are also methods to request a single chunk or a single byte offset,
+//! as well as a special constructor for the last chunk of a blob.
+//!
+//! ```rust
+//! # use iroh_blobs::protocol::{GetRequest, ChunkRanges, ChunkRangesExt, ChunkRangesSeq};
+//! # let hash: iroh_blobs::Hash = [0; 32].into();
+//! let request = GetRequest::builder()
+//!     .root(ChunkRanges::offset(1) | ChunkRanges::last_chunk())
+//!     .build(hash);
 //! ```
 //!
 //! To specify chunk ranges, we use the [`ChunkRanges`] type alias.
@@ -165,82 +184,83 @@
 //! [`RangeSet`]: range_collections::range_set::RangeSet
 //! [`RangeSet2`]: range_collections::range_set::RangeSet2
 //!
-//! ### Collections
+//! ### Hash sequences
 //!
-//! In this case the provider has a collection that contains multiple blobs.
-//! We want to retrieve all blobs in the collection.
+//! In this case the provider has a hash sequence that refers multiple blobs.
+//! We want to retrieve all blobs in the hash sequence.
 //!
-//! When used for collections, the first element of a [`RangeSpecSeq`] refers
-//! to the collection itself, and all subsequent elements refer to the blobs
-//! in the collection. When a [`RangeSpecSeq`] specifies ranges for more than
-//! one blob, the provider will interpret this as a request for a collection.
+//! When used for hash sequences, the first element of a [`ChunkRangesSeq`] refers
+//! to the hash seq itself, and all subsequent elements refer to the blobs
+//! in the hash seq. When a [`ChunkRangesSeq`] specifies ranges for more than
+//! one blob, the provider will interpret this as a request for a hash seq.
 //!
 //! One thing to note is that we might not yet know how many blobs are in the
-//! collection. Therefore, it is not possible to download an entire collection
+//! hash sequence. Therefore, it is not possible to download an entire hash seq
 //! by just specifying [`ChunkRanges::all()`] for all children.
 //!
-//! Instead, [`RangeSpecSeq`] allows defining infinite sequences of range sets.
-//! The [`RangeSpecSeq::all()`] method returns a [`RangeSpecSeq`] that, when iterated
+//! Instead, [`ChunkRangesSeq`] allows defining infinite sequences of range sets.
+//! The [`ChunkRangesSeq::all()`] method returns a [`ChunkRangesSeq`] that, when iterated
 //! over, will yield [`ChunkRanges::all()`] forever.
 //!
-//! So specifying a collection would work like this:
+//! So a get request to download a hash sequence blob and all its children
+//! would look like this:
 //!
 //! ```rust
-//! # use bao_tree::{ChunkNum, ChunkRanges};
-//! # use iroh_blobs::protocol::{GetRequest, RangeSpecSeq};
+//! # use iroh_blobs::protocol::{ChunkRanges, ChunkRangesExt, GetRequest};
 //! # let hash: iroh_blobs::Hash = [0; 32].into();
-//! let spec = RangeSpecSeq::all();
-//! let request = GetRequest::new(hash, spec);
+//! let request = GetRequest::builder()
+//!     .root(ChunkRanges::all())
+//!     .build_open(hash); // repeats the last range forever
 //! ```
 //!
-//! Downloading an entire collection is also a very common case, so there is a
+//! Downloading an entire hash seq is also a very common case, so there is a
 //! convenience method [`GetRequest::all`] that only requires the hash of the
-//! collection.
+//! hash sequence blob.
 //!
-//! ### Parts of collections
+//! ```rust
+//! # use iroh_blobs::protocol::{ChunkRanges, ChunkRangesExt, GetRequest};
+//! # let hash: iroh_blobs::Hash = [0; 32].into();
+//! let request = GetRequest::all(hash);
+//! ```
 //!
-//! The most complex common case is when we have retrieved a collection and
+//! ### Parts of hash sequences
+//!
+//! The most complex common case is when we have retrieved a hash seq and
 //! it's children, but were interrupted before we could retrieve all children.
 //!
-//! In this case we need to specify the collection we want to retrieve, but
+//! In this case we need to specify the hash seq we want to retrieve, but
 //! exclude the children and parts of children that we already have.
 //!
-//! For example, if we have a collection with 3 children, and we already have
+//! For example, if we have a hash with 3 children, and we already have
 //! the first child and the first 1000000 chunks of the second child.
 //!
 //! We would create a [`GetRequest`] like this:
 //!
 //! ```rust
-//! # use bao_tree::{ChunkNum, ChunkRanges};
-//! # use iroh_blobs::protocol::{GetRequest, RangeSpecSeq};
+//! # use iroh_blobs::protocol::{GetRequest, ChunkRanges, ChunkRangesExt};
 //! # let hash: iroh_blobs::Hash = [0; 32].into();
-//! let spec = RangeSpecSeq::from_ranges([
-//!   ChunkRanges::empty(), // we don't need the collection itself
-//!   ChunkRanges::empty(), // we don't need the first child either
-//!   ChunkRanges::from(ChunkNum(1000000)..), // we need the second child from chunk 1000000 onwards
-//!   ChunkRanges::all(), // we need the third child completely
-//! ]);
-//! let request = GetRequest::new(hash, spec);
+//! let request = GetRequest::builder()
+//!     .child(1, ChunkRanges::chunks(1000000..)) // we don't need the first child;
+//!     .next(ChunkRanges::all()) // we need the second child and all subsequent children completely
+//!     .build_open(hash);
 //! ```
 //!
 //! ### Requesting chunks for each child
 //!
-//! The RangeSpecSeq allows some scenarios that are not covered above. E.g. you
-//! might want to request a collection and the first chunk of each child blob to
+//! The ChunkRangesSeq allows some scenarios that are not covered above. E.g. you
+//! might want to request a hash seq and the first chunk of each child blob to
 //! do something like mime type detection.
 //!
 //! You do not know how many children the collection has, so you need to use
 //! an infinite sequence.
 //!
 //! ```rust
-//! # use bao_tree::{ChunkNum, ChunkRanges};
-//! # use iroh_blobs::protocol::{GetRequest, RangeSpecSeq};
+//! # use iroh_blobs::protocol::{GetRequest, ChunkRanges, ChunkRangesExt, ChunkRangesSeq};
 //! # let hash: iroh_blobs::Hash = [0; 32].into();
-//! let spec = RangeSpecSeq::from_ranges_infinite([
-//!     ChunkRanges::all(),               // the collection itself
-//!     ChunkRanges::from(..ChunkNum(1)), // the first chunk of each child
-//! ]);
-//! let request = GetRequest::new(hash, spec);
+//! let request = GetRequest::builder()
+//!     .root(ChunkRanges::all())
+//!     .next(ChunkRanges::chunk(1)) // the first chunk of each child)
+//!     .build_open(hash);
 //! ```
 //!
 //! ### Requesting a single child
@@ -249,45 +269,40 @@
 //! the following would download the second child of a collection:
 //!
 //! ```rust
-//! # use bao_tree::{ChunkNum, ChunkRanges};
-//! # use iroh_blobs::protocol::{GetRequest, RangeSpecSeq};
+//! # use iroh_blobs::protocol::{GetRequest, ChunkRanges, ChunkRangesExt};
 //! # let hash: iroh_blobs::Hash = [0; 32].into();
-//! let spec = RangeSpecSeq::from_ranges([
-//!     ChunkRanges::empty(), // we don't need the collection itself
-//!     ChunkRanges::empty(), // we don't need the first child either
-//!     ChunkRanges::all(),   // we need the second child completely
-//! ]);
-//! let request = GetRequest::new(hash, spec);
+//! let request = GetRequest::builder()
+//!     .child(1, ChunkRanges::all()) // we need the second child completely
+//!     .build(hash);
 //! ```
 //!
 //! However, if you already have the collection, you might as well locally
 //! look up the hash of the child and request it directly.
 //!
 //! ```rust
-//! # use bao_tree::{ChunkNum, ChunkRanges};
-//! # use iroh_blobs::protocol::{GetRequest, RangeSpecSeq};
+//! # use iroh_blobs::protocol::{GetRequest, ChunkRanges, ChunkRangesSeq};
 //! # let child_hash: iroh_blobs::Hash = [0; 32].into();
-//! let request = GetRequest::single(child_hash);
+//! let request = GetRequest::blob(child_hash);
 //! ```
 //!
-//! ### Why RangeSpec and RangeSpecSeq?
+//! ### Why ChunkRanges and ChunkRangesSeq?
 //!
-//! You might wonder why we have [`RangeSpec`] and [`RangeSpecSeq`], when a simple
+//! You might wonder why we have [`ChunkRangesSeq`], when a simple
 //! sequence of [`ChunkRanges`] might also do.
 //!
-//! The [`RangeSpec`] and [`RangeSpecSeq`] types exist to provide an efficient
-//! representation of the request on the wire. In the [`RangeSpec`] type,
-//! sequences of ranges are encoded alternating intervals of selected and
-//! non-selected chunks. This results in smaller numbers that will result in fewer bytes
-//! on the wire when using the [postcard](https://crates.io/crates/postcard) encoding
-//! format that uses variable length integers.
+//! The [`ChunkRangesSeq`] type exist to provide an efficient
+//! representation of the request on the wire. In the wire encoding of [`ChunkRangesSeq`],
+//! [`ChunkRanges`] are encoded alternating intervals of selected and non-selected chunks.
+//! This results in smaller numbers that will result in fewer bytes on the wire when using
+//! the [postcard](https://crates.io/crates/postcard) encoding format that uses variable
+//! length integers.
 //!
-//! Likewise, the [`RangeSpecSeq`] type is a sequence of [`RangeSpec`]s that
+//! Likewise, the [`ChunkRangesSeq`] type
 //! does run length encoding to remove repeating elements. It also allows infinite
-//! sequences of [`RangeSpec`]s to be encoded, unlike a simple sequence of
+//! sequences of [`ChunkRanges`] to be encoded, unlike a simple sequence of
 //! [`ChunkRanges`]s.
 //!
-//! [`RangeSpecSeq`] should be efficient even in case of very fragmented availability
+//! [`ChunkRangesSeq`] should be efficient even in case of very fragmented availability
 //! of chunks, like a download from multiple providers that was frequently interrupted.
 //!
 //! # Responses
@@ -295,18 +310,13 @@
 //! The response stream contains the bao encoded bytes for the requested data.
 //! The data will be sent in the order in which it was requested, so ascending
 //! chunks for each blob, and blobs in the order in which they appear in the
-//! collection.
+//! hash seq.
 //!
 //! For details on the bao encoding, see the [bao specification](https://github.com/oconnor663/bao/blob/master/docs/spec.md)
 //! and the [bao-tree](https://crates.io/crates/bao-tree) crate. The bao-tree crate
-//! is identical to the bao crate, except that it allows combining multiple blake3
+//! is identical to the bao crate, except that it allows combining multiple BLAKE3
 //! chunks to chunk groups for efficiency.
 //!
-//! As a consequence of the chunk group optimization, chunk ranges in the response
-//! will be rounded up to chunk groups ranges, so e.g. if you ask for chunks 0..10,
-//! you will get chunks 0-16. This is done to reduce metadata overhead, and might
-//! change in the future.
-//!
 //! For a complete response, the chunks are guaranteed to completely cover the
 //! requested ranges.
 //!
@@ -323,33 +333,67 @@
 //!
 //! # Requesting multiple unrelated blobs
 //!
-//! Currently, the protocol does not support requesting multiple unrelated blobs
-//! in a single request. As an alternative, you can create a collection
-//! on the provider side and use that to efficiently retrieve the blobs.
+//! Let's say you don't have a hash sequence on the provider side, but you
+//! nevertheless want to request multiple unrelated blobs in a single request.
 //!
-//! If that is not possible, you can create a custom request handler that
-//! accepts a custom request struct that contains the hashes of the blobs.
+//! For this, there is the [`GetManyRequest`] type, which also comes with a
+//! builder API.
 //!
-//! If neither of these options are possible, you have no choice but to do
-//! multiple requests. However, note that multiple requests will be multiplexed
-//! over a single connection, and the overhead of a new QUIC stream on an existing
-//! connection is very low.
+//! ```rust
+//! # use iroh_blobs::protocol::{GetManyRequest, ChunkRanges, ChunkRangesExt};
+//! # let hash1: iroh_blobs::Hash = [0; 32].into();
+//! # let hash2: iroh_blobs::Hash = [1; 32].into();
+//! GetManyRequest::builder()
+//!     .hash(hash1, ChunkRanges::all())
+//!     .hash(hash2, ChunkRanges::all())
+//!     .build();
+//! ```
+//! If you accidentally or intentionally request ranges for the same hash
+//! multiple times, they will be merged into a single [`ChunkRanges`].
 //!
-//! In case nodes are permanently exchanging data, it is probably valuable to
-//! keep a connection open and reuse it for multiple requests.
-use bao_tree::{ChunkNum, ChunkRanges};
+//! ```rust
+//! # use iroh_blobs::protocol::{GetManyRequest, ChunkRanges, ChunkRangesExt};
+//! # let hash1: iroh_blobs::Hash = [0; 32].into();
+//! # let hash2: iroh_blobs::Hash = [1; 32].into();
+//! GetManyRequest::builder()
+//!     .hash(hash1, ChunkRanges::chunk(1))
+//!     .hash(hash2, ChunkRanges::all())
+//!     .hash(hash1, ChunkRanges::last_chunk())
+//!     .build();
+//! ```
+//!
+//! This is mostly useful for requesting multiple tiny blobs in a single request.
+//! For large or even medium sized blobs, multiple requests are not expensive.
+//! Multiple requests just create multiple streams on the same connection,
+//! which is *very* cheap in QUIC.
+//!
+//! In case nodes are permanently exchanging data, it is somewhat valuable to
+//! keep a connection open and reuse it for multiple requests. However, creating
+//! a new connection is also very cheap, so you would only do this to optimize
+//! a large existing system that has demonstrated performance issues.
+//!
+//! If in doubt, just use multiple requests and multiple connections.
+use std::io;
+
+use builder::GetRequestBuilder;
 use derive_more::From;
 use iroh::endpoint::VarInt;
+use irpc::util::AsyncReadVarintExt;
+use postcard::experimental::max_size::MaxSize;
 use serde::{Deserialize, Serialize};
 mod range_spec;
-pub use range_spec::{NonEmptyRequestRangeSpecIter, RangeSpec, RangeSpecSeq};
+pub use bao_tree::ChunkRanges;
+pub use range_spec::{ChunkRangesSeq, NonEmptyRequestRangeSpecIter, RangeSpec};
+use snafu::{GenerateImplicitData, Snafu};
+use tokio::io::AsyncReadExt;
 
-use crate::Hash;
+pub use crate::util::ChunkRangesExt;
+use crate::{api::blobs::Bitfield, provider::CountingReader, BlobFormat, Hash, HashAndFormat};
 
 /// Maximum message size is limited to 100MiB for now.
-pub const MAX_MESSAGE_SIZE: usize = 1024 * 1024 * 100;
+pub const MAX_MESSAGE_SIZE: usize = 1024 * 1024;
 
-/// The ALPN used with quic for the iroh bytes protocol.
+/// The ALPN used with quic for the iroh blobs protocol.
 pub const ALPN: &[u8] = b"/iroh-bytes/4";
 
 #[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone, From)]
@@ -357,65 +401,242 @@ pub const ALPN: &[u8] = b"/iroh-bytes/4";
 pub enum Request {
     /// A get request for a blob or collection
     Get(GetRequest),
+    Observe(ObserveRequest),
+    Slot2,
+    Slot3,
+    Slot4,
+    Slot5,
+    Slot6,
+    Slot7,
+    /// The inverse of a get request - push data to the provider
+    ///
+    /// Note that providers will in many cases reject this request, e.g. if
+    /// they don't have write access to the store or don't want to ingest
+    /// unknown data.
+    Push(PushRequest),
+    /// Get multiple blobs in a single request, from a single provider
+    ///
+    /// This is identical to a [`GetRequest`] for a [`crate::hashseq::HashSeq`], but the provider
+    /// does not need to have the hash seq.
+    GetMany(GetManyRequest),
 }
 
-/// A request
-#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone)]
+/// This must contain the request types in the same order as the full requests
+#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone, Copy, MaxSize)]
+pub enum RequestType {
+    Get,
+    Observe,
+    Slot2,
+    Slot3,
+    Slot4,
+    Slot5,
+    Slot6,
+    Slot7,
+    Push,
+    GetMany,
+}
+
+impl Request {
+    pub async fn read_async(
+        reader: &mut CountingReader<&mut iroh::endpoint::RecvStream>,
+    ) -> io::Result<Self> {
+        let request_type = reader.read_u8().await?;
+        let request_type: RequestType = postcard::from_bytes(std::slice::from_ref(&request_type))
+            .map_err(|_| {
+            io::Error::new(
+                io::ErrorKind::InvalidData,
+                "failed to deserialize request type",
+            )
+        })?;
+        Ok(match request_type {
+            RequestType::Get => reader
+                .read_to_end_as::<GetRequest>(MAX_MESSAGE_SIZE)
+                .await?
+                .into(),
+            RequestType::GetMany => reader
+                .read_to_end_as::<GetManyRequest>(MAX_MESSAGE_SIZE)
+                .await?
+                .into(),
+            RequestType::Observe => reader
+                .read_to_end_as::<ObserveRequest>(MAX_MESSAGE_SIZE)
+                .await?
+                .into(),
+            RequestType::Push => reader
+                .read_length_prefixed::<PushRequest>(MAX_MESSAGE_SIZE)
+                .await?
+                .into(),
+            _ => {
+                return Err(io::Error::new(
+                    io::ErrorKind::InvalidData,
+                    "failed to deserialize request type",
+                ));
+            }
+        })
+    }
+}
+
+/// A get request
+#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone, Hash)]
 pub struct GetRequest {
     /// blake3 hash
     pub hash: Hash,
     /// The range of data to request
     ///
     /// The first element is the parent, all subsequent elements are children.
-    pub ranges: RangeSpecSeq,
+    pub ranges: ChunkRangesSeq,
+}
+
+impl From<HashAndFormat> for GetRequest {
+    fn from(value: HashAndFormat) -> Self {
+        match value.format {
+            BlobFormat::Raw => Self::blob(value.hash),
+            BlobFormat::HashSeq => Self::all(value.hash),
+        }
+    }
 }
 
 impl GetRequest {
+    pub fn builder() -> GetRequestBuilder {
+        GetRequestBuilder::default()
+    }
+
+    pub fn content(&self) -> HashAndFormat {
+        HashAndFormat {
+            hash: self.hash,
+            format: if self.ranges.is_blob() {
+                BlobFormat::Raw
+            } else {
+                BlobFormat::HashSeq
+            },
+        }
+    }
+
     /// Request a blob or collection with specified ranges
-    pub fn new(hash: Hash, ranges: RangeSpecSeq) -> Self {
+    pub fn new(hash: Hash, ranges: ChunkRangesSeq) -> Self {
         Self { hash, ranges }
     }
 
     /// Request a collection and all its children
-    pub fn all(hash: Hash) -> Self {
+    pub fn all(hash: impl Into<Hash>) -> Self {
         Self {
-            hash,
-            ranges: RangeSpecSeq::all(),
+            hash: hash.into(),
+            ranges: ChunkRangesSeq::all(),
         }
     }
 
     /// Request just a single blob
-    pub fn single(hash: Hash) -> Self {
+    pub fn blob(hash: impl Into<Hash>) -> Self {
         Self {
-            hash,
-            ranges: RangeSpecSeq::from_ranges([ChunkRanges::all()]),
+            hash: hash.into(),
+            ranges: ChunkRangesSeq::from_ranges([ChunkRanges::all()]),
         }
     }
 
-    /// Request the last chunk of a single blob
-    ///
-    /// This can be used to get the verified size of a blob.
-    pub fn last_chunk(hash: Hash) -> Self {
+    /// Request ranges from a single blob
+    pub fn blob_ranges(hash: Hash, ranges: ChunkRanges) -> Self {
         Self {
             hash,
-            ranges: RangeSpecSeq::from_ranges([ChunkRanges::from(ChunkNum(u64::MAX)..)]),
+            ranges: ChunkRangesSeq::from_ranges([ranges]),
         }
     }
+}
+
+/// A push request contains a description of what to push, but will be followed
+/// by the data to push.
+#[derive(
+    Deserialize, Serialize, Debug, PartialEq, Eq, Clone, derive_more::From, derive_more::Deref,
+)]
+pub struct PushRequest(GetRequest);
+
+impl PushRequest {
+    pub fn new(hash: Hash, ranges: ChunkRangesSeq) -> Self {
+        Self(GetRequest::new(hash, ranges))
+    }
+}
 
-    /// Request the last chunk for all children
+/// A GetMany request is a request to get multiple blobs via a single request.
+///
+/// It is identical to a [`GetRequest`] for a HashSeq, but the HashSeq is provided
+/// by the requester.
+#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone)]
+pub struct GetManyRequest {
+    /// The hashes of the blobs to get
+    pub hashes: Vec<Hash>,
+    /// The ranges of data to request
     ///
-    /// This can be used to get the verified size of all children.
-    pub fn last_chunks(hash: Hash) -> Self {
+    /// There is no range request for the parent, since we just sent the hashes
+    /// and therefore have the parent already.
+    pub ranges: ChunkRangesSeq,
+}
+
+impl<I: Into<Hash>> FromIterator<I> for GetManyRequest {
+    fn from_iter<T: IntoIterator<Item = I>>(iter: T) -> Self {
+        let mut res = iter.into_iter().map(Into::into).collect::<Vec<Hash>>();
+        res.sort();
+        res.dedup();
+        let n = res.len() as u64;
         Self {
-            hash,
-            ranges: RangeSpecSeq::from_ranges_infinite([
-                ChunkRanges::all(),
-                ChunkRanges::from(ChunkNum(u64::MAX)..),
+            hashes: res,
+            ranges: ChunkRangesSeq(smallvec::smallvec![
+                (0, ChunkRanges::all()),
+                (n, ChunkRanges::empty())
             ]),
         }
     }
 }
 
+impl GetManyRequest {
+    pub fn new(hashes: Vec<Hash>, ranges: ChunkRangesSeq) -> Self {
+        Self { hashes, ranges }
+    }
+
+    pub fn builder() -> builder::GetManyRequestBuilder {
+        builder::GetManyRequestBuilder::default()
+    }
+}
+
+/// A request to observe a raw blob bitfield.
+#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone, Hash)]
+pub struct ObserveRequest {
+    /// blake3 hash
+    pub hash: Hash,
+    /// ranges to observe.
+    pub ranges: RangeSpec,
+}
+
+impl ObserveRequest {
+    pub fn new(hash: Hash) -> Self {
+        Self {
+            hash,
+            ranges: RangeSpec::all(),
+        }
+    }
+}
+
+#[derive(Deserialize, Serialize, Debug, PartialEq, Eq)]
+pub struct ObserveItem {
+    pub size: u64,
+    pub ranges: ChunkRanges,
+}
+
+impl From<&Bitfield> for ObserveItem {
+    fn from(value: &Bitfield) -> Self {
+        Self {
+            size: value.size,
+            ranges: value.ranges.clone(),
+        }
+    }
+}
+
+impl From<&ObserveItem> for Bitfield {
+    fn from(value: &ObserveItem) -> Self {
+        Self {
+            size: value.size,
+            ranges: value.ranges.clone(),
+        }
+    }
+}
+
 /// Reasons to close connections or stop streams.
 ///
 /// A QUIC **connection** can be *closed* and a **stream** can request the other side to
@@ -464,9 +685,21 @@ impl From<Closed> for VarInt {
 }
 
 /// Unknown error_code, can not be converted into [`Closed`].
-#[derive(thiserror::Error, Debug)]
-#[error("Unknown error_code: {0}")]
-pub struct UnknownErrorCode(u64);
+#[derive(Debug, Snafu)]
+#[snafu(display("Unknown error_code: {code}"))]
+pub struct UnknownErrorCode {
+    code: u64,
+    backtrace: Option<snafu::Backtrace>,
+}
+
+impl UnknownErrorCode {
+    pub(crate) fn new(code: u64) -> Self {
+        Self {
+            code,
+            backtrace: GenerateImplicitData::generate(),
+        }
+    }
+}
 
 impl TryFrom<VarInt> for Closed {
     type Error = UnknownErrorCode;
@@ -476,26 +709,269 @@ impl TryFrom<VarInt> for Closed {
             0 => Ok(Self::StreamDropped),
             1 => Ok(Self::ProviderTerminating),
             2 => Ok(Self::RequestReceived),
-            val => Err(UnknownErrorCode(val)),
+            val => Err(UnknownErrorCode::new(val)),
+        }
+    }
+}
+
+pub mod builder {
+    use std::collections::BTreeMap;
+
+    use bao_tree::ChunkRanges;
+
+    use super::ChunkRangesSeq;
+    use crate::{
+        protocol::{GetManyRequest, GetRequest},
+        Hash,
+    };
+
+    #[derive(Debug, Clone, Default)]
+    pub struct ChunkRangesSeqBuilder {
+        ranges: BTreeMap<u64, ChunkRanges>,
+    }
+
+    #[derive(Debug, Clone, Default)]
+    pub struct GetRequestBuilder {
+        builder: ChunkRangesSeqBuilder,
+    }
+
+    impl GetRequestBuilder {
+        /// Add a range to the request.
+        pub fn offset(mut self, offset: u64, ranges: impl Into<ChunkRanges>) -> Self {
+            self.builder = self.builder.offset(offset, ranges);
+            self
+        }
+
+        /// Add a range to the request.
+        pub fn child(mut self, child: u64, ranges: impl Into<ChunkRanges>) -> Self {
+            self.builder = self.builder.offset(child + 1, ranges);
+            self
+        }
+
+        /// Specify ranges for the root blob (the HashSeq)
+        pub fn root(mut self, ranges: impl Into<ChunkRanges>) -> Self {
+            self.builder = self.builder.offset(0, ranges);
+            self
+        }
+
+        /// Specify ranges for the next offset.
+        pub fn next(mut self, ranges: impl Into<ChunkRanges>) -> Self {
+            self.builder = self.builder.next(ranges);
+            self
+        }
+
+        /// Build a get request for the given hash, with the ranges specified in the builder.
+        pub fn build(self, hash: impl Into<Hash>) -> GetRequest {
+            let ranges = self.builder.build();
+            GetRequest::new(hash.into(), ranges)
+        }
+
+        /// Build a get request for the given hash, with the ranges specified in the builder
+        /// and the last non-empty range repeating indefinitely.
+        pub fn build_open(self, hash: impl Into<Hash>) -> GetRequest {
+            let ranges = self.builder.build_open();
+            GetRequest::new(hash.into(), ranges)
+        }
+    }
+
+    impl ChunkRangesSeqBuilder {
+        /// Add a range to the request.
+        pub fn offset(self, offset: u64, ranges: impl Into<ChunkRanges>) -> Self {
+            self.at_offset(offset, ranges.into())
+        }
+
+        /// Specify ranges for the next offset.
+        pub fn next(self, ranges: impl Into<ChunkRanges>) -> Self {
+            let offset = self.next_offset_value();
+            self.at_offset(offset, ranges.into())
+        }
+
+        /// Build a get request for the given hash, with the ranges specified in the builder.
+        pub fn build(self) -> ChunkRangesSeq {
+            ChunkRangesSeq::from_ranges(self.build0())
+        }
+
+        /// Build a get request for the given hash, with the ranges specified in the builder
+        /// and the last non-empty range repeating indefinitely.
+        pub fn build_open(self) -> ChunkRangesSeq {
+            ChunkRangesSeq::from_ranges_infinite(self.build0())
+        }
+
+        /// Add ranges at the given offset.
+        fn at_offset(mut self, offset: u64, ranges: ChunkRanges) -> Self {
+            self.ranges
+                .entry(offset)
+                .and_modify(|v| *v |= ranges.clone())
+                .or_insert(ranges);
+            self
+        }
+
+        /// Build the request.
+        fn build0(mut self) -> impl Iterator<Item = ChunkRanges> {
+            let mut ranges = Vec::new();
+            self.ranges.retain(|_, v| !v.is_empty());
+            let until_key = self.next_offset_value();
+            for offset in 0..until_key {
+                ranges.push(self.ranges.remove(&offset).unwrap_or_default());
+            }
+            ranges.into_iter()
+        }
+
+        /// Get the next offset value.
+        fn next_offset_value(&self) -> u64 {
+            self.ranges
+                .last_key_value()
+                .map(|(k, _)| *k + 1)
+                .unwrap_or_default()
+        }
+    }
+
+    #[derive(Debug, Clone, Default)]
+    pub struct GetManyRequestBuilder {
+        ranges: BTreeMap<Hash, ChunkRanges>,
+    }
+
+    impl GetManyRequestBuilder {
+        /// Specify ranges for the given hash.
+        ///
+        /// Note that if you specify a hash that is already in the request, the ranges will be
+        /// merged with the existing ranges.
+        pub fn hash(mut self, hash: impl Into<Hash>, ranges: impl Into<ChunkRanges>) -> Self {
+            let ranges = ranges.into();
+            let hash = hash.into();
+            self.ranges
+                .entry(hash)
+                .and_modify(|v| *v |= ranges.clone())
+                .or_insert(ranges);
+            self
+        }
+
+        /// Build a `GetManyRequest`.
+        pub fn build(self) -> GetManyRequest {
+            let (hashes, ranges): (Vec<Hash>, Vec<ChunkRanges>) = self
+                .ranges
+                .into_iter()
+                .filter(|(_, v)| !v.is_empty())
+                .unzip();
+            let ranges = ChunkRangesSeq::from_ranges(ranges);
+            GetManyRequest { hashes, ranges }
+        }
+    }
+
+    #[cfg(test)]
+    mod tests {
+        use bao_tree::ChunkNum;
+
+        use super::*;
+        use crate::{protocol::GetManyRequest, util::ChunkRangesExt};
+
+        #[test]
+        fn chunk_ranges_ext() {
+            let ranges = ChunkRanges::bytes(1..2)
+                | ChunkRanges::chunks(100..=200)
+                | ChunkRanges::offset(1024 * 10)
+                | ChunkRanges::chunk(1024)
+                | ChunkRanges::last_chunk();
+            assert_eq!(
+                ranges,
+                ChunkRanges::from(ChunkNum(0)..ChunkNum(1)) // byte range 1..2
+                    | ChunkRanges::from(ChunkNum(10)..ChunkNum(11)) // chunk at byte offset 1024*10
+                    | ChunkRanges::from(ChunkNum(100)..ChunkNum(201)) // chunk range 100..=200
+                    | ChunkRanges::from(ChunkNum(1024)..ChunkNum(1025)) // chunk 1024
+                    | ChunkRanges::last_chunk() // last chunk
+            );
+        }
+
+        #[test]
+        fn get_request_builder() {
+            let hash = [0; 32];
+            let request = GetRequest::builder()
+                .root(ChunkRanges::all())
+                .next(ChunkRanges::all())
+                .next(ChunkRanges::bytes(0..100))
+                .build(hash);
+            assert_eq!(request.hash.as_bytes(), &hash);
+            assert_eq!(
+                request.ranges,
+                ChunkRangesSeq::from_ranges([
+                    ChunkRanges::all(),
+                    ChunkRanges::all(),
+                    ChunkRanges::from(..ChunkNum(1)),
+                ])
+            );
+
+            let request = GetRequest::builder()
+                .root(ChunkRanges::all())
+                .child(2, ChunkRanges::bytes(0..100))
+                .build(hash);
+            assert_eq!(request.hash.as_bytes(), &hash);
+            assert_eq!(
+                request.ranges,
+                ChunkRangesSeq::from_ranges([
+                    ChunkRanges::all(),               // root
+                    ChunkRanges::empty(),             // child 0
+                    ChunkRanges::empty(),             // child 1
+                    ChunkRanges::from(..ChunkNum(1))  // child 2,
+                ])
+            );
+
+            let request = GetRequest::builder()
+                .root(ChunkRanges::all())
+                .next(ChunkRanges::bytes(0..1024) | ChunkRanges::last_chunk())
+                .build_open(hash);
+            assert_eq!(request.hash.as_bytes(), &[0; 32]);
+            assert_eq!(
+                request.ranges,
+                ChunkRangesSeq::from_ranges_infinite([
+                    ChunkRanges::all(),
+                    ChunkRanges::from(..ChunkNum(1)) | ChunkRanges::last_chunk(),
+                ])
+            );
+        }
+
+        #[test]
+        fn get_many_request_builder() {
+            let hash1 = [0; 32];
+            let hash2 = [1; 32];
+            let hash3 = [2; 32];
+            let request = GetManyRequest::builder()
+                .hash(hash1, ChunkRanges::all())
+                .hash(hash2, ChunkRanges::empty()) // will be ignored!
+                .hash(hash3, ChunkRanges::bytes(0..100))
+                .build();
+            assert_eq!(
+                request.hashes,
+                vec![Hash::from([0; 32]), Hash::from([2; 32])]
+            );
+            assert_eq!(
+                request.ranges,
+                ChunkRangesSeq::from_ranges([
+                    ChunkRanges::all(),               // hash 0
+                    ChunkRanges::from(..ChunkNum(1)), // hash 2
+                ])
+            );
         }
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use super::{GetRequest, Request};
-    use crate::{assert_eq_hex, util::hexdump::parse_hexdump};
+    use iroh_test::{assert_eq_hex, hexdump::parse_hexdump};
+    use postcard::experimental::max_size::MaxSize;
+
+    use super::{GetRequest, Request, RequestType};
+    use crate::Hash;
 
     #[test]
     fn request_wire_format() {
-        let hash = [0xda; 32].into();
+        let hash: Hash = [0xda; 32].into();
         let cases = [
             (
-                Request::from(GetRequest::single(hash)),
+                Request::from(GetRequest::blob(hash)),
                 r"
                     00 # enum variant for GetRequest
                     dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada # the hash
-                    020001000100 # the RangeSpecSeq
+                    020001000100 # the ChunkRangesSeq
             ",
             ),
             (
@@ -503,7 +979,7 @@ mod tests {
                 r"
                     00 # enum variant for GetRequest
                     dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada # the hash
-                    01000100 # the RangeSpecSeq
+                    01000100 # the ChunkRangesSeq
             ",
             ),
         ];
@@ -513,4 +989,9 @@ mod tests {
             assert_eq_hex!(bytes, expected);
         }
     }
+
+    #[test]
+    fn request_type_size() {
+        assert_eq!(RequestType::POSTCARD_MAX_SIZE, 1);
+    }
 }
diff --git a/src/protocol/range_spec.rs b/src/protocol/range_spec.rs
index db3390a29..c60414dea 100644
--- a/src/protocol/range_spec.rs
+++ b/src/protocol/range_spec.rs
@@ -1,18 +1,347 @@
 //! Specifications for ranges selection in blobs and sequences of blobs.
 //!
-//! The [`RangeSpec`] allows specifying which BAO chunks inside a single blob should be
+//! The [`ChunkRanges`] allows specifying which BAO chunks inside a single blob should be
 //! selected.
 //!
-//! The [`RangeSpecSeq`] builds on top of this to select blob chunks in an entire
+//! The [`ChunkRangesSeq`] builds on top of this to select blob chunks in an entire
 //! collection.
-use std::fmt;
+use std::{fmt, sync::OnceLock};
 
 use bao_tree::{ChunkNum, ChunkRanges, ChunkRangesRef};
 use serde::{Deserialize, Serialize};
 use smallvec::{smallvec, SmallVec};
 
+static CHUNK_RANGES_EMPTY: OnceLock<ChunkRanges> = OnceLock::new();
+
+fn chunk_ranges_empty() -> &'static ChunkRanges {
+    CHUNK_RANGES_EMPTY.get_or_init(ChunkRanges::empty)
+}
+
+use crate::util::ChunkRangesExt;
+
+#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
+#[serde(from = "wire::RangeSpecSeq", into = "wire::RangeSpecSeq")]
+pub struct ChunkRangesSeq(pub(crate) SmallVec<[(u64, ChunkRanges); 2]>);
+
+impl std::hash::Hash for ChunkRangesSeq {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        for (i, r) in &self.0 {
+            i.hash(state);
+            r.boundaries().hash(state);
+        }
+    }
+}
+
+impl std::ops::Index<u64> for ChunkRangesSeq {
+    type Output = ChunkRanges;
+
+    fn index(&self, index: u64) -> &Self::Output {
+        match self.0.binary_search_by(|(o, _)| o.cmp(&index)) {
+            Ok(i) => &self.0[i].1,
+            Err(i) => {
+                if i == 0 {
+                    chunk_ranges_empty()
+                } else {
+                    &self.0[i - 1].1
+                }
+            }
+        }
+    }
+}
+
+impl ChunkRangesSeq {
+    pub const fn empty() -> Self {
+        Self(SmallVec::new_const())
+    }
+
+    /// Request just the first blob.
+    pub fn root() -> Self {
+        let mut inner = SmallVec::new();
+        inner.push((0, ChunkRanges::all()));
+        inner.push((1, ChunkRanges::empty()));
+        Self(inner)
+    }
+
+    /// A [`ChunkRangesSeq`] containing all chunks from all blobs.
+    ///
+    /// [`ChunkRangesSeq::iter`], will return a full range forever.
+    pub fn all() -> Self {
+        let mut inner = SmallVec::new();
+        inner.push((0, ChunkRanges::all()));
+        Self(inner)
+    }
+
+    /// A [`ChunkRangesSeq`] getting the verified size for the first blob.
+    pub fn verified_size() -> Self {
+        let mut inner = SmallVec::new();
+        inner.push((0, ChunkRanges::last_chunk()));
+        inner.push((1, ChunkRanges::empty()));
+        Self(inner)
+    }
+
+    /// A [`ChunkRangesSeq`] getting the entire first blob and verified sizes for all others.
+    pub fn verified_child_sizes() -> Self {
+        let mut inner = SmallVec::new();
+        inner.push((0, ChunkRanges::all()));
+        inner.push((1, ChunkRanges::last_chunk()));
+        Self(inner)
+    }
+
+    /// Checks if this [`ChunkRangesSeq`] does not select any chunks in the blob.
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+
+    /// Checks if this [`ChunkRangesSeq`] selects all chunks in the blob.
+    pub fn is_all(&self) -> bool {
+        if self.0.len() != 1 {
+            return false;
+        }
+        let Some((_, ranges)) = self.0.iter().next() else {
+            return false;
+        };
+        ranges.is_all()
+    }
+
+    /// If this range seq describes a range for a single item, returns the offset
+    /// and range spec for that item
+    pub fn as_single(&self) -> Option<(u64, &ChunkRanges)> {
+        // we got two elements,
+        // the first element starts at offset 0,
+        // and the second element is empty
+        if self.0.len() != 2 {
+            return None;
+        }
+        let (o1, v1) = self.0.iter().next().unwrap();
+        let (o2, v2) = self.0.iter().next_back().unwrap();
+        if *o1 == (o2 - 1) && v2.is_empty() {
+            Some((*o1, v1))
+        } else {
+            None
+        }
+    }
+
+    pub fn is_blob(&self) -> bool {
+        #[allow(clippy::match_like_matches_macro)]
+        match self.as_single() {
+            Some((0, _)) => true,
+            _ => false,
+        }
+    }
+
+    /// Convenience function to create a [`ChunkRangesSeq`] from an iterator of
+    /// chunk ranges. If the last element is non-empty, it will be repeated
+    /// forever.
+    pub fn from_ranges_infinite(ranges: impl IntoIterator<Item = ChunkRanges>) -> Self {
+        let (ranges, _) = from_ranges_inner(ranges);
+        Self(ranges)
+    }
+
+    /// Convenience function to create a [`ChunkRangesSeq`] from an iterator of
+    /// chunk ranges. If the last element is non-empty, an empty range will be
+    /// added immediately after it to terminate the sequence.
+    pub fn from_ranges(ranges: impl IntoIterator<Item = ChunkRanges>) -> Self {
+        let (mut res, next) = from_ranges_inner(ranges);
+        if let Some((_, r)) = res.iter().next_back() {
+            if !r.is_empty() {
+                res.push((next, ChunkRanges::empty()));
+            }
+        }
+        Self(res)
+    }
+
+    /// An iterator over blobs in the sequence with a non-empty range spec.
+    ///
+    /// This iterator will only yield items for blobs which have at least one chunk
+    /// selected.
+    ///
+    /// This iterator is infinite if the [`ChunkRangesSeq`] ends on a non-empty [`ChunkRanges`],
+    /// that is all further blobs have selected chunks spans.
+    pub fn iter_non_empty_infinite(&self) -> NonEmptyRequestRangeSpecIter<'_> {
+        NonEmptyRequestRangeSpecIter::new(self.iter_infinite())
+    }
+
+    /// True if this range spec sequence repeats the last range spec forever.
+    pub fn is_infinite(&self) -> bool {
+        self.0
+            .iter()
+            .next_back()
+            .map(|(_, v)| !v.is_empty())
+            .unwrap_or_default()
+    }
+
+    pub fn iter_infinite(&self) -> ChunkRangesSeqIterInfinite<'_> {
+        ChunkRangesSeqIterInfinite {
+            current: chunk_ranges_empty(),
+            offset: 0,
+            remaining: self.0.iter().peekable(),
+        }
+    }
+
+    pub fn iter(&self) -> ChunkRangesSeqIter<'_> {
+        ChunkRangesSeqIter {
+            current: chunk_ranges_empty(),
+            offset: 0,
+            remaining: self.0.iter().peekable(),
+        }
+    }
+}
+
+fn from_ranges_inner(
+    ranges: impl IntoIterator<Item = ChunkRanges>,
+) -> (SmallVec<[(u64, ChunkRanges); 2]>, u64) {
+    let mut res = SmallVec::new();
+    let mut i = 0;
+    for range in ranges.into_iter() {
+        if range
+            != res
+                .iter()
+                .next_back()
+                .map(|(_, v)| v)
+                .unwrap_or(&ChunkRanges::empty())
+        {
+            res.push((i, range));
+        }
+        i += 1;
+    }
+    (res, i)
+}
+
+/// An infinite iterator yielding [`RangeSpec`]s for each blob in a sequence.
+///
+/// The first item yielded is the [`RangeSpec`] for the first blob in the sequence, the
+/// next item is the [`RangeSpec`] for the next blob, etc.
+#[derive(Debug)]
+pub struct ChunkRangesSeqIterInfinite<'a> {
+    /// current value
+    current: &'a ChunkRanges,
+    /// current offset
+    offset: u64,
+    /// remaining ranges
+    remaining: std::iter::Peekable<std::slice::Iter<'a, (u64, ChunkRanges)>>,
+}
+
+impl<'a> ChunkRangesSeqIterInfinite<'a> {
+    /// True if we are at the end of the iterator.
+    ///
+    /// This does not mean that the iterator is terminated, it just means that
+    /// it will repeat the same value forever.
+    pub fn is_at_end(&mut self) -> bool {
+        self.remaining.peek().is_none()
+    }
+}
+
+impl<'a> Iterator for ChunkRangesSeqIterInfinite<'a> {
+    type Item = &'a ChunkRanges;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            match self.remaining.peek() {
+                Some((offset, _)) if self.offset < *offset => {
+                    // emit current value until we reach the next offset
+                    self.offset += 1;
+                    return Some(self.current);
+                }
+                None => {
+                    // no more values, just repeat current forever
+                    self.offset += 1;
+                    return Some(self.current);
+                }
+                Some((_, ranges)) => {
+                    // get next current value, new count, and set remaining
+                    self.current = ranges;
+                    self.remaining.next();
+                }
+            }
+        }
+    }
+}
+
+/// An infinite iterator yielding [`RangeSpec`]s for each blob in a sequence.
+///
+/// The first item yielded is the [`RangeSpec`] for the first blob in the sequence, the
+/// next item is the [`RangeSpec`] for the next blob, etc.
+#[derive(Debug)]
+pub struct ChunkRangesSeqIter<'a> {
+    /// current value
+    current: &'a ChunkRanges,
+    /// current offset
+    offset: u64,
+    /// remaining ranges
+    remaining: std::iter::Peekable<std::slice::Iter<'a, (u64, ChunkRanges)>>,
+}
+
+impl<'a> Iterator for ChunkRangesSeqIter<'a> {
+    type Item = &'a ChunkRanges;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self.remaining.peek()? {
+            (offset, _) if self.offset < *offset => {
+                // emit current value until we reach the next offset
+                self.offset += 1;
+                Some(self.current)
+            }
+            (_, ranges) => {
+                // get next current value, new count, and set remaining
+                self.current = ranges;
+                self.remaining.next();
+                self.offset += 1;
+                Some(self.current)
+            }
+        }
+    }
+}
+
+/// An iterator over blobs in the sequence with a non-empty range specs.
+///
+/// default is what to use if the children of this RequestRangeSpec are empty.
+#[derive(Debug)]
+pub struct NonEmptyRequestRangeSpecIter<'a> {
+    inner: ChunkRangesSeqIterInfinite<'a>,
+    count: u64,
+}
+
+impl<'a> NonEmptyRequestRangeSpecIter<'a> {
+    fn new(inner: ChunkRangesSeqIterInfinite<'a>) -> Self {
+        Self { inner, count: 0 }
+    }
+
+    pub(crate) fn offset(&self) -> u64 {
+        self.count
+    }
+
+    pub fn is_at_end(&mut self) -> bool {
+        self.inner.is_at_end()
+    }
+}
+
+impl<'a> Iterator for NonEmptyRequestRangeSpecIter<'a> {
+    type Item = (u64, &'a ChunkRanges);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            // unwrapping is safe because we know that the inner iterator will never terminate
+            let curr = self.inner.next().unwrap();
+            let count = self.count;
+            // increase count in any case until we are at the end of possible u64 values
+            // we are unlikely to ever reach this limit.
+            self.count = self.count.checked_add(1)?;
+            // yield only if the current value is non-empty
+            if !curr.is_empty() {
+                break Some((count, curr));
+            } else if self.inner.is_at_end() {
+                // terminate instead of looping until we run out of u64 values
+                break None;
+            }
+        }
+    }
+}
+
 /// A chunk range specification as a sequence of chunk offsets.
 ///
+/// This is just the wire encoding of a [`ChunkRanges`]. You should rarely have to
+/// interact with this directly.
+///
 /// Offsets encode alternating spans starting on 0, where the first span is always
 /// deselected.
 ///
@@ -68,6 +397,11 @@ impl RangeSpec {
         Self(smallvec![0])
     }
 
+    /// Creates a [`RangeSpec`] selecting the last chunk, which is also a size proof.
+    pub fn verified_size() -> Self {
+        Self(smallvec![u64::MAX])
+    }
+
     /// Checks if this [`RangeSpec`] does not select any chunks in the blob.
     pub fn is_empty(&self) -> bool {
         self.0.is_empty()
@@ -78,6 +412,22 @@ impl RangeSpec {
         self.0.len() == 1 && self.0[0] == 0
     }
 
+    /// Returns the number of chunks selected by this [`RangeSpec`], as a tuple
+    /// with the minimum and maximum number of chunks.
+    pub fn chunks(&self) -> (u64, Option<u64>) {
+        let mut min = 0;
+        for i in 0..self.0.len() / 2 {
+            min += self.0[2 * i + 1];
+        }
+        let max = if self.0.len() % 2 != 0 {
+            // spec is open ended
+            None
+        } else {
+            Some(min)
+        };
+        (min, max)
+    }
+
     /// Creates a [`ChunkRanges`] from this [`RangeSpec`].
     pub fn to_chunk_ranges(&self) -> ChunkRanges {
         // this is zero allocation for single ranges
@@ -102,250 +452,52 @@ impl RangeSpec {
 
 impl fmt::Debug for RangeSpec {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        if f.alternate() {
-            f.debug_list()
-                .entries(self.to_chunk_ranges().iter())
-                .finish()
-        } else if self.is_all() {
+        if self.is_all() {
             write!(f, "all")
         } else if self.is_empty() {
             write!(f, "empty")
+        } else if !f.alternate() {
+            f.debug_list()
+                .entries(self.to_chunk_ranges().iter())
+                .finish()
         } else {
             f.debug_list().entries(self.0.iter()).finish()
         }
     }
 }
 
-/// A chunk range specification for a sequence of blobs.
-///
-/// To select chunks in a sequence of blobs this is encoded as a sequence of `(blob_offset,
-/// range_spec)` tuples. Offsets are interpreted in an accumulating fashion.
-///
-/// ## Example:
-///
-/// Suppose two [`RangeSpec`]s `range_a` and `range_b`.
-///
-/// - `[(0, range_a), (2, empty), (3, range_b), (1, empty)]` encodes:
-///   - Select `range_a` for children in the range `[0, 2)`
-///   - do no selection (empty) for children in the range `[2, 2+3) = [2, 5)` (3 children)
-///   - Select `range_b` for children in the range `[5, 5+1) = [5, 6)` (1 children)
-///   - do no selection (empty) for children in the open range `[6, inf)`
-///
-/// Another way to understand this is that offsets represent the number of times the
-/// previous range appears.
-///
-/// Other examples:
-///
-/// - Select `range_a` from all blobs after the 5th one in the sequence: `[(5, range_a)]`.
-///
-/// - Select `range_a` from all blobs in the sequence: `[(0, range_a)]`.
-///
-/// - Select `range_a` from blob 1234: `[(1234, range_a), (1, empty)]`.
-///
-/// - Select nothing: `[]`.
-///
-/// This is a smallvec so that we can avoid allocations in the common case of a single child
-/// range.
-#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone, Hash)]
-#[repr(transparent)]
-pub struct RangeSpecSeq(SmallVec<[(u64, RangeSpec); 2]>);
-
-impl RangeSpecSeq {
-    #[allow(dead_code)]
-    /// A [`RangeSpecSeq`] containing no chunks from any blobs in the sequence.
-    ///
-    /// [`RangeSpecSeq::iter`], will return an empty range forever.
-    pub const fn empty() -> Self {
-        Self(SmallVec::new_const())
-    }
-
-    /// If this range seq describes a range for a single item, returns the offset
-    /// and range spec for that item
-    pub fn as_single(&self) -> Option<(u64, &RangeSpec)> {
-        // we got two elements,
-        // the first element starts at offset 0,
-        // and the second element is empty
-        if self.0.len() != 2 {
-            return None;
-        }
-        let (fst_ofs, fst_val) = &self.0[0];
-        let (snd_ofs, snd_val) = &self.0[1];
-        if *snd_ofs == 1 && snd_val.is_empty() {
-            Some((*fst_ofs, fst_val))
-        } else {
-            None
-        }
-    }
+mod wire {
 
-    /// A [`RangeSpecSeq`] containing all chunks from all blobs.
-    ///
-    /// [`RangeSpecSeq::iter`], will return a full range forever.
-    pub fn all() -> Self {
-        Self(smallvec![(0, RangeSpec::all())])
-    }
+    use serde::{Deserialize, Serialize};
+    use smallvec::SmallVec;
 
-    /// Convenience function to create a [`RangeSpecSeq`] from a finite sequence of range sets.
-    pub fn from_ranges(ranges: impl IntoIterator<Item = impl AsRef<ChunkRangesRef>>) -> Self {
-        Self::new(
-            ranges
-                .into_iter()
-                .map(RangeSpec::new)
-                .chain(std::iter::once(RangeSpec::EMPTY)),
-        )
-    }
+    use super::{ChunkRangesSeq, RangeSpec};
 
-    /// Convenience function to create a [`RangeSpecSeq`] from a sequence of range sets.
-    ///
-    /// Compared to [`RangeSpecSeq::from_ranges`], this will not add an empty range spec at the end, so the final
-    /// range spec will repeat forever.
-    pub fn from_ranges_infinite(
-        ranges: impl IntoIterator<Item = impl AsRef<ChunkRangesRef>>,
-    ) -> Self {
-        Self::new(ranges.into_iter().map(RangeSpec::new))
-    }
+    #[derive(Deserialize, Serialize)]
+    pub struct RangeSpecSeq(SmallVec<[(u64, RangeSpec); 2]>);
 
-    /// Creates a new range spec sequence from a sequence of range specs.
-    ///
-    /// This will merge adjacent range specs with the same value and thus make
-    /// sure that the resulting sequence is as compact as possible.
-    pub fn new(children: impl IntoIterator<Item = RangeSpec>) -> Self {
-        let mut count = 0;
-        let mut res = SmallVec::new();
-        let before_all = RangeSpec::EMPTY;
-        for v in children.into_iter() {
-            let prev = res.last().map(|(_count, spec)| spec).unwrap_or(&before_all);
-            if &v == prev {
-                count += 1;
-            } else {
-                res.push((count, v.clone()));
-                count = 1;
+    impl From<RangeSpecSeq> for ChunkRangesSeq {
+        fn from(wire: RangeSpecSeq) -> Self {
+            let mut offset = 0;
+            let mut res = SmallVec::new();
+            for (delta, spec) in wire.0.iter() {
+                offset += *delta;
+                res.push((offset, spec.to_chunk_ranges()));
             }
+            Self(res)
         }
-        Self(res)
-    }
-
-    /// An infinite iterator of range specs for blobs in the sequence.
-    ///
-    /// Each item yielded by the iterator is the [`RangeSpec`] for a blob in the sequence.
-    /// Thus the first call to `.next()` returns the range spec for the first blob, the next
-    /// call returns the range spec of the second blob, etc.
-    pub fn iter(&self) -> RequestRangeSpecIter<'_> {
-        let before_first = self.0.first().map(|(c, _)| *c).unwrap_or_default();
-        RequestRangeSpecIter {
-            current: &EMPTY_RANGE_SPEC,
-            count: before_first,
-            remaining: &self.0,
-        }
-    }
-
-    /// An iterator over blobs in the sequence with a non-empty range spec.
-    ///
-    /// This iterator will only yield items for blobs which have at least one chunk
-    /// selected.
-    ///
-    /// This iterator is infinite if the [`RangeSpecSeq`] ends on a non-empty [`RangeSpec`],
-    /// that is all further blobs have selected chunks spans.
-    pub fn iter_non_empty(&self) -> NonEmptyRequestRangeSpecIter<'_> {
-        NonEmptyRequestRangeSpecIter::new(self.iter())
     }
-}
 
-static EMPTY_RANGE_SPEC: RangeSpec = RangeSpec::EMPTY;
-
-/// An infinite iterator yielding [`RangeSpec`]s for each blob in a sequence.
-///
-/// The first item yielded is the [`RangeSpec`] for the first blob in the sequence, the
-/// next item is the [`RangeSpec`] for the next blob, etc.
-#[derive(Debug)]
-pub struct RequestRangeSpecIter<'a> {
-    /// current value
-    current: &'a RangeSpec,
-    /// number of times to emit current before grabbing next value
-    /// if remaining is empty, this is ignored and current is emitted forever
-    count: u64,
-    /// remaining ranges
-    remaining: &'a [(u64, RangeSpec)],
-}
-
-impl<'a> RequestRangeSpecIter<'a> {
-    pub fn new(ranges: &'a [(u64, RangeSpec)]) -> Self {
-        let before_first = ranges.first().map(|(c, _)| *c).unwrap_or_default();
-        RequestRangeSpecIter {
-            current: &EMPTY_RANGE_SPEC,
-            count: before_first,
-            remaining: ranges,
-        }
-    }
-
-    /// True if we are at the end of the iterator.
-    ///
-    /// This does not mean that the iterator is terminated, it just means that
-    /// it will repeat the same value forever.
-    pub fn is_at_end(&self) -> bool {
-        self.count == 0 && self.remaining.is_empty()
-    }
-}
-
-impl<'a> Iterator for RequestRangeSpecIter<'a> {
-    type Item = &'a RangeSpec;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        Some(loop {
-            break if self.count > 0 {
-                // emit current value count times
-                self.count -= 1;
-                self.current
-            } else if let Some(((_, new), rest)) = self.remaining.split_first() {
-                // get next current value, new count, and set remaining
-                self.current = new;
-                self.count = rest.first().map(|(c, _)| *c).unwrap_or_default();
-                self.remaining = rest;
-                continue;
-            } else {
-                // no more values, just repeat current forever
-                self.current
-            };
-        })
-    }
-}
-
-/// An iterator over blobs in the sequence with a non-empty range specs.
-///
-/// default is what to use if the children of this RequestRangeSpec are empty.
-#[derive(Debug)]
-pub struct NonEmptyRequestRangeSpecIter<'a> {
-    inner: RequestRangeSpecIter<'a>,
-    count: u64,
-}
-
-impl<'a> NonEmptyRequestRangeSpecIter<'a> {
-    fn new(inner: RequestRangeSpecIter<'a>) -> Self {
-        Self { inner, count: 0 }
-    }
-
-    pub(crate) fn offset(&self) -> u64 {
-        self.count
-    }
-}
-
-impl<'a> Iterator for NonEmptyRequestRangeSpecIter<'a> {
-    type Item = (u64, &'a RangeSpec);
-
-    fn next(&mut self) -> Option<Self::Item> {
-        loop {
-            // unwrapping is safe because we know that the inner iterator will never terminate
-            let curr = self.inner.next().unwrap();
-            let count = self.count;
-            // increase count in any case until we are at the end of possible u64 values
-            // we are unlikely to ever reach this limit.
-            self.count = self.count.checked_add(1)?;
-            // yield only if the current value is non-empty
-            if !curr.is_empty() {
-                break Some((count, curr));
-            } else if self.inner.is_at_end() {
-                // terminate instead of looping until we run out of u64 values
-                break None;
+    impl From<ChunkRangesSeq> for RangeSpecSeq {
+        fn from(value: ChunkRangesSeq) -> Self {
+            let mut res = SmallVec::new();
+            let mut offset = 0;
+            for (i, r) in value.0.iter() {
+                let delta = *i - offset;
+                res.push((delta, RangeSpec::new(r)));
+                offset = *i;
             }
+            Self(res)
         }
     }
 }
@@ -354,10 +506,11 @@ impl<'a> Iterator for NonEmptyRequestRangeSpecIter<'a> {
 mod tests {
     use std::ops::Range;
 
+    use iroh_test::{assert_eq_hex, hexdump::parse_hexdump};
     use proptest::prelude::*;
 
     use super::*;
-    use crate::{assert_eq_hex, util::hexdump::parse_hexdump};
+    use crate::util::ChunkRangesExt;
 
     fn ranges(value_range: Range<u64>) -> impl Strategy<Value = ChunkRanges> {
         prop::collection::vec((value_range.clone(), value_range), 0..16).prop_map(|v| {
@@ -365,34 +518,34 @@ mod tests {
             for (a, b) in v {
                 let start = a.min(b);
                 let end = a.max(b);
-                res |= ChunkRanges::from(ChunkNum(start)..ChunkNum(end));
+                res |= ChunkRanges::chunks(start..end);
             }
             res
         })
     }
 
     fn range_spec_seq_roundtrip_impl(ranges: &[ChunkRanges]) -> Vec<ChunkRanges> {
-        let spec = RangeSpecSeq::from_ranges(ranges.iter().cloned());
-        spec.iter()
-            .map(|x| x.to_chunk_ranges())
+        let spec = ChunkRangesSeq::from_ranges(ranges.iter().cloned());
+        spec.iter_infinite()
             .take(ranges.len())
+            .cloned()
             .collect::<Vec<_>>()
     }
 
     fn range_spec_seq_bytes_roundtrip_impl(ranges: &[ChunkRanges]) -> Vec<ChunkRanges> {
-        let spec = RangeSpecSeq::from_ranges(ranges.iter().cloned());
+        let spec = ChunkRangesSeq::from_ranges(ranges.iter().cloned());
         let bytes = postcard::to_allocvec(&spec).unwrap();
-        let spec2: RangeSpecSeq = postcard::from_bytes(&bytes).unwrap();
+        let spec2: ChunkRangesSeq = postcard::from_bytes(&bytes).unwrap();
         spec2
-            .iter()
-            .map(|x| x.to_chunk_ranges())
+            .iter_infinite()
             .take(ranges.len())
+            .cloned()
             .collect::<Vec<_>>()
     }
 
     fn mk_case(case: Vec<Range<u64>>) -> Vec<ChunkRanges> {
         case.iter()
-            .map(|x| ChunkRanges::from(ChunkNum(x.start)..ChunkNum(x.end)))
+            .map(|x| ChunkRanges::chunks(x.start..x.end))
             .collect::<Vec<_>>()
     }
 
@@ -409,21 +562,21 @@ mod tests {
                 ",
             ),
             (
-                RangeSpec::new(ChunkRanges::from(ChunkNum(64)..)),
+                RangeSpec::new(ChunkRanges::chunks(64..)),
                 r"
                     01 # length prefix - 1 element
                     40 # span width - 64. everything starting from 64 is included
                 ",
             ),
             (
-                RangeSpec::new(ChunkRanges::from(ChunkNum(10000)..)),
+                RangeSpec::new(ChunkRanges::chunks(10000..)),
                 r"
                     01 # length prefix - 1 element
                     904E # span width - 10000, 904E in postcard varint encoding. everything starting from 10000 is included
                 ",
             ),
             (
-                RangeSpec::new(ChunkRanges::from(..ChunkNum(64))),
+                RangeSpec::new(ChunkRanges::chunks(..64)),
                 r"
                     02 # length prefix - 2 elements
                     00 # span width - 0. everything stating from 0 is included
@@ -431,10 +584,7 @@ mod tests {
                 ",
             ),
             (
-                RangeSpec::new(
-                    &ChunkRanges::from(ChunkNum(1)..ChunkNum(3))
-                        | &ChunkRanges::from(ChunkNum(9)..ChunkNum(13)),
-                ),
+                RangeSpec::new(&ChunkRanges::chunks(1..3) | &ChunkRanges::chunks(9..13)),
                 r"
                     04 # length prefix - 4 elements
                     01 # span width - 1
@@ -453,9 +603,9 @@ mod tests {
     #[test]
     fn range_spec_seq_wire_format() {
         let cases = [
-            (RangeSpecSeq::empty(), "00"),
+            (ChunkRangesSeq::empty(), "00"),
             (
-                RangeSpecSeq::all(),
+                ChunkRangesSeq::all(),
                 r"
                     01 # 1 tuple in total
                     # first tuple
@@ -464,9 +614,9 @@ mod tests {
             ",
             ),
             (
-                RangeSpecSeq::from_ranges([
-                    ChunkRanges::from(ChunkNum(1)..ChunkNum(3)),
-                    ChunkRanges::from(ChunkNum(7)..ChunkNum(13)),
+                ChunkRangesSeq::from_ranges([
+                    ChunkRanges::chunks(1..3),
+                    ChunkRanges::chunks(7..13),
                 ]),
                 r"
                     03 # 3 tuples in total
@@ -482,11 +632,11 @@ mod tests {
                 ",
             ),
             (
-                RangeSpecSeq::from_ranges_infinite([
+                ChunkRangesSeq::from_ranges_infinite([
                     ChunkRanges::empty(),
                     ChunkRanges::empty(),
                     ChunkRanges::empty(),
-                    ChunkRanges::from(ChunkNum(7)..),
+                    ChunkRanges::chunks(7..),
                     ChunkRanges::all(),
                 ]),
                 r"
@@ -530,12 +680,13 @@ mod tests {
             (vec![1..2, 1..2, 2..3, 2..3], 3),
         ] {
             let case = mk_case(case);
-            let spec = RangeSpecSeq::from_ranges(case);
+            let spec = ChunkRangesSeq::from_ranges(case);
             assert_eq!(spec.0.len(), expected_count);
         }
     }
 
     proptest! {
+
         #[test]
         fn range_spec_roundtrip(ranges in ranges(0..1000)) {
             let spec = RangeSpec::new(&ranges);
diff --git a/src/provider.rs b/src/provider.rs
index bf55fe751..ff2d4a0e5 100644
--- a/src/provider.rs
+++ b/src/provider.rs
@@ -1,658 +1,762 @@
-//! The server side API
-use std::{fmt::Debug, sync::Arc, time::Duration};
+//! The low level server side API
+//!
+//! Note that while using this API directly is fine, the standard way
+//! to provide data is to just register a [`crate::net_protocol`] protocol
+//! handler with an [`iroh::Endpoint`](iroh::protocol::Router).
+use std::{
+    fmt::Debug,
+    io,
+    ops::{Deref, DerefMut},
+    pin::Pin,
+    task::Poll,
+    time::Duration,
+};
 
 use anyhow::{Context, Result};
-use bao_tree::io::{
-    fsm::{encode_ranges_validated, Outboard},
-    EncodeError,
-};
-use futures_lite::future::Boxed as BoxFuture;
-use iroh::endpoint::{self, RecvStream, SendStream};
-use iroh_io::{
-    stats::{SliceReaderStats, StreamWriterStats, TrackingSliceReader, TrackingStreamWriter},
-    AsyncSliceReader, AsyncStreamWriter, TokioStreamWriter,
+use bao_tree::ChunkRanges;
+use iroh::{
+    endpoint::{self, RecvStream, SendStream},
+    NodeId,
 };
-use serde::{Deserialize, Serialize};
-use tracing::{debug, debug_span, info, trace, warn};
-use tracing_futures::Instrument;
+use irpc::channel::oneshot;
+use n0_future::StreamExt;
+use serde::de::DeserializeOwned;
+use tokio::{io::AsyncRead, select, sync::mpsc};
+use tracing::{debug, debug_span, error, warn, Instrument};
 
 use crate::{
-    hashseq::parse_hash_seq,
-    protocol::{GetRequest, RangeSpec, Request},
-    store::*,
-    util::{local_pool::LocalPoolHandle, Tag},
-    BlobFormat, Hash,
+    api::{self, blobs::Bitfield, Store},
+    hashseq::HashSeq,
+    protocol::{
+        ChunkRangesSeq, GetManyRequest, GetRequest, ObserveItem, ObserveRequest, PushRequest,
+        Request,
+    },
+    Hash,
 };
 
-/// Events emitted by the provider informing about the current status.
-#[derive(Debug, Clone)]
+/// Provider progress events, to keep track of what the provider is doing.
+///
+/// ClientConnected ->
+///    (GetRequestReceived -> (TransferStarted -> TransferProgress*n)*n -> (TransferCompleted | TransferAborted))*n ->
+/// ConnectionClosed
+#[derive(Debug)]
 pub enum Event {
-    /// A new collection or tagged blob has been added
-    TaggedBlobAdded {
-        /// The hash of the added data
-        hash: Hash,
-        /// The format of the added data
-        format: BlobFormat,
-        /// The tag of the added data
-        tag: Tag,
-    },
-    /// A new client connected to the node.
+    /// A new client connected to the provider.
     ClientConnected {
-        /// An unique connection id.
         connection_id: u64,
+        node_id: NodeId,
+        permitted: oneshot::Sender<bool>,
     },
-    /// A request was received from a client.
+    /// Connection closed.
+    ConnectionClosed { connection_id: u64 },
+    /// A new get request was received from the provider.
     GetRequestReceived {
-        /// An unique connection id.
+        /// The connection id. Multiple requests can be sent over the same connection.
         connection_id: u64,
-        /// An identifier uniquely identifying this transfer request.
+        /// The request id. There is a new id for each request.
         request_id: u64,
-        /// The hash for which the client wants to receive data.
+        /// The root hash of the request.
         hash: Hash,
+        /// The exact query ranges of the request.
+        ranges: ChunkRangesSeq,
     },
-    /// A sequence of hashes has been found and is being transferred.
-    TransferHashSeqStarted {
-        /// An unique connection id.
+    /// A new get request was received from the provider.
+    GetManyRequestReceived {
+        /// The connection id. Multiple requests can be sent over the same connection.
         connection_id: u64,
-        /// An identifier uniquely identifying this transfer request.
+        /// The request id. There is a new id for each request.
         request_id: u64,
-        /// The number of blobs in the sequence.
-        num_blobs: u64,
+        /// The root hash of the request.
+        hashes: Vec<Hash>,
+        /// The exact query ranges of the request.
+        ranges: ChunkRangesSeq,
     },
-    /// A chunk of a blob was transferred.
-    ///
-    /// These events will be sent with try_send, so you can not assume that you
-    /// will receive all of them.
-    TransferProgress {
-        /// An unique connection id.
+    /// A new get request was received from the provider.
+    PushRequestReceived {
+        /// The connection id. Multiple requests can be sent over the same connection.
         connection_id: u64,
-        /// An identifier uniquely identifying this transfer request.
+        /// The request id. There is a new id for each request.
         request_id: u64,
-        /// The hash for which we are transferring data.
+        /// The root hash of the request.
         hash: Hash,
-        /// Offset up to which we have transferred data.
-        end_offset: u64,
+        /// The exact query ranges of the request.
+        ranges: ChunkRangesSeq,
+        /// Complete this to permit the request.
+        permitted: oneshot::Sender<bool>,
     },
-    /// A blob in a sequence was transferred.
-    TransferBlobCompleted {
-        /// An unique connection id.
+    /// Transfer for the nth blob started.
+    TransferStarted {
+        /// The connection id. Multiple requests can be sent over the same connection.
         connection_id: u64,
-        /// An identifier uniquely identifying this transfer request.
+        /// The request id. There is a new id for each request.
         request_id: u64,
-        /// The hash of the blob
-        hash: Hash,
-        /// The index of the blob in the sequence.
+        /// The index of the blob in the request. 0 for the first blob or for raw blob requests.
         index: u64,
-        /// The size of the blob transferred.
+        /// The hash of the blob. This is the hash of the request for the first blob, the child hash (index-1) for subsequent blobs.
+        hash: Hash,
+        /// The size of the blob. This is the full size of the blob, not the size we are sending.
         size: u64,
     },
-    /// A request was completed and the data was sent to the client.
+    /// Progress of the transfer.
+    TransferProgress {
+        /// The connection id. Multiple requests can be sent over the same connection.
+        connection_id: u64,
+        /// The request id. There is a new id for each request.
+        request_id: u64,
+        /// The index of the blob in the request. 0 for the first blob or for raw blob requests.
+        index: u64,
+        /// The end offset of the chunk that was sent.
+        end_offset: u64,
+    },
+    /// Entire transfer completed.
     TransferCompleted {
-        /// An unique connection id.
+        /// The connection id. Multiple requests can be sent over the same connection.
         connection_id: u64,
-        /// An identifier uniquely identifying this transfer request.
+        /// The request id. There is a new id for each request.
         request_id: u64,
-        /// statistics about the transfer
+        /// Statistics about the transfer.
         stats: Box<TransferStats>,
     },
-    /// A request was aborted because the client disconnected.
+    /// Entire transfer aborted
     TransferAborted {
-        /// The quic connection id.
+        /// The connection id. Multiple requests can be sent over the same connection.
         connection_id: u64,
-        /// An identifier uniquely identifying this request.
+        /// The request id. There is a new id for each request.
         request_id: u64,
-        /// statistics about the transfer. This is None if the transfer
-        /// was aborted before any data was sent.
+        /// Statistics about the part of the transfer that was aborted.
         stats: Option<Box<TransferStats>>,
     },
 }
 
-/// The stats for a transfer of a collection or blob.
-#[derive(Debug, Clone, Copy, Default)]
+/// Statistics about a successful or failed transfer.
+#[derive(Debug)]
 pub struct TransferStats {
-    /// Stats for sending to the client.
-    pub send: StreamWriterStats,
-    /// Stats for reading from disk.
-    pub read: SliceReaderStats,
-    /// The total duration of the transfer.
-    pub duration: Duration,
-}
-
-/// Progress updates for the add operation.
-#[derive(Debug, Serialize, Deserialize)]
-pub enum AddProgress {
-    /// An item was found with name `name`, from now on referred to via `id`
-    Found {
-        /// A new unique id for this entry.
-        id: u64,
-        /// The name of the entry.
-        name: String,
-        /// The size of the entry in bytes.
-        size: u64,
-    },
-    /// We got progress ingesting item `id`.
-    Progress {
-        /// The unique id of the entry.
-        id: u64,
-        /// The offset of the progress, in bytes.
-        offset: u64,
-    },
-    /// We are done with `id`, and the hash is `hash`.
-    Done {
-        /// The unique id of the entry.
-        id: u64,
-        /// The hash of the entry.
-        hash: Hash,
-    },
-    /// We are done with the whole operation.
-    AllDone {
-        /// The hash of the created data.
-        hash: Hash,
-        /// The format of the added data.
-        format: BlobFormat,
-        /// The tag of the added data.
-        tag: Tag,
-    },
-    /// We got an error and need to abort.
+    /// The number of bytes sent that are part of the payload.
+    pub payload_bytes_sent: u64,
+    /// The number of bytes sent that are not part of the payload.
     ///
-    /// This will be the last message in the stream.
-    Abort(serde_error::Error),
-}
-
-/// Progress updates for the batch add operation.
-#[derive(Debug, Serialize, Deserialize)]
-pub enum BatchAddPathProgress {
-    /// An item was found with the given size
-    Found {
-        /// The size of the entry in bytes.
-        size: u64,
-    },
-    /// We got progress ingesting the item.
-    Progress {
-        /// The offset of the progress, in bytes.
-        offset: u64,
-    },
-    /// We are done, and the hash is `hash`.
-    Done {
-        /// The hash of the entry.
-        hash: Hash,
-    },
-    /// We got an error and need to abort.
+    /// Hash pairs and the initial size header.
+    pub other_bytes_sent: u64,
+    /// The number of bytes read from the stream.
     ///
-    /// This will be the last message in the stream.
-    Abort(serde_error::Error),
+    /// This is the size of the request.
+    pub bytes_read: u64,
+    /// Total duration from reading the request to transfer completed.
+    pub duration: Duration,
 }
 
 /// Read the request from the getter.
 ///
-/// Will fail if there is an error while reading, if the reader
-/// contains more data than the Request, or if no valid request is sent.
+/// Will fail if there is an error while reading, or if no valid request is sent.
+///
+/// This will read exactly the number of bytes needed for the request, and
+/// leave the rest of the stream for the caller to read.
 ///
-/// When successful, the buffer is empty after this function call.
-pub async fn read_request(mut reader: RecvStream) -> Result<Request> {
-    let payload = reader
-        .read_to_end(crate::protocol::MAX_MESSAGE_SIZE)
-        .await?;
-    let request: Request = postcard::from_bytes(&payload)?;
-    Ok(request)
+/// It is up to the caller do decide if there should be more data.
+pub async fn read_request(reader: &mut ProgressReader) -> Result<Request> {
+    let mut counting = CountingReader::new(&mut reader.inner);
+    let res = Request::read_async(&mut counting).await?;
+    reader.bytes_read += counting.read();
+    Ok(res)
 }
 
-/// Transfers a blob or hash sequence to the client.
-///
-/// The difference to [`handle_get`] is that we already have a reader for the
-/// root blob and outboard.
-///
-/// First, it transfers the root blob. Then, if needed, it sequentially
-/// transfers each individual blob data.
-///
-/// The transfer fail if there is an error writing to the writer or reading from
-/// the database.
-///
-/// If a blob from the hash sequence cannot be found in the database, the
-/// transfer will return with [`SentStatus::NotFound`]. If the transfer completes
-/// successfully, it will return with [`SentStatus::Sent`].
-pub(crate) async fn transfer_hash_seq<D: Map>(
-    request: GetRequest,
-    // Store from which to fetch blobs.
-    db: &D,
-    // Response writer, containing the quinn stream.
-    writer: &mut ResponseWriter,
-    // the collection to transfer
-    mut outboard: impl Outboard,
-    mut data: impl AsyncSliceReader,
-    stats: &mut TransferStats,
-) -> Result<SentStatus> {
-    let hash = request.hash;
-    let events = writer.events.clone();
-    let request_id = writer.request_id();
-    let connection_id = writer.connection_id();
-
-    // if the request is just for the root, we don't need to deserialize the collection
-    let just_root = matches!(request.ranges.as_single(), Some((0, _)));
-    let mut c = if !just_root {
-        // parse the hash seq
-        let (stream, num_blobs) = parse_hash_seq(&mut data).await?;
-        writer
-            .events
-            .send(|| Event::TransferHashSeqStarted {
-                connection_id: writer.connection_id(),
-                request_id: writer.request_id(),
-                num_blobs,
-            })
-            .await;
-        Some(stream)
-    } else {
-        None
-    };
+#[derive(Debug)]
+pub struct StreamContext {
+    /// The connection ID from the connection
+    pub connection_id: u64,
+    /// The request ID from the recv stream
+    pub request_id: u64,
+    /// The number of bytes written that are part of the payload
+    pub payload_bytes_sent: u64,
+    /// The number of bytes written that are not part of the payload
+    pub other_bytes_sent: u64,
+    /// The number of bytes read from the stream
+    pub bytes_read: u64,
+    /// The progress sender to send events to
+    pub progress: EventSender,
+}
 
-    let mk_progress = |end_offset| Event::TransferProgress {
-        connection_id,
-        request_id,
-        hash,
-        end_offset,
-    };
+/// Wrapper for a [`quinn::SendStream`] with additional per request information.
+#[derive(Debug)]
+pub struct ProgressWriter {
+    /// The quinn::SendStream to write to
+    pub inner: SendStream,
+    pub(crate) context: StreamContext,
+}
 
-    let mut prev = 0;
-    for (offset, ranges) in request.ranges.iter_non_empty() {
-        // create a tracking writer so we can get some stats for writing
-        let mut tw = writer.tracking_writer();
-        if offset == 0 {
-            debug!("writing ranges '{:?}' of sequence {}", ranges, hash);
-            // wrap the data reader in a tracking reader so we can get some stats for reading
-            let mut tracking_reader = TrackingSliceReader::new(&mut data);
-            let mut sending_reader =
-                SendingSliceReader::new(&mut tracking_reader, &events, mk_progress);
-            // send the root
-            tw.write(outboard.tree().size().to_le_bytes().as_slice())
-                .await?;
-            encode_ranges_validated(
-                &mut sending_reader,
-                &mut outboard,
-                &ranges.to_chunk_ranges(),
-                &mut tw,
-            )
-            .await?;
-            stats.read += tracking_reader.stats();
-            stats.send += tw.stats();
-            debug!(
-                "finished writing ranges '{:?}' of collection {}",
-                ranges, hash
-            );
-        } else {
-            let c = c.as_mut().context("collection parser not available")?;
-            debug!("wrtiting ranges '{:?}' of child {}", ranges, offset);
-            // skip to the next blob if there is a gap
-            if prev < offset - 1 {
-                c.skip(offset - prev - 1).await?;
-            }
-            if let Some(hash) = c.next().await? {
-                tokio::task::yield_now().await;
-                let (status, size, blob_read_stats) =
-                    send_blob(db, hash, ranges, &mut tw, events.clone(), mk_progress).await?;
-                stats.send += tw.stats();
-                stats.read += blob_read_stats;
-                if SentStatus::NotFound == status {
-                    writer.inner.finish()?;
-                    return Ok(status);
-                }
+impl Deref for ProgressWriter {
+    type Target = StreamContext;
 
-                writer
-                    .events
-                    .send(|| Event::TransferBlobCompleted {
-                        connection_id: writer.connection_id(),
-                        request_id: writer.request_id(),
-                        hash,
-                        index: offset - 1,
-                        size,
-                    })
-                    .await;
-            } else {
-                // nothing more we can send
-                break;
-            }
-            prev = offset;
-        }
+    fn deref(&self) -> &Self::Target {
+        &self.context
     }
-
-    debug!("done writing");
-    Ok(SentStatus::Sent)
 }
 
-struct SendingSliceReader<'a, R, F> {
-    inner: R,
-    sender: &'a EventSender,
-    make_event: F,
-}
-
-impl<'a, R: AsyncSliceReader, F: Fn(u64) -> Event> SendingSliceReader<'a, R, F> {
-    fn new(inner: R, sender: &'a EventSender, make_event: F) -> Self {
-        Self {
-            inner,
-            sender,
-            make_event,
-        }
+impl DerefMut for ProgressWriter {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.context
     }
 }
 
-impl<R: AsyncSliceReader, F: Fn(u64) -> Event> AsyncSliceReader for SendingSliceReader<'_, R, F> {
-    async fn read_at(&mut self, offset: u64, len: usize) -> std::io::Result<bytes::Bytes> {
-        let res = self.inner.read_at(offset, len).await;
-        if let Ok(res) = res.as_ref() {
-            let end_offset = offset + res.len() as u64;
-            self.sender.try_send(|| (self.make_event)(end_offset));
-        }
-        res
+impl StreamContext {
+    /// Increase the write count due to a non-payload write.
+    pub fn log_other_write(&mut self, len: usize) {
+        self.other_bytes_sent += len as u64;
     }
 
-    async fn size(&mut self) -> std::io::Result<u64> {
-        self.inner.size().await
+    pub async fn send_transfer_completed(&mut self) {
+        self.progress
+            .send(|| Event::TransferCompleted {
+                connection_id: self.connection_id,
+                request_id: self.request_id,
+                stats: Box::new(TransferStats {
+                    payload_bytes_sent: self.payload_bytes_sent,
+                    other_bytes_sent: self.other_bytes_sent,
+                    bytes_read: self.bytes_read,
+                    duration: Duration::ZERO,
+                }),
+            })
+            .await;
     }
-}
-
-/// Trait for sending blob events.
-pub trait CustomEventSender: std::fmt::Debug + Sync + Send + 'static {
-    /// Send an event and wait for it to be sent.
-    fn send(&self, event: Event) -> BoxFuture<()>;
 
-    /// Try to send an event.
-    fn try_send(&self, event: Event);
-}
-
-/// A sender for events related to blob transfers.
-///
-/// The sender is disabled by default.
-#[derive(Debug, Clone, Default)]
-pub struct EventSender {
-    inner: Option<Arc<dyn CustomEventSender>>,
-}
+    pub async fn send_transfer_aborted(&mut self) {
+        self.progress
+            .send(|| Event::TransferAborted {
+                connection_id: self.connection_id,
+                request_id: self.request_id,
+                stats: Some(Box::new(TransferStats {
+                    payload_bytes_sent: self.payload_bytes_sent,
+                    other_bytes_sent: self.other_bytes_sent,
+                    bytes_read: self.bytes_read,
+                    duration: Duration::ZERO,
+                })),
+            })
+            .await;
+    }
 
-impl<T: CustomEventSender> From<T> for EventSender {
-    fn from(inner: T) -> Self {
-        Self {
-            inner: Some(Arc::new(inner)),
-        }
+    /// Increase the write count due to a payload write, and notify the progress sender.
+    ///
+    /// `index` is the index of the blob in the request.
+    /// `offset` is the offset in the blob where the write started.
+    /// `len` is the length of the write.
+    pub fn notify_payload_write(&mut self, index: u64, offset: u64, len: usize) {
+        self.payload_bytes_sent += len as u64;
+        self.progress.try_send(|| Event::TransferProgress {
+            connection_id: self.connection_id,
+            request_id: self.request_id,
+            index,
+            end_offset: offset + len as u64,
+        });
     }
-}
 
-impl EventSender {
-    /// Create a new event sender.
-    pub fn new(inner: Option<Arc<dyn CustomEventSender>>) -> Self {
-        Self { inner }
+    /// Send a get request received event.
+    ///
+    /// This sends all the required information to make sense of subsequent events such as
+    /// [`Event::TransferStarted`] and [`Event::TransferProgress`].
+    pub async fn send_get_request_received(&self, hash: &Hash, ranges: &ChunkRangesSeq) {
+        self.progress
+            .send(|| Event::GetRequestReceived {
+                connection_id: self.connection_id,
+                request_id: self.request_id,
+                hash: *hash,
+                ranges: ranges.clone(),
+            })
+            .await;
     }
 
-    /// Send an event.
+    /// Send a get request received event.
     ///
-    /// If the inner sender is not set, the function to produce the event will
-    /// not be called. So any cost associated with gathering information for the
-    /// event will not be incurred.
-    pub async fn send(&self, event: impl FnOnce() -> Event) {
-        if let Some(inner) = &self.inner {
-            let event = event();
-            inner.as_ref().send(event).await;
-        }
+    /// This sends all the required information to make sense of subsequent events such as
+    /// [`Event::TransferStarted`] and [`Event::TransferProgress`].
+    pub async fn send_get_many_request_received(&self, hashes: &[Hash], ranges: &ChunkRangesSeq) {
+        self.progress
+            .send(|| Event::GetManyRequestReceived {
+                connection_id: self.connection_id,
+                request_id: self.request_id,
+                hashes: hashes.to_vec(),
+                ranges: ranges.clone(),
+            })
+            .await;
     }
 
-    /// Try to send an event.
+    /// Authorize a push request.
+    ///
+    /// This will send a request to the event sender, and wait for a response if a
+    /// progress sender is enabled. If not, it will always fail.
     ///
-    /// This will just drop the event if it can not be sent immediately. So it
-    /// is only appropriate for events that are not critical, such as
-    /// self-contained progress updates.
-    pub fn try_send(&self, event: impl FnOnce() -> Event) {
-        if let Some(inner) = &self.inner {
-            let event = event();
-            inner.as_ref().try_send(event);
+    /// We want to make accepting push requests very explicit, since this allows
+    /// remote nodes to add arbitrary data to our store.
+    #[must_use = "permit should be checked by the caller"]
+    pub async fn authorize_push_request(&self, hash: &Hash, ranges: &ChunkRangesSeq) -> bool {
+        let mut wait_for_permit = None;
+        // send the request, including the permit channel
+        self.progress
+            .send(|| {
+                let (tx, rx) = oneshot::channel();
+                wait_for_permit = Some(rx);
+                Event::PushRequestReceived {
+                    connection_id: self.connection_id,
+                    request_id: self.request_id,
+                    hash: *hash,
+                    ranges: ranges.clone(),
+                    permitted: tx,
+                }
+            })
+            .await;
+        // wait for the permit, if necessary
+        if let Some(wait_for_permit) = wait_for_permit {
+            // if somebody does not handle the request, they will drop the channel,
+            // and this will fail immediately.
+            wait_for_permit.await.unwrap_or(false)
+        } else {
+            false
         }
     }
+
+    /// Send a transfer started event.
+    pub async fn send_transfer_started(&self, index: u64, hash: &Hash, size: u64) {
+        self.progress
+            .send(|| Event::TransferStarted {
+                connection_id: self.connection_id,
+                request_id: self.request_id,
+                index,
+                hash: *hash,
+                size,
+            })
+            .await;
+    }
 }
 
 /// Handle a single connection.
-pub async fn handle_connection<D: Map>(
+pub async fn handle_connection(
     connection: endpoint::Connection,
-    db: D,
-    events: EventSender,
-    rt: LocalPoolHandle,
+    store: Store,
+    progress: EventSender,
 ) {
     let connection_id = connection.stable_id() as u64;
     let span = debug_span!("connection", connection_id);
     async move {
+        let Ok(node_id) = connection.remote_node_id() else {
+            warn!("failed to get node id");
+            return;
+        };
+        if !progress
+            .authorize_client_connection(connection_id, node_id)
+            .await
+        {
+            debug!("client not authorized to connect");
+            return;
+        }
         while let Ok((writer, reader)) = connection.accept_bi().await {
             // The stream ID index is used to identify this request.  Requests only arrive in
             // bi-directional RecvStreams initiated by the client, so this uniquely identifies them.
             let request_id = reader.id().index();
             let span = debug_span!("stream", stream_id = %request_id);
-            let writer = ResponseWriter {
-                connection_id,
-                events: events.clone(),
+            let store = store.clone();
+            let mut writer = ProgressWriter {
                 inner: writer,
+                context: StreamContext {
+                    connection_id,
+                    request_id,
+                    payload_bytes_sent: 0,
+                    other_bytes_sent: 0,
+                    bytes_read: 0,
+                    progress: progress.clone(),
+                },
             };
-            events
-                .send(|| Event::ClientConnected { connection_id })
-                .await;
-            let db = db.clone();
-            rt.spawn_detached(|| {
+            tokio::spawn(
                 async move {
-                    if let Err(err) = handle_stream(db, reader, writer).await {
-                        warn!("error: {err:#?}",);
+                    match handle_stream(store, reader, &mut writer).await {
+                        Ok(()) => {
+                            writer.send_transfer_completed().await;
+                        }
+                        Err(err) => {
+                            warn!("error: {err:#?}",);
+                            writer.send_transfer_aborted().await;
+                        }
                     }
                 }
-                .instrument(span)
-            });
+                .instrument(span),
+            );
         }
+        progress
+            .send(Event::ConnectionClosed { connection_id })
+            .await;
     }
     .instrument(span)
     .await
 }
 
-async fn handle_stream<D: Map>(db: D, reader: RecvStream, writer: ResponseWriter) -> Result<()> {
+async fn handle_stream(
+    store: Store,
+    reader: RecvStream,
+    writer: &mut ProgressWriter,
+) -> Result<()> {
     // 1. Decode the request.
     debug!("reading request");
-    let request = match read_request(reader).await {
-        Ok(r) => r,
+    let mut reader = ProgressReader {
+        inner: reader,
+        context: StreamContext {
+            connection_id: writer.connection_id,
+            request_id: writer.request_id,
+            payload_bytes_sent: 0,
+            other_bytes_sent: 0,
+            bytes_read: 0,
+            progress: writer.progress.clone(),
+        },
+    };
+    let request = match read_request(&mut reader).await {
+        Ok(request) => request,
         Err(e) => {
-            writer.notify_transfer_aborted(None).await;
+            // todo: increase invalid requests metric counter
             return Err(e);
         }
     };
 
     match request {
-        Request::Get(request) => handle_get(db, request, writer).await,
+        Request::Get(request) => {
+            // we expect no more bytes after the request, so if there are more bytes, it is an invalid request.
+            reader.inner.read_to_end(0).await?;
+            // move the context so we don't lose the bytes read
+            writer.context = reader.context;
+            handle_get(store, request, writer).await
+        }
+        Request::GetMany(request) => {
+            // we expect no more bytes after the request, so if there are more bytes, it is an invalid request.
+            reader.inner.read_to_end(0).await?;
+            // move the context so we don't lose the bytes read
+            writer.context = reader.context;
+            handle_get_many(store, request, writer).await
+        }
+        Request::Observe(request) => {
+            // we expect no more bytes after the request, so if there are more bytes, it is an invalid request.
+            reader.inner.read_to_end(0).await?;
+            handle_observe(store, request, writer).await
+        }
+        Request::Push(request) => {
+            writer.inner.finish()?;
+            handle_push(store, request, reader).await
+        }
+        _ => anyhow::bail!("unsupported request: {request:?}"),
+        // Request::Push(request) => handle_push(store, request, writer).await,
     }
 }
 
 /// Handle a single get request.
 ///
-/// Requires the request, a database, and a writer.
-pub async fn handle_get<D: Map>(
-    db: D,
+/// Requires a database, the request, and a writer.
+pub async fn handle_get(
+    store: Store,
     request: GetRequest,
-    mut writer: ResponseWriter,
+    writer: &mut ProgressWriter,
 ) -> Result<()> {
     let hash = request.hash;
-    debug!(%hash, "received request");
+    debug!(%hash, "get received request");
+
     writer
-        .events
-        .send(|| Event::GetRequestReceived {
-            hash,
-            connection_id: writer.connection_id(),
-            request_id: writer.request_id(),
-        })
+        .send_get_request_received(&hash, &request.ranges)
         .await;
-
-    // 4. Attempt to find hash
-    match db.get(&hash).await? {
-        // Collection or blob request
-        Some(entry) => {
-            let mut stats = Box::<TransferStats>::default();
-            let t0 = std::time::Instant::now();
-            // 5. Transfer data!
-            let res = transfer_hash_seq(
-                request,
-                &db,
-                &mut writer,
-                entry.outboard().await?,
-                entry.data_reader().await?,
-                &mut stats,
-            )
-            .await;
-            stats.duration = t0.elapsed();
-            match res {
-                Ok(SentStatus::Sent) => {
-                    writer.notify_transfer_completed(&hash, stats).await;
-                }
-                Ok(SentStatus::NotFound) => {
-                    writer.notify_transfer_aborted(Some(stats)).await;
-                }
-                Err(e) => {
-                    writer.notify_transfer_aborted(Some(stats)).await;
-                    return Err(e);
+    let mut hash_seq = None;
+    for (offset, ranges) in request.ranges.iter_non_empty_infinite() {
+        if offset == 0 {
+            send_blob(&store, offset, hash, ranges.clone(), writer).await?;
+        } else {
+            // todo: this assumes that 1. the hashseq is complete and 2. it is
+            // small enough to fit in memory.
+            //
+            // This should really read the hashseq from the store in chunks,
+            // only where needed, so we can deal with holes and large hashseqs.
+            let hash_seq = match &hash_seq {
+                Some(b) => b,
+                None => {
+                    let bytes = store.get_bytes(hash).await?;
+                    let hs = HashSeq::try_from(bytes)?;
+                    hash_seq = Some(hs);
+                    hash_seq.as_ref().unwrap()
                 }
-            }
-
-            debug!("finished response");
+            };
+            let o = usize::try_from(offset - 1).context("offset too large")?;
+            let Some(hash) = hash_seq.get(o) else {
+                break;
+            };
+            send_blob(&store, offset, hash, ranges.clone(), writer).await?;
         }
-        None => {
-            debug!("not found {}", hash);
-            writer.notify_transfer_aborted(None).await;
-            writer.inner.finish()?;
+    }
+
+    Ok(())
+}
+
+/// Handle a single get request.
+///
+/// Requires a database, the request, and a writer.
+pub async fn handle_get_many(
+    store: Store,
+    request: GetManyRequest,
+    writer: &mut ProgressWriter,
+) -> Result<()> {
+    debug!("get_many received request");
+    writer
+        .send_get_many_request_received(&request.hashes, &request.ranges)
+        .await;
+    let request_ranges = request.ranges.iter_infinite();
+    for (child, (hash, ranges)) in request.hashes.iter().zip(request_ranges).enumerate() {
+        if !ranges.is_empty() {
+            send_blob(&store, child as u64, *hash, ranges.clone(), writer).await?;
         }
+    }
+    Ok(())
+}
+
+/// Handle a single push request.
+///
+/// Requires a database, the request, and a reader.
+pub async fn handle_push(
+    store: Store,
+    request: PushRequest,
+    mut reader: ProgressReader,
+) -> Result<()> {
+    let hash = request.hash;
+    debug!(%hash, "push received request");
+    if !reader.authorize_push_request(&hash, &request.ranges).await {
+        debug!("push request not authorized");
+        return Ok(());
     };
+    let mut request_ranges = request.ranges.iter_infinite();
+    let root_ranges = request_ranges.next().expect("infinite iterator");
+    if !root_ranges.is_empty() {
+        // todo: send progress from import_bao_quinn or rename to import_bao_quinn_with_progress
+        store
+            .import_bao_quinn(hash, root_ranges.clone(), &mut reader.inner)
+            .await?;
+    }
+    if request.ranges.is_blob() {
+        debug!("push request complete");
+        return Ok(());
+    }
+    // todo: we assume here that the hash sequence is complete. For some requests this might not be the case. We would need `LazyHashSeq` for that, but it is buggy as of now!
+    let hash_seq = store.get_bytes(hash).await?;
+    let hash_seq = HashSeq::try_from(hash_seq)?;
+    for (child_hash, child_ranges) in hash_seq.into_iter().zip(request_ranges) {
+        if child_ranges.is_empty() {
+            continue;
+        }
+        store
+            .import_bao_quinn(child_hash, child_ranges.clone(), &mut reader.inner)
+            .await?;
+    }
+    Ok(())
+}
+
+/// Send a blob to the client.
+pub(crate) async fn send_blob(
+    store: &Store,
+    index: u64,
+    hash: Hash,
+    ranges: ChunkRanges,
+    writer: &mut ProgressWriter,
+) -> api::Result<()> {
+    Ok(store
+        .export_bao(hash, ranges)
+        .write_quinn_with_progress(&mut writer.inner, &mut writer.context, &hash, index)
+        .await?)
+}
+
+/// Handle a single push request.
+///
+/// Requires a database, the request, and a reader.
+pub async fn handle_observe(
+    store: Store,
+    request: ObserveRequest,
+    writer: &mut ProgressWriter,
+) -> Result<()> {
+    let mut stream = store.observe(request.hash).stream().await?;
+    let mut old = stream
+        .next()
+        .await
+        .ok_or(anyhow::anyhow!("observe stream closed before first value"))?;
+    // send the initial bitfield
+    send_observe_item(writer, &old).await?;
+    // send updates until the remote loses interest
+    loop {
+        select! {
+            new = stream.next() => {
+                let new = new.context("observe stream closed")?;
+                let diff = old.diff(&new);
+                if diff.is_empty() {
+                    continue;
+                }
+                send_observe_item(writer, &diff).await?;
+                old = new;
+            }
+            _ = writer.inner.stopped() => {
+                debug!("observer closed");
+                break;
+            }
+        }
+    }
+    Ok(())
+}
 
+async fn send_observe_item(writer: &mut ProgressWriter, item: &Bitfield) -> Result<()> {
+    use irpc::util::AsyncWriteVarintExt;
+    let item = ObserveItem::from(item);
+    let len = writer.inner.write_length_prefixed(item).await?;
+    writer.log_other_write(len);
     Ok(())
 }
 
-/// A helper struct that combines a quinn::SendStream with auxiliary information
-#[derive(Debug)]
-pub struct ResponseWriter {
-    inner: SendStream,
-    events: EventSender,
-    connection_id: u64,
+/// Helper to lazyly create an [`Event`], in the case that the event creation
+/// is expensive and we want to avoid it if the progress sender is disabled.
+pub trait LazyEvent {
+    fn call(self) -> Event;
 }
 
-impl ResponseWriter {
-    fn tracking_writer(&mut self) -> TrackingStreamWriter<TokioStreamWriter<&mut SendStream>> {
-        TrackingStreamWriter::new(TokioStreamWriter(&mut self.inner))
+impl<T> LazyEvent for T
+where
+    T: FnOnce() -> Event,
+{
+    fn call(self) -> Event {
+        self()
     }
+}
 
-    fn connection_id(&self) -> u64 {
-        self.connection_id
+impl LazyEvent for Event {
+    fn call(self) -> Event {
+        self
     }
+}
+
+/// A sender for provider events.
+#[derive(Debug, Clone)]
+pub struct EventSender(EventSenderInner);
 
-    fn request_id(&self) -> u64 {
-        self.inner.id().index()
+#[derive(Debug, Clone)]
+enum EventSenderInner {
+    Disabled,
+    Enabled(mpsc::Sender<Event>),
+}
+
+impl EventSender {
+    pub fn new(sender: Option<mpsc::Sender<Event>>) -> Self {
+        match sender {
+            Some(sender) => Self(EventSenderInner::Enabled(sender)),
+            None => Self(EventSenderInner::Disabled),
+        }
     }
 
-    fn print_stats(stats: &TransferStats) {
-        let send = stats.send.total();
-        let read = stats.read.total();
-        let total_sent_bytes = send.size;
-        let send_duration = send.stats.duration;
-        let read_duration = read.stats.duration;
-        let total_duration = stats.duration;
-        let other_duration = total_duration
-            .saturating_sub(send_duration)
-            .saturating_sub(read_duration);
-        let avg_send_size = total_sent_bytes.checked_div(send.stats.count).unwrap_or(0);
-        info!(
-            "sent {} bytes in {}s",
-            total_sent_bytes,
-            total_duration.as_secs_f64()
-        );
-        debug!(
-            "{}s sending, {}s reading, {}s other",
-            send_duration.as_secs_f64(),
-            read_duration.as_secs_f64(),
-            other_duration.as_secs_f64()
-        );
-        trace!(
-            "send_count: {} avg_send_size {}",
-            send.stats.count,
-            avg_send_size,
-        )
+    /// Send a client connected event, if the progress sender is enabled.
+    ///
+    /// This will permit the client to connect if the sender is disabled.
+    #[must_use = "permit should be checked by the caller"]
+    pub async fn authorize_client_connection(&self, connection_id: u64, node_id: NodeId) -> bool {
+        let mut wait_for_permit = None;
+        self.send(|| {
+            let (tx, rx) = oneshot::channel();
+            wait_for_permit = Some(rx);
+            Event::ClientConnected {
+                connection_id,
+                node_id,
+                permitted: tx,
+            }
+        })
+        .await;
+        if let Some(wait_for_permit) = wait_for_permit {
+            // if we have events configured, and they drop the channel, we consider that as a no!
+            // todo: this will be confusing and needs to be properly documented.
+            wait_for_permit.await.unwrap_or(false)
+        } else {
+            true
+        }
     }
 
-    async fn notify_transfer_completed(&self, hash: &Hash, stats: Box<TransferStats>) {
-        info!("transfer completed for {}", hash);
-        Self::print_stats(&stats);
-        self.events
-            .send(move || Event::TransferCompleted {
-                connection_id: self.connection_id(),
-                request_id: self.request_id(),
-                stats,
-            })
-            .await;
+    /// Send an ephemeral event, if the progress sender is enabled.
+    ///
+    /// The event will only be created if the sender is enabled.
+    fn try_send(&self, event: impl LazyEvent) {
+        match &self.0 {
+            EventSenderInner::Enabled(sender) => {
+                let value = event.call();
+                sender.try_send(value).ok();
+            }
+            EventSenderInner::Disabled => {}
+        }
     }
 
-    async fn notify_transfer_aborted(&self, stats: Option<Box<TransferStats>>) {
-        if let Some(stats) = &stats {
-            Self::print_stats(stats);
-        };
-        self.events
-            .send(move || Event::TransferAborted {
-                connection_id: self.connection_id(),
-                request_id: self.request_id(),
-                stats,
-            })
-            .await;
+    /// Send a mandatory event, if the progress sender is enabled.
+    ///
+    /// The event only be created if the sender is enabled.
+    async fn send(&self, event: impl LazyEvent) {
+        match &self.0 {
+            EventSenderInner::Enabled(sender) => {
+                let value = event.call();
+                if let Err(err) = sender.send(value).await {
+                    error!("failed to send progress event: {:?}", err);
+                }
+            }
+            EventSenderInner::Disabled => {}
+        }
     }
 }
 
-/// Status  of a send operation
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub enum SentStatus {
-    /// The requested data was sent
-    Sent,
-    /// The requested data was not found
-    NotFound,
+pub struct ProgressReader {
+    inner: RecvStream,
+    context: StreamContext,
 }
 
-/// Send a blob to the client.
-pub async fn send_blob<D: Map, W: AsyncStreamWriter>(
-    db: &D,
-    hash: Hash,
-    ranges: &RangeSpec,
-    mut writer: W,
-    events: EventSender,
-    mk_progress: impl Fn(u64) -> Event,
-) -> Result<(SentStatus, u64, SliceReaderStats)> {
-    match db.get(&hash).await? {
-        Some(entry) => {
-            let outboard = entry.outboard().await?;
-            let size = outboard.tree().size();
-            let mut file_reader = TrackingSliceReader::new(entry.data_reader().await?);
-            let mut sending_reader =
-                SendingSliceReader::new(&mut file_reader, &events, mk_progress);
-            writer.write(size.to_le_bytes().as_slice()).await?;
-            encode_ranges_validated(
-                &mut sending_reader,
-                outboard,
-                &ranges.to_chunk_ranges(),
-                writer,
-            )
-            .await
-            .map_err(|e| encode_error_to_anyhow(e, &hash))?;
+impl Deref for ProgressReader {
+    type Target = StreamContext;
 
-            Ok((SentStatus::Sent, size, file_reader.stats()))
-        }
-        _ => {
-            debug!("blob not found {}", hash.to_hex());
-            Ok((SentStatus::NotFound, 0, SliceReaderStats::default()))
-        }
+    fn deref(&self) -> &Self::Target {
+        &self.context
+    }
+}
+
+impl DerefMut for ProgressReader {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.context
+    }
+}
+
+pub struct CountingReader<R> {
+    pub inner: R,
+    pub read: u64,
+}
+
+impl<R> CountingReader<R> {
+    pub fn new(inner: R) -> Self {
+        Self { inner, read: 0 }
+    }
+
+    pub fn read(&self) -> u64 {
+        self.read
+    }
+}
+
+impl CountingReader<&mut iroh::endpoint::RecvStream> {
+    pub async fn read_to_end_as<T: DeserializeOwned>(&mut self, max_size: usize) -> io::Result<T> {
+        let data = self
+            .inner
+            .read_to_end(max_size)
+            .await
+            .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
+        let value = postcard::from_bytes(&data)
+            .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
+        self.read += data.len() as u64;
+        Ok(value)
     }
 }
 
-fn encode_error_to_anyhow(err: EncodeError, hash: &Hash) -> anyhow::Error {
-    match err {
-        EncodeError::LeafHashMismatch(x) => anyhow::Error::from(EncodeError::LeafHashMismatch(x))
-            .context(format!("hash {} offset {}", hash.to_hex(), x.to_bytes())),
-        EncodeError::ParentHashMismatch(n) => {
-            let r = n.chunk_range();
-            anyhow::Error::from(EncodeError::ParentHashMismatch(n)).context(format!(
-                "hash {} range {}..{}",
-                hash.to_hex(),
-                r.start.to_bytes(),
-                r.end.to_bytes()
-            ))
+impl<R: AsyncRead + Unpin> AsyncRead for CountingReader<R> {
+    fn poll_read(
+        self: Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &mut tokio::io::ReadBuf<'_>,
+    ) -> Poll<io::Result<()>> {
+        let this = self.get_mut();
+        let result = Pin::new(&mut this.inner).poll_read(cx, buf);
+        if let Poll::Ready(Ok(())) = result {
+            this.read += buf.filled().len() as u64;
         }
-        e => anyhow::Error::from(e).context(format!("hash {}", hash.to_hex())),
+        result
     }
 }
diff --git a/src/rpc.rs b/src/rpc.rs
deleted file mode 100644
index d17364aec..000000000
--- a/src/rpc.rs
+++ /dev/null
@@ -1,1109 +0,0 @@
-//! Provides a rpc protocol as well as a client for the protocol
-
-use std::{
-    io,
-    ops::Deref,
-    sync::{Arc, Mutex},
-};
-
-use anyhow::anyhow;
-use client::{
-    blobs::{self, BlobInfo, BlobStatus, DownloadMode, IncompleteBlobInfo, MemClient, WrapOption},
-    tags::TagInfo,
-    MemConnector,
-};
-use futures_buffered::BufferedStreamExt;
-use futures_lite::StreamExt;
-use futures_util::{FutureExt, Stream};
-use genawaiter::sync::{Co, Gen};
-use iroh::{Endpoint, NodeAddr};
-use iroh_io::AsyncSliceReader;
-use proto::{
-    blobs::{
-        AddPathRequest, AddPathResponse, AddStreamRequest, AddStreamResponse, AddStreamUpdate,
-        BatchAddPathRequest, BatchAddPathResponse, BatchAddStreamRequest, BatchAddStreamResponse,
-        BatchAddStreamUpdate, BatchCreateRequest, BatchCreateResponse, BatchCreateTempTagRequest,
-        BatchUpdate, BlobStatusRequest, BlobStatusResponse, ConsistencyCheckRequest,
-        CreateCollectionRequest, CreateCollectionResponse, DeleteRequest, DownloadResponse,
-        ExportRequest, ExportResponse, ListIncompleteRequest, ListRequest, ReadAtRequest,
-        ReadAtResponse, ValidateRequest,
-    },
-    tags::{
-        CreateRequest as TagsCreateRequest, DeleteRequest as TagDeleteRequest,
-        ListRequest as TagListRequest, RenameRequest, SetRequest as TagsSetRequest, SyncMode,
-    },
-    Request, RpcError, RpcResult, RpcService,
-};
-use quic_rpc::{
-    server::{ChannelTypes, RpcChannel, RpcServerError},
-    RpcClient, RpcServer,
-};
-use tokio_util::task::AbortOnDropHandle;
-use tracing::{debug, warn};
-
-use crate::{
-    downloader::{DownloadRequest, Downloader},
-    export::ExportProgress,
-    format::collection::Collection,
-    get::{
-        db::{DownloadProgress, GetState},
-        Stats,
-    },
-    net_protocol::{BlobDownloadRequest, Blobs, BlobsInner},
-    provider::{AddProgress, BatchAddPathProgress},
-    store::{ConsistencyCheckProgress, ImportProgress, MapEntry, ValidateProgress},
-    util::{
-        local_pool::LocalPoolHandle,
-        progress::{AsyncChannelProgressSender, ProgressSender},
-        SetTagOption,
-    },
-    BlobFormat, HashAndFormat, Tag,
-};
-pub mod client;
-pub mod proto;
-
-/// Chunk size for getting blobs over RPC
-const RPC_BLOB_GET_CHUNK_SIZE: usize = 1024 * 64;
-/// Channel cap for getting blobs over RPC
-const RPC_BLOB_GET_CHANNEL_CAP: usize = 2;
-
-impl<D: crate::store::Store> Blobs<D> {
-    /// Get a client for the blobs protocol
-    pub fn client(&self) -> &blobs::MemClient {
-        &self
-            .rpc_handler
-            .get_or_init(|| RpcHandler::new(&self.inner))
-            .client
-    }
-
-    /// Handle an RPC request
-    pub async fn handle_rpc_request<C>(
-        self,
-        msg: Request,
-        chan: RpcChannel<RpcService, C>,
-    ) -> std::result::Result<(), RpcServerError<C>>
-    where
-        C: ChannelTypes<RpcService>,
-    {
-        Handler(self.inner.clone())
-            .handle_rpc_request(msg, chan)
-            .await
-    }
-}
-
-/// This is just an internal helper so I don't have to
-/// define all the rpc methods on `self: Arc<BlobsInner<S>>`
-#[derive(Clone)]
-struct Handler<S>(Arc<BlobsInner<S>>);
-
-impl<S> Deref for Handler<S> {
-    type Target = BlobsInner<S>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-impl<D: crate::store::Store> Handler<D> {
-    fn store(&self) -> &D {
-        &self.0.store
-    }
-
-    fn rt(&self) -> &LocalPoolHandle {
-        self.0.rt()
-    }
-
-    fn endpoint(&self) -> &Endpoint {
-        &self.0.endpoint
-    }
-
-    fn downloader(&self) -> &Downloader {
-        &self.0.downloader
-    }
-
-    #[cfg(feature = "rpc")]
-    pub(crate) async fn batches(
-        &self,
-    ) -> tokio::sync::MutexGuard<'_, crate::net_protocol::BlobBatches> {
-        self.0.batches.lock().await
-    }
-
-    /// Handle an RPC request
-    pub async fn handle_rpc_request<C>(
-        self,
-        msg: Request,
-        chan: RpcChannel<RpcService, C>,
-    ) -> std::result::Result<(), RpcServerError<C>>
-    where
-        C: ChannelTypes<RpcService>,
-    {
-        match msg {
-            Request::Blobs(msg) => self.handle_blobs_request(msg, chan).await,
-            Request::Tags(msg) => self.handle_tags_request(msg, chan).await,
-        }
-    }
-
-    /// Handle a tags request
-    pub async fn handle_tags_request<C>(
-        self,
-        msg: proto::tags::Request,
-        chan: RpcChannel<proto::RpcService, C>,
-    ) -> std::result::Result<(), RpcServerError<C>>
-    where
-        C: ChannelTypes<proto::RpcService>,
-    {
-        use proto::tags::Request::*;
-        match msg {
-            Create(msg) => chan.rpc(msg, self, Self::tags_create).await,
-            Set(msg) => chan.rpc(msg, self, Self::tags_set).await,
-            DeleteTag(msg) => chan.rpc(msg, self, Self::blob_delete_tag).await,
-            ListTags(msg) => chan.server_streaming(msg, self, Self::blob_list_tags).await,
-            Rename(msg) => chan.rpc(msg, self, Self::tags_rename).await,
-        }
-    }
-
-    /// Handle a blobs request
-    pub async fn handle_blobs_request<C>(
-        self,
-        msg: proto::blobs::Request,
-        chan: RpcChannel<proto::RpcService, C>,
-    ) -> std::result::Result<(), RpcServerError<C>>
-    where
-        C: ChannelTypes<proto::RpcService>,
-    {
-        use proto::blobs::Request::*;
-        match msg {
-            List(msg) => chan.server_streaming(msg, self, Self::blob_list).await,
-            ListIncomplete(msg) => {
-                chan.server_streaming(msg, self, Self::blob_list_incomplete)
-                    .await
-            }
-            CreateCollection(msg) => chan.rpc(msg, self, Self::create_collection).await,
-            Delete(msg) => chan.rpc(msg, self, Self::blob_delete_blob).await,
-            AddPath(msg) => {
-                chan.server_streaming(msg, self, Self::blob_add_from_path)
-                    .await
-            }
-            Download(msg) => chan.server_streaming(msg, self, Self::blob_download).await,
-            Export(msg) => chan.server_streaming(msg, self, Self::blob_export).await,
-            Validate(msg) => chan.server_streaming(msg, self, Self::blob_validate).await,
-            Fsck(msg) => {
-                chan.server_streaming(msg, self, Self::blob_consistency_check)
-                    .await
-            }
-            ReadAt(msg) => chan.server_streaming(msg, self, Self::blob_read_at).await,
-            AddStream(msg) => chan.bidi_streaming(msg, self, Self::blob_add_stream).await,
-            AddStreamUpdate(_msg) => Err(RpcServerError::UnexpectedUpdateMessage),
-            BlobStatus(msg) => chan.rpc(msg, self, Self::blob_status).await,
-            BatchCreate(msg) => chan.bidi_streaming(msg, self, Self::batch_create).await,
-            BatchUpdate(_) => Err(RpcServerError::UnexpectedStartMessage),
-            BatchAddStream(msg) => chan.bidi_streaming(msg, self, Self::batch_add_stream).await,
-            BatchAddStreamUpdate(_) => Err(RpcServerError::UnexpectedStartMessage),
-            BatchAddPath(msg) => {
-                chan.server_streaming(msg, self, Self::batch_add_from_path)
-                    .await
-            }
-            BatchCreateTempTag(msg) => chan.rpc(msg, self, Self::batch_create_temp_tag).await,
-        }
-    }
-
-    async fn blob_status(self, msg: BlobStatusRequest) -> RpcResult<BlobStatusResponse> {
-        let blobs = self;
-        let entry = blobs
-            .store()
-            .get(&msg.hash)
-            .await
-            .map_err(|e| RpcError::new(&e))?;
-        Ok(BlobStatusResponse(match entry {
-            Some(entry) => {
-                if entry.is_complete() {
-                    BlobStatus::Complete {
-                        size: entry.size().value(),
-                    }
-                } else {
-                    BlobStatus::Partial { size: entry.size() }
-                }
-            }
-            None => BlobStatus::NotFound,
-        }))
-    }
-
-    async fn blob_list_impl(self, co: &Co<RpcResult<BlobInfo>>) -> io::Result<()> {
-        use bao_tree::io::fsm::Outboard;
-
-        let blobs = self;
-        let db = blobs.store();
-        for blob in db.blobs().await? {
-            let blob = blob?;
-            let Some(entry) = db.get(&blob).await? else {
-                continue;
-            };
-            let hash = entry.hash();
-            let size = entry.outboard().await?.tree().size();
-            let path = "".to_owned();
-            co.yield_(Ok(BlobInfo { hash, size, path })).await;
-        }
-        Ok(())
-    }
-
-    async fn blob_list_incomplete_impl(
-        self,
-        co: &Co<RpcResult<IncompleteBlobInfo>>,
-    ) -> io::Result<()> {
-        let blobs = self;
-        let db = blobs.store();
-        for hash in db.partial_blobs().await? {
-            let hash = hash?;
-            let Ok(Some(entry)) = db.get_mut(&hash).await else {
-                continue;
-            };
-            if entry.is_complete() {
-                continue;
-            }
-            let size = 0;
-            let expected_size = entry.size().value();
-            co.yield_(Ok(IncompleteBlobInfo {
-                hash,
-                size,
-                expected_size,
-            }))
-            .await;
-        }
-        Ok(())
-    }
-
-    fn blob_list(
-        self,
-        _msg: ListRequest,
-    ) -> impl Stream<Item = RpcResult<BlobInfo>> + Send + 'static {
-        Gen::new(|co| async move {
-            if let Err(e) = self.blob_list_impl(&co).await {
-                co.yield_(Err(RpcError::new(&e))).await;
-            }
-        })
-    }
-
-    fn blob_list_incomplete(
-        self,
-        _msg: ListIncompleteRequest,
-    ) -> impl Stream<Item = RpcResult<IncompleteBlobInfo>> + Send + 'static {
-        Gen::new(move |co| async move {
-            if let Err(e) = self.blob_list_incomplete_impl(&co).await {
-                co.yield_(Err(RpcError::new(&e))).await;
-            }
-        })
-    }
-
-    async fn blob_delete_tag(self, msg: TagDeleteRequest) -> RpcResult<()> {
-        self.store()
-            .delete_tags(msg.from, msg.to)
-            .await
-            .map_err(|e| RpcError::new(&e))?;
-        Ok(())
-    }
-
-    async fn blob_delete_blob(self, msg: DeleteRequest) -> RpcResult<()> {
-        self.store()
-            .delete(vec![msg.hash])
-            .await
-            .map_err(|e| RpcError::new(&e))?;
-        Ok(())
-    }
-
-    fn blob_list_tags(self, msg: TagListRequest) -> impl Stream<Item = TagInfo> + Send + 'static {
-        tracing::info!("blob_list_tags");
-        let blobs = self;
-        Gen::new(|co| async move {
-            let tags = blobs.store().tags(msg.from, msg.to).await.unwrap();
-            #[allow(clippy::manual_flatten)]
-            for item in tags {
-                if let Ok((name, HashAndFormat { hash, format })) = item {
-                    if (format.is_raw() && msg.raw) || (format.is_hash_seq() && msg.hash_seq) {
-                        co.yield_(TagInfo { name, hash, format }).await;
-                    }
-                }
-            }
-        })
-    }
-
-    /// Invoke validate on the database and stream out the result
-    fn blob_validate(
-        self,
-        msg: ValidateRequest,
-    ) -> impl Stream<Item = ValidateProgress> + Send + 'static {
-        let (tx, rx) = async_channel::bounded(1);
-        let tx2 = tx.clone();
-        let blobs = self;
-        tokio::task::spawn(async move {
-            if let Err(e) = blobs
-                .store()
-                .validate(msg.repair, AsyncChannelProgressSender::new(tx).boxed())
-                .await
-            {
-                tx2.send(ValidateProgress::Abort(RpcError::new(&e)))
-                    .await
-                    .ok();
-            }
-        });
-        rx
-    }
-
-    /// Invoke validate on the database and stream out the result
-    fn blob_consistency_check(
-        self,
-        msg: ConsistencyCheckRequest,
-    ) -> impl Stream<Item = ConsistencyCheckProgress> + Send + 'static {
-        let (tx, rx) = async_channel::bounded(1);
-        let tx2 = tx.clone();
-        let blobs = self;
-        tokio::task::spawn(async move {
-            if let Err(e) = blobs
-                .store()
-                .consistency_check(msg.repair, AsyncChannelProgressSender::new(tx).boxed())
-                .await
-            {
-                tx2.send(ConsistencyCheckProgress::Abort(RpcError::new(&e)))
-                    .await
-                    .ok();
-            }
-        });
-        rx
-    }
-
-    fn blob_add_from_path(self, msg: AddPathRequest) -> impl Stream<Item = AddPathResponse> {
-        // provide a little buffer so that we don't slow down the sender
-        let (tx, rx) = async_channel::bounded(32);
-        let tx2 = tx.clone();
-        let rt = self.rt().clone();
-        rt.spawn_detached(|| async move {
-            if let Err(e) = self.blob_add_from_path0(msg, tx).await {
-                tx2.send(AddProgress::Abort(RpcError::new(&*e))).await.ok();
-            }
-        });
-        rx.map(AddPathResponse)
-    }
-
-    async fn tags_rename(self, msg: RenameRequest) -> RpcResult<()> {
-        let blobs = self;
-        blobs
-            .store()
-            .rename_tag(msg.from, msg.to)
-            .await
-            .map_err(|e| RpcError::new(&e))?;
-        Ok(())
-    }
-
-    async fn tags_set(self, msg: TagsSetRequest) -> RpcResult<()> {
-        let blobs = self;
-        blobs
-            .store()
-            .set_tag(msg.name, msg.value)
-            .await
-            .map_err(|e| RpcError::new(&e))?;
-        if let SyncMode::Full = msg.sync {
-            blobs.store().sync().await.map_err(|e| RpcError::new(&e))?;
-        }
-        if let Some(batch) = msg.batch {
-            blobs
-                .batches()
-                .await
-                .remove_one(batch, &msg.value)
-                .map_err(|e| RpcError::new(&*e))?;
-        }
-        Ok(())
-    }
-
-    async fn tags_create(self, msg: TagsCreateRequest) -> RpcResult<Tag> {
-        let blobs = self;
-        let tag = blobs
-            .store()
-            .create_tag(msg.value)
-            .await
-            .map_err(|e| RpcError::new(&e))?;
-        if let SyncMode::Full = msg.sync {
-            blobs.store().sync().await.map_err(|e| RpcError::new(&e))?;
-        }
-        if let Some(batch) = msg.batch {
-            blobs
-                .batches()
-                .await
-                .remove_one(batch, &msg.value)
-                .map_err(|e| RpcError::new(&*e))?;
-        }
-        Ok(tag)
-    }
-
-    fn blob_download(self, msg: BlobDownloadRequest) -> impl Stream<Item = DownloadResponse> {
-        let (sender, receiver) = async_channel::bounded(1024);
-        let endpoint = self.endpoint().clone();
-        let progress = AsyncChannelProgressSender::new(sender);
-
-        let blobs_protocol = self.clone();
-
-        self.rt().spawn_detached(move || async move {
-            if let Err(err) = blobs_protocol
-                .download(endpoint, msg, progress.clone())
-                .await
-            {
-                progress
-                    .send(DownloadProgress::Abort(RpcError::new(&*err)))
-                    .await
-                    .ok();
-            }
-        });
-
-        receiver.map(DownloadResponse)
-    }
-
-    fn blob_export(self, msg: ExportRequest) -> impl Stream<Item = ExportResponse> {
-        let (tx, rx) = async_channel::bounded(1024);
-        let progress = AsyncChannelProgressSender::new(tx);
-        let rt = self.rt().clone();
-        rt.spawn_detached(move || async move {
-            let res = crate::export::export(
-                self.store(),
-                msg.hash,
-                msg.path,
-                msg.format,
-                msg.mode,
-                progress.clone(),
-            )
-            .await;
-            match res {
-                Ok(()) => progress.send(ExportProgress::AllDone).await.ok(),
-                Err(err) => progress
-                    .send(ExportProgress::Abort(RpcError::new(&*err)))
-                    .await
-                    .ok(),
-            };
-        });
-        rx.map(ExportResponse)
-    }
-
-    async fn blob_add_from_path0(
-        self,
-        msg: AddPathRequest,
-        progress: async_channel::Sender<AddProgress>,
-    ) -> anyhow::Result<()> {
-        use std::collections::BTreeMap;
-
-        use crate::store::ImportMode;
-
-        let blobs = self.clone();
-        let progress = AsyncChannelProgressSender::new(progress);
-        let names = Arc::new(Mutex::new(BTreeMap::new()));
-        // convert import progress to provide progress
-        let import_progress = progress.clone().with_filter_map(move |x| match x {
-            ImportProgress::Found { id, name } => {
-                names.lock().unwrap().insert(id, name);
-                None
-            }
-            ImportProgress::Size { id, size } => {
-                let name = names.lock().unwrap().remove(&id)?;
-                Some(AddProgress::Found { id, name, size })
-            }
-            ImportProgress::OutboardProgress { id, offset } => {
-                Some(AddProgress::Progress { id, offset })
-            }
-            ImportProgress::OutboardDone { hash, id } => Some(AddProgress::Done { hash, id }),
-            _ => None,
-        });
-        let AddPathRequest {
-            wrap,
-            path: root,
-            in_place,
-            tag,
-        } = msg;
-        // Check that the path is absolute and exists.
-        anyhow::ensure!(root.is_absolute(), "path must be absolute");
-        anyhow::ensure!(
-            root.exists(),
-            "trying to add missing path: {}",
-            root.display()
-        );
-
-        let import_mode = match in_place {
-            true => ImportMode::TryReference,
-            false => ImportMode::Copy,
-        };
-
-        let create_collection = match wrap {
-            WrapOption::Wrap { .. } => true,
-            WrapOption::NoWrap => root.is_dir(),
-        };
-
-        let temp_tag = if create_collection {
-            // import all files below root recursively
-            let data_sources = crate::util::fs::scan_path(root, wrap)?;
-            let blobs = self;
-
-            const IO_PARALLELISM: usize = 4;
-            let result: Vec<_> = futures_lite::stream::iter(data_sources)
-                .map(|source| {
-                    let import_progress = import_progress.clone();
-                    let blobs = blobs.clone();
-                    async move {
-                        let name = source.name().to_string();
-                        let (tag, size) = blobs
-                            .store()
-                            .import_file(
-                                source.path().to_owned(),
-                                import_mode,
-                                BlobFormat::Raw,
-                                import_progress,
-                            )
-                            .await?;
-                        let hash = *tag.hash();
-                        io::Result::Ok((name, hash, size, tag))
-                    }
-                })
-                .buffered_ordered(IO_PARALLELISM)
-                .try_collect()
-                .await?;
-
-            // create a collection
-            let (collection, _child_tags): (Collection, Vec<_>) = result
-                .into_iter()
-                .map(|(name, hash, _, tag)| ((name, hash), tag))
-                .unzip();
-
-            collection.store(blobs.store()).await?
-        } else {
-            // import a single file
-            let (tag, _size) = blobs
-                .store()
-                .import_file(root, import_mode, BlobFormat::Raw, import_progress)
-                .await?;
-            tag
-        };
-
-        let hash_and_format = temp_tag.inner();
-        let HashAndFormat { hash, format } = *hash_and_format;
-        let tag = match tag {
-            SetTagOption::Named(tag) => {
-                blobs.store().set_tag(tag.clone(), *hash_and_format).await?;
-                tag
-            }
-            SetTagOption::Auto => blobs.store().create_tag(*hash_and_format).await?,
-        };
-        progress
-            .send(AddProgress::AllDone {
-                hash,
-                format,
-                tag: tag.clone(),
-            })
-            .await?;
-        Ok(())
-    }
-
-    async fn batch_create_temp_tag(self, msg: BatchCreateTempTagRequest) -> RpcResult<()> {
-        let blobs = self;
-        let tag = blobs.store().temp_tag(msg.content);
-        blobs.batches().await.store(msg.batch, tag);
-        Ok(())
-    }
-
-    fn batch_add_stream(
-        self,
-        msg: BatchAddStreamRequest,
-        stream: impl Stream<Item = BatchAddStreamUpdate> + Send + Unpin + 'static,
-    ) -> impl Stream<Item = BatchAddStreamResponse> {
-        let (tx, rx) = async_channel::bounded(32);
-        let this = self.clone();
-
-        self.rt().spawn_detached(|| async move {
-            if let Err(err) = this.batch_add_stream0(msg, stream, tx.clone()).await {
-                tx.send(BatchAddStreamResponse::Abort(RpcError::new(&*err)))
-                    .await
-                    .ok();
-            }
-        });
-        rx
-    }
-
-    fn batch_add_from_path(
-        self,
-        msg: BatchAddPathRequest,
-    ) -> impl Stream<Item = BatchAddPathResponse> {
-        // provide a little buffer so that we don't slow down the sender
-        let (tx, rx) = async_channel::bounded(32);
-        let tx2 = tx.clone();
-        let this = self.clone();
-        self.rt().spawn_detached(|| async move {
-            if let Err(e) = this.batch_add_from_path0(msg, tx).await {
-                tx2.send(BatchAddPathProgress::Abort(RpcError::new(&*e)))
-                    .await
-                    .ok();
-            }
-        });
-        rx.map(BatchAddPathResponse)
-    }
-
-    async fn batch_add_stream0(
-        self,
-        msg: BatchAddStreamRequest,
-        stream: impl Stream<Item = BatchAddStreamUpdate> + Send + Unpin + 'static,
-        progress: async_channel::Sender<BatchAddStreamResponse>,
-    ) -> anyhow::Result<()> {
-        let blobs = self;
-        let progress = AsyncChannelProgressSender::new(progress);
-
-        let stream = stream.map(|item| match item {
-            BatchAddStreamUpdate::Chunk(chunk) => Ok(chunk),
-            BatchAddStreamUpdate::Abort => {
-                Err(io::Error::new(io::ErrorKind::Interrupted, "Remote abort"))
-            }
-        });
-
-        let import_progress = progress.clone().with_filter_map(move |x| match x {
-            ImportProgress::OutboardProgress { offset, .. } => {
-                Some(BatchAddStreamResponse::OutboardProgress { offset })
-            }
-            _ => None,
-        });
-        let (temp_tag, _len) = blobs
-            .store()
-            .import_stream(stream, msg.format, import_progress)
-            .await?;
-        let hash = temp_tag.inner().hash;
-        blobs.batches().await.store(msg.batch, temp_tag);
-        progress
-            .send(BatchAddStreamResponse::Result { hash })
-            .await?;
-        Ok(())
-    }
-
-    async fn batch_add_from_path0(
-        self,
-        msg: BatchAddPathRequest,
-        progress: async_channel::Sender<BatchAddPathProgress>,
-    ) -> anyhow::Result<()> {
-        let progress = AsyncChannelProgressSender::new(progress);
-        // convert import progress to provide progress
-        let import_progress = progress.clone().with_filter_map(move |x| match x {
-            ImportProgress::Size { size, .. } => Some(BatchAddPathProgress::Found { size }),
-            ImportProgress::OutboardProgress { offset, .. } => {
-                Some(BatchAddPathProgress::Progress { offset })
-            }
-            ImportProgress::OutboardDone { hash, .. } => Some(BatchAddPathProgress::Done { hash }),
-            _ => None,
-        });
-        let BatchAddPathRequest {
-            path: root,
-            import_mode,
-            format,
-            batch,
-        } = msg;
-        // Check that the path is absolute and exists.
-        anyhow::ensure!(root.is_absolute(), "path must be absolute");
-        anyhow::ensure!(
-            root.exists(),
-            "trying to add missing path: {}",
-            root.display()
-        );
-        let blobs = self;
-        let (tag, _) = blobs
-            .store()
-            .import_file(root, import_mode, format, import_progress)
-            .await?;
-        let hash = *tag.hash();
-        blobs.batches().await.store(batch, tag);
-
-        progress.send(BatchAddPathProgress::Done { hash }).await?;
-        Ok(())
-    }
-
-    fn blob_add_stream(
-        self,
-        msg: AddStreamRequest,
-        stream: impl Stream<Item = AddStreamUpdate> + Send + Unpin + 'static,
-    ) -> impl Stream<Item = AddStreamResponse> {
-        let (tx, rx) = async_channel::bounded(32);
-        let this = self.clone();
-
-        self.rt().spawn_detached(|| async move {
-            if let Err(err) = this.blob_add_stream0(msg, stream, tx.clone()).await {
-                tx.send(AddProgress::Abort(RpcError::new(&*err))).await.ok();
-            }
-        });
-
-        rx.map(AddStreamResponse)
-    }
-
-    async fn blob_add_stream0(
-        self,
-        msg: AddStreamRequest,
-        stream: impl Stream<Item = AddStreamUpdate> + Send + Unpin + 'static,
-        progress: async_channel::Sender<AddProgress>,
-    ) -> anyhow::Result<()> {
-        let progress = AsyncChannelProgressSender::new(progress);
-
-        let stream = stream.map(|item| match item {
-            AddStreamUpdate::Chunk(chunk) => Ok(chunk),
-            AddStreamUpdate::Abort => {
-                Err(io::Error::new(io::ErrorKind::Interrupted, "Remote abort"))
-            }
-        });
-
-        let name_cache = Arc::new(Mutex::new(None));
-        let import_progress = progress.clone().with_filter_map(move |x| match x {
-            ImportProgress::Found { id: _, name } => {
-                let _ = name_cache.lock().unwrap().insert(name);
-                None
-            }
-            ImportProgress::Size { id, size } => {
-                let name = name_cache.lock().unwrap().take()?;
-                Some(AddProgress::Found { id, name, size })
-            }
-            ImportProgress::OutboardProgress { id, offset } => {
-                Some(AddProgress::Progress { id, offset })
-            }
-            ImportProgress::OutboardDone { hash, id } => Some(AddProgress::Done { hash, id }),
-            _ => None,
-        });
-        let blobs = self;
-        let (temp_tag, _len) = blobs
-            .store()
-            .import_stream(stream, BlobFormat::Raw, import_progress)
-            .await?;
-        let hash_and_format = *temp_tag.inner();
-        let HashAndFormat { hash, format } = hash_and_format;
-        let tag = match msg.tag {
-            SetTagOption::Named(tag) => {
-                blobs.store().set_tag(tag.clone(), hash_and_format).await?;
-                tag
-            }
-            SetTagOption::Auto => blobs.store().create_tag(hash_and_format).await?,
-        };
-        progress
-            .send(AddProgress::AllDone { hash, tag, format })
-            .await?;
-        Ok(())
-    }
-
-    fn blob_read_at(
-        self,
-        req: ReadAtRequest,
-    ) -> impl Stream<Item = RpcResult<ReadAtResponse>> + Send + 'static {
-        let (tx, rx) = async_channel::bounded(RPC_BLOB_GET_CHANNEL_CAP);
-        let db = self.store().clone();
-        self.rt().spawn_detached(move || async move {
-            if let Err(err) = read_loop(req, db, tx.clone(), RPC_BLOB_GET_CHUNK_SIZE).await {
-                tx.send(RpcResult::Err(RpcError::new(&*err))).await.ok();
-            }
-        });
-
-        async fn read_loop<D: crate::store::Store>(
-            req: ReadAtRequest,
-            db: D,
-            tx: async_channel::Sender<RpcResult<ReadAtResponse>>,
-            max_chunk_size: usize,
-        ) -> anyhow::Result<()> {
-            let entry = db.get(&req.hash).await?;
-            let entry = entry.ok_or_else(|| anyhow!("Blob not found"))?;
-            let size = entry.size();
-
-            anyhow::ensure!(
-                req.offset <= size.value(),
-                "requested offset is out of range: {} > {:?}",
-                req.offset,
-                size
-            );
-
-            let len: usize = req
-                .len
-                .as_result_len(size.value() - req.offset)
-                .try_into()?;
-
-            anyhow::ensure!(
-                req.offset + len as u64 <= size.value(),
-                "requested range is out of bounds: offset: {}, len: {} > {:?}",
-                req.offset,
-                len,
-                size
-            );
-
-            tx.send(Ok(ReadAtResponse::Entry {
-                size,
-                is_complete: entry.is_complete(),
-            }))
-            .await?;
-            let mut reader = entry.data_reader().await?;
-
-            let (num_chunks, chunk_size) = if len <= max_chunk_size {
-                (1, len)
-            } else {
-                let num_chunks = len / max_chunk_size + (len % max_chunk_size != 0) as usize;
-                (num_chunks, max_chunk_size)
-            };
-
-            let mut read = 0u64;
-            for i in 0..num_chunks {
-                let chunk_size = if i == num_chunks - 1 {
-                    // last chunk might be smaller
-                    len - read as usize
-                } else {
-                    chunk_size
-                };
-                let chunk = reader.read_at(req.offset + read, chunk_size).await?;
-                let chunk_len = chunk.len();
-                if !chunk.is_empty() {
-                    tx.send(Ok(ReadAtResponse::Data { chunk })).await?;
-                }
-                if chunk_len < chunk_size {
-                    break;
-                } else {
-                    read += chunk_len as u64;
-                }
-            }
-            Ok(())
-        }
-
-        rx
-    }
-
-    fn batch_create(
-        self,
-        _: BatchCreateRequest,
-        mut updates: impl Stream<Item = BatchUpdate> + Send + Unpin + 'static,
-    ) -> impl Stream<Item = BatchCreateResponse> {
-        let blobs = self;
-        async move {
-            let batch = blobs.batches().await.create();
-            tokio::spawn(async move {
-                while let Some(item) = updates.next().await {
-                    match item {
-                        BatchUpdate::Drop(content) => {
-                            // this can not fail, since we keep the batch alive.
-                            // therefore it is safe to ignore the result.
-                            let _ = blobs.batches().await.remove_one(batch, &content);
-                        }
-                        BatchUpdate::Ping => {}
-                    }
-                }
-                blobs.batches().await.remove(batch);
-            });
-            BatchCreateResponse::Id(batch)
-        }
-        .into_stream()
-    }
-
-    async fn create_collection(
-        self,
-        req: CreateCollectionRequest,
-    ) -> RpcResult<CreateCollectionResponse> {
-        let CreateCollectionRequest {
-            collection,
-            tag,
-            tags_to_delete,
-        } = req;
-
-        let blobs = self;
-
-        let temp_tag = collection
-            .store(blobs.store())
-            .await
-            .map_err(|e| RpcError::new(&*e))?;
-        let hash_and_format = temp_tag.inner();
-        let HashAndFormat { hash, .. } = *hash_and_format;
-        let tag = match tag {
-            SetTagOption::Named(tag) => {
-                blobs
-                    .store()
-                    .set_tag(tag.clone(), *hash_and_format)
-                    .await
-                    .map_err(|e| RpcError::new(&e))?;
-                tag
-            }
-            SetTagOption::Auto => blobs
-                .store()
-                .create_tag(*hash_and_format)
-                .await
-                .map_err(|e| RpcError::new(&e))?,
-        };
-
-        for tag in tags_to_delete {
-            blobs
-                .store()
-                .delete_tags(Some(tag.clone()), Some(tag.successor()))
-                .await
-                .map_err(|e| RpcError::new(&e))?;
-        }
-
-        Ok(CreateCollectionResponse { hash, tag })
-    }
-
-    pub(crate) async fn download(
-        &self,
-        endpoint: Endpoint,
-        req: BlobDownloadRequest,
-        progress: AsyncChannelProgressSender<DownloadProgress>,
-    ) -> anyhow::Result<()> {
-        let BlobDownloadRequest {
-            hash,
-            format,
-            nodes,
-            tag,
-            mode,
-        } = req;
-        let hash_and_format = HashAndFormat { hash, format };
-        let temp_tag = self.store().temp_tag(hash_and_format);
-        let stats = match mode {
-            DownloadMode::Queued => {
-                self.download_queued(endpoint, hash_and_format, nodes, progress.clone())
-                    .await?
-            }
-            DownloadMode::Direct => {
-                self.download_direct_from_nodes(endpoint, hash_and_format, nodes, progress.clone())
-                    .await?
-            }
-        };
-
-        progress.send(DownloadProgress::AllDone(stats)).await.ok();
-        match tag {
-            SetTagOption::Named(tag) => {
-                self.store().set_tag(tag, hash_and_format).await?;
-            }
-            SetTagOption::Auto => {
-                self.store().create_tag(hash_and_format).await?;
-            }
-        }
-        drop(temp_tag);
-
-        Ok(())
-    }
-
-    async fn download_queued(
-        &self,
-        endpoint: Endpoint,
-        hash_and_format: HashAndFormat,
-        nodes: Vec<NodeAddr>,
-        progress: AsyncChannelProgressSender<DownloadProgress>,
-    ) -> anyhow::Result<Stats> {
-        /// Name used for logging when new node addresses are added from gossip.
-        const BLOB_DOWNLOAD_SOURCE_NAME: &str = "blob_download";
-
-        let mut node_ids = Vec::with_capacity(nodes.len());
-        let mut any_added = false;
-        for node in nodes {
-            node_ids.push(node.node_id);
-            if !node.is_empty() {
-                endpoint.add_node_addr_with_source(node, BLOB_DOWNLOAD_SOURCE_NAME)?;
-                any_added = true;
-            }
-        }
-        let can_download = !node_ids.is_empty() && (any_added || endpoint.discovery().is_some());
-        anyhow::ensure!(can_download, "no way to reach a node for download");
-        let req = DownloadRequest::new(hash_and_format, node_ids).progress_sender(progress);
-        let handle = self.downloader().queue(req).await;
-        let stats = handle.await?;
-        Ok(stats)
-    }
-
-    #[tracing::instrument("download_direct", skip_all, fields(hash=%hash_and_format.hash.fmt_short()))]
-    async fn download_direct_from_nodes(
-        &self,
-        endpoint: Endpoint,
-        hash_and_format: HashAndFormat,
-        nodes: Vec<NodeAddr>,
-        progress: AsyncChannelProgressSender<DownloadProgress>,
-    ) -> anyhow::Result<Stats> {
-        let mut last_err = None;
-        let mut remaining_nodes = nodes.len();
-        let mut nodes_iter = nodes.into_iter();
-        'outer: loop {
-            match crate::get::db::get_to_db_in_steps(
-                self.store().clone(),
-                hash_and_format,
-                progress.clone(),
-            )
-            .await?
-            {
-                GetState::Complete(stats) => return Ok(stats),
-                GetState::NeedsConn(needs_conn) => {
-                    let (conn, node_id) = 'inner: loop {
-                        match nodes_iter.next() {
-                            None => break 'outer,
-                            Some(node) => {
-                                remaining_nodes -= 1;
-                                let node_id = node.node_id;
-                                if node_id == endpoint.node_id() {
-                                    debug!(
-                                        ?remaining_nodes,
-                                        "skip node {} (it is the node id of ourselves)",
-                                        node_id.fmt_short()
-                                    );
-                                    continue 'inner;
-                                }
-                                match endpoint.connect(node, crate::protocol::ALPN).await {
-                                    Ok(conn) => break 'inner (conn, node_id),
-                                    Err(err) => {
-                                        debug!(
-                                            ?remaining_nodes,
-                                            "failed to connect to {}: {err}",
-                                            node_id.fmt_short()
-                                        );
-                                        continue 'inner;
-                                    }
-                                }
-                            }
-                        }
-                    };
-                    match needs_conn.proceed(conn).await {
-                        Ok(stats) => return Ok(stats),
-                        Err(err) => {
-                            warn!(
-                                ?remaining_nodes,
-                                "failed to download from {}: {err}",
-                                node_id.fmt_short()
-                            );
-                            last_err = Some(err);
-                        }
-                    }
-                }
-            }
-        }
-        match last_err {
-            Some(err) => Err(err.into()),
-            None => Err(anyhow!("No nodes to download from provided")),
-        }
-    }
-}
-
-/// An in memory rpc handler for the blobs rpc protocol
-///
-/// This struct contains both a task that handles rpc requests and a client
-/// that can be used to send rpc requests.
-///
-/// Dropping it will stop the handler task, so you need to put it somewhere
-/// where it will be kept alive. This struct will capture a copy of
-/// [`crate::net_protocol::Blobs`] and keep it alive.
-#[derive(Debug)]
-pub(crate) struct RpcHandler {
-    /// Client to hand out
-    client: MemClient,
-    /// Handler task
-    _handler: AbortOnDropHandle<()>,
-}
-
-impl Deref for RpcHandler {
-    type Target = MemClient;
-
-    fn deref(&self) -> &Self::Target {
-        &self.client
-    }
-}
-
-impl RpcHandler {
-    fn new<D: crate::store::Store>(blobs: &Arc<BlobsInner<D>>) -> Self {
-        let blobs = blobs.clone();
-        let (listener, connector) = quic_rpc::transport::flume::channel(1);
-        let listener = RpcServer::new(listener);
-        let client = RpcClient::new(connector);
-        let client = MemClient::new(client);
-        let _handler = listener.spawn_accept_loop(move |req, chan| {
-            Handler(blobs.clone()).handle_rpc_request(req, chan)
-        });
-        Self { client, _handler }
-    }
-}
diff --git a/src/rpc/client.rs b/src/rpc/client.rs
deleted file mode 100644
index a2450f496..000000000
--- a/src/rpc/client.rs
+++ /dev/null
@@ -1,25 +0,0 @@
-//! Iroh blobs and tags client
-use anyhow::Result;
-use futures_util::{Stream, StreamExt};
-use quic_rpc::transport::flume::FlumeConnector;
-
-pub mod blobs;
-pub mod tags;
-
-/// Type alias for a memory-backed client.
-pub(crate) type MemConnector =
-    FlumeConnector<crate::rpc::proto::Response, crate::rpc::proto::Request>;
-
-fn flatten<T, E1, E2>(
-    s: impl Stream<Item = Result<Result<T, E1>, E2>>,
-) -> impl Stream<Item = Result<T>>
-where
-    E1: std::error::Error + Send + Sync + 'static,
-    E2: std::error::Error + Send + Sync + 'static,
-{
-    s.map(|res| match res {
-        Ok(Ok(res)) => Ok(res),
-        Ok(Err(err)) => Err(err.into()),
-        Err(err) => Err(err.into()),
-    })
-}
diff --git a/src/rpc/client/blobs.rs b/src/rpc/client/blobs.rs
deleted file mode 100644
index f7e484c3c..000000000
--- a/src/rpc/client/blobs.rs
+++ /dev/null
@@ -1,1926 +0,0 @@
-//! API for blobs management.
-//!
-//! The main entry point is the [`Client`].
-//!
-//! ## Interacting with the local blob store
-//!
-//! ### Importing data
-//!
-//! There are several ways to import data into the local blob store:
-//!
-//! - [`add_bytes`](Client::add_bytes)
-//!   imports in memory data.
-//! - [`add_stream`](Client::add_stream)
-//!   imports data from a stream of bytes.
-//! - [`add_reader`](Client::add_reader)
-//!   imports data from an [async reader](tokio::io::AsyncRead).
-//! - [`add_from_path`](Client::add_from_path)
-//!   imports data from a file.
-//!
-//! The last method imports data from a file on the local filesystem.
-//! This is the most efficient way to import large amounts of data.
-//!
-//! ### Exporting data
-//!
-//! There are several ways to export data from the local blob store:
-//!
-//! - [`read_to_bytes`](Client::read_to_bytes) reads data into memory.
-//! - [`read`](Client::read) creates a [reader](Reader) to read data from.
-//! - [`export`](Client::export) eports data to a file on the local filesystem.
-//!
-//! ## Interacting with remote nodes
-//!
-//! - [`download`](Client::download) downloads data from a remote node.
-//!   remote node.
-//!
-//! ## Interacting with the blob store itself
-//!
-//! These are more advanced operations that are usually not needed in normal
-//! operation.
-//!
-//! - [`consistency_check`](Client::consistency_check) checks the internal
-//!   consistency of the local blob store.
-//! - [`validate`](Client::validate) validates the locally stored data against
-//!   their BLAKE3 hashes.
-//! - [`delete_blob`](Client::delete_blob) deletes a blob from the local store.
-//!
-//! ### Batch operations
-//!
-//! For complex update operations, there is a [`batch`](Client::batch) API that
-//! allows you to add multiple blobs in a single logical batch.
-//!
-//! Operations in a batch return [temporary tags](crate::util::TempTag) that
-//! protect the added data from garbage collection as long as the batch is
-//! alive.
-//!
-//! To store the data permanently, a temp tag needs to be upgraded to a
-//! permanent tag using [`persist`](crate::rpc::client::blobs::Batch::persist) or
-//! [`persist_to`](crate::rpc::client::blobs::Batch::persist_to).
-use std::{
-    future::Future,
-    io,
-    path::PathBuf,
-    pin::Pin,
-    sync::Arc,
-    task::{Context, Poll},
-};
-
-use anyhow::{anyhow, Context as _, Result};
-use bytes::Bytes;
-use futures_lite::{Stream, StreamExt};
-use futures_util::SinkExt;
-use genawaiter::sync::{Co, Gen};
-use iroh::NodeAddr;
-use portable_atomic::{AtomicU64, Ordering};
-use quic_rpc::{
-    client::{BoxStreamSync, BoxedConnector},
-    transport::boxed::BoxableConnector,
-    Connector, RpcClient,
-};
-use serde::{Deserialize, Serialize};
-use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf};
-use tokio_util::io::{ReaderStream, StreamReader};
-use tracing::warn;
-
-pub use crate::net_protocol::DownloadMode;
-use crate::{
-    export::ExportProgress as BytesExportProgress,
-    format::collection::{Collection, SimpleStore},
-    get::db::DownloadProgress as BytesDownloadProgress,
-    net_protocol::BlobDownloadRequest,
-    rpc::proto::{Request, Response, RpcService},
-    store::{BaoBlobSize, ConsistencyCheckProgress, ExportFormat, ExportMode, ValidateProgress},
-    util::SetTagOption,
-    BlobFormat, Hash, Tag,
-};
-
-mod batch;
-pub use batch::{AddDirOpts, AddFileOpts, AddReaderOpts, Batch};
-
-use super::{flatten, tags};
-use crate::rpc::proto::blobs::{
-    AddPathRequest, AddStreamRequest, AddStreamUpdate, BatchCreateRequest, BatchCreateResponse,
-    BlobStatusRequest, ConsistencyCheckRequest, CreateCollectionRequest, CreateCollectionResponse,
-    DeleteRequest, ExportRequest, ListIncompleteRequest, ListRequest, ReadAtRequest,
-    ReadAtResponse, ValidateRequest,
-};
-
-/// Iroh blobs client.
-#[derive(Debug, Clone)]
-#[repr(transparent)]
-pub struct Client<C = BoxedConnector<RpcService>> {
-    pub(crate) rpc: RpcClient<RpcService, C>,
-}
-
-/// Type alias for a memory-backed client.
-pub type MemClient = Client<crate::rpc::MemConnector>;
-
-impl<C> Client<C>
-where
-    C: Connector<RpcService>,
-{
-    /// Create a new client
-    pub fn new(rpc: RpcClient<RpcService, C>) -> Self {
-        Self { rpc }
-    }
-
-    /// Box the client to avoid having to provide the connector type.
-    pub fn boxed(&self) -> Client<BoxedConnector<RpcService>>
-    where
-        C: BoxableConnector<Response, Request>,
-    {
-        Client::new(self.rpc.clone().boxed())
-    }
-
-    /// Get a tags client.
-    pub fn tags(&self) -> tags::Client<C> {
-        tags::Client::new(self.rpc.clone())
-    }
-
-    /// Check if a blob is completely stored on the node.
-    ///
-    /// Note that this will return false for blobs that are partially stored on
-    /// the node.
-    pub async fn status(&self, hash: Hash) -> Result<BlobStatus> {
-        let status = self.rpc.rpc(BlobStatusRequest { hash }).await??;
-        Ok(status.0)
-    }
-
-    /// Check if a blob is completely stored on the node.
-    ///
-    /// This is just a convenience wrapper around `status` that returns a boolean.
-    pub async fn has(&self, hash: Hash) -> Result<bool> {
-        match self.status(hash).await {
-            Ok(BlobStatus::Complete { .. }) => Ok(true),
-            Ok(_) => Ok(false),
-            Err(err) => Err(err),
-        }
-    }
-
-    /// Create a new batch for adding data.
-    ///
-    /// A batch is a context in which temp tags are created and data is added to the node. Temp tags
-    /// are automatically deleted when the batch is dropped, leading to the data being garbage collected
-    /// unless a permanent tag is created for it.
-    pub async fn batch(&self) -> Result<Batch<C>> {
-        let (updates, mut stream) = self.rpc.bidi(BatchCreateRequest).await?;
-        let BatchCreateResponse::Id(batch) = stream.next().await.context("expected scope id")??;
-        let rpc = self.rpc.clone();
-        Ok(Batch::new(batch, rpc, updates, 1024))
-    }
-
-    /// Stream the contents of a a single blob.
-    ///
-    /// Returns a [`Reader`], which can report the size of the blob before reading it.
-    pub async fn read(&self, hash: Hash) -> Result<Reader> {
-        Reader::from_rpc_read(&self.rpc, hash).await
-    }
-
-    /// Read offset + len from a single blob.
-    ///
-    /// If `len` is `None` it will read the full blob.
-    pub async fn read_at(&self, hash: Hash, offset: u64, len: ReadAtLen) -> Result<Reader> {
-        Reader::from_rpc_read_at(&self.rpc, hash, offset, len).await
-    }
-
-    /// Read all bytes of single blob.
-    ///
-    /// This allocates a buffer for the full blob. Use only if you know that the blob you're
-    /// reading is small. If not sure, use [`Self::read`] and check the size with
-    /// [`Reader::size`] before calling [`Reader::read_to_bytes`].
-    pub async fn read_to_bytes(&self, hash: Hash) -> Result<Bytes> {
-        Reader::from_rpc_read(&self.rpc, hash)
-            .await?
-            .read_to_bytes()
-            .await
-    }
-
-    /// Read all bytes of single blob at `offset` for length `len`.
-    ///
-    /// This allocates a buffer for the full length.
-    pub async fn read_at_to_bytes(&self, hash: Hash, offset: u64, len: ReadAtLen) -> Result<Bytes> {
-        Reader::from_rpc_read_at(&self.rpc, hash, offset, len)
-            .await?
-            .read_to_bytes()
-            .await
-    }
-
-    /// Import a blob from a filesystem path.
-    ///
-    /// `path` should be an absolute path valid for the file system on which
-    /// the node runs.
-    /// If `in_place` is true, Iroh will assume that the data will not change and will share it in
-    /// place without copying to the Iroh data directory.
-    pub async fn add_from_path(
-        &self,
-        path: PathBuf,
-        in_place: bool,
-        tag: SetTagOption,
-        wrap: WrapOption,
-    ) -> Result<AddProgress> {
-        let stream = self
-            .rpc
-            .server_streaming(AddPathRequest {
-                path,
-                in_place,
-                tag,
-                wrap,
-            })
-            .await?;
-        Ok(AddProgress::new(stream))
-    }
-
-    /// Create a collection from already existing blobs.
-    ///
-    /// For automatically clearing the tags for the passed in blobs you can set
-    /// `tags_to_delete` to those tags, and they will be deleted once the collection is created.
-    pub async fn create_collection(
-        &self,
-        collection: Collection,
-        tag: SetTagOption,
-        tags_to_delete: Vec<Tag>,
-    ) -> anyhow::Result<(Hash, Tag)> {
-        let CreateCollectionResponse { hash, tag } = self
-            .rpc
-            .rpc(CreateCollectionRequest {
-                collection,
-                tag,
-                tags_to_delete,
-            })
-            .await??;
-        Ok((hash, tag))
-    }
-
-    /// Write a blob by passing an async reader.
-    pub async fn add_reader(
-        &self,
-        reader: impl AsyncRead + Unpin + Send + 'static,
-        tag: SetTagOption,
-    ) -> anyhow::Result<AddProgress> {
-        const CAP: usize = 1024 * 64; // send 64KB per request by default
-        let input = ReaderStream::with_capacity(reader, CAP);
-        self.add_stream(input, tag).await
-    }
-
-    /// Write a blob by passing a stream of bytes.
-    pub async fn add_stream(
-        &self,
-        input: impl Stream<Item = io::Result<Bytes>> + Send + Unpin + 'static,
-        tag: SetTagOption,
-    ) -> anyhow::Result<AddProgress> {
-        let (mut sink, progress) = self.rpc.bidi(AddStreamRequest { tag }).await?;
-        let mut input = input.map(|chunk| match chunk {
-            Ok(chunk) => Ok(AddStreamUpdate::Chunk(chunk)),
-            Err(err) => {
-                warn!("Abort send, reason: failed to read from source stream: {err:?}");
-                Ok(AddStreamUpdate::Abort)
-            }
-        });
-        tokio::spawn(async move {
-            if let Err(err) = sink.send_all(&mut input).await {
-                // if we get an error in send_all due to the connection being closed, this will just fail again.
-                // if we get an error due to something else (serialization or size limit), tell the remote to abort.
-                sink.send(AddStreamUpdate::Abort).await.ok();
-                warn!("Failed to send input stream to remote: {err:?}");
-            }
-        });
-
-        Ok(AddProgress::new(progress))
-    }
-
-    /// Write a blob by passing bytes.
-    pub async fn add_bytes(&self, bytes: impl Into<Bytes>) -> anyhow::Result<AddOutcome> {
-        let input = chunked_bytes_stream(bytes.into(), 1024 * 64).map(Ok);
-        self.add_stream(input, SetTagOption::Auto).await?.await
-    }
-
-    /// Write a blob by passing bytes, setting an explicit tag name.
-    pub async fn add_bytes_named(
-        &self,
-        bytes: impl Into<Bytes>,
-        name: impl Into<Tag>,
-    ) -> anyhow::Result<AddOutcome> {
-        let input = chunked_bytes_stream(bytes.into(), 1024 * 64).map(Ok);
-        self.add_stream(input, SetTagOption::Named(name.into()))
-            .await?
-            .await
-    }
-
-    /// Validate hashes on the running node.
-    ///
-    /// If `repair` is true, repair the store by removing invalid data.
-    pub async fn validate(
-        &self,
-        repair: bool,
-    ) -> Result<impl Stream<Item = Result<ValidateProgress>>> {
-        let stream = self
-            .rpc
-            .server_streaming(ValidateRequest { repair })
-            .await?;
-        Ok(stream.map(|res| res.map_err(anyhow::Error::from)))
-    }
-
-    /// Validate hashes on the running node.
-    ///
-    /// If `repair` is true, repair the store by removing invalid data.
-    pub async fn consistency_check(
-        &self,
-        repair: bool,
-    ) -> Result<impl Stream<Item = Result<ConsistencyCheckProgress>>> {
-        let stream = self
-            .rpc
-            .server_streaming(ConsistencyCheckRequest { repair })
-            .await?;
-        Ok(stream.map(|r| r.map_err(anyhow::Error::from)))
-    }
-
-    /// Download a blob from another node and add it to the local database.
-    pub async fn download(&self, hash: Hash, node: NodeAddr) -> Result<DownloadProgress> {
-        self.download_with_opts(
-            hash,
-            DownloadOptions {
-                format: BlobFormat::Raw,
-                nodes: vec![node],
-                tag: SetTagOption::Auto,
-                mode: DownloadMode::Queued,
-            },
-        )
-        .await
-    }
-
-    /// Download a hash sequence from another node and add it to the local database.
-    pub async fn download_hash_seq(&self, hash: Hash, node: NodeAddr) -> Result<DownloadProgress> {
-        self.download_with_opts(
-            hash,
-            DownloadOptions {
-                format: BlobFormat::HashSeq,
-                nodes: vec![node],
-                tag: SetTagOption::Auto,
-                mode: DownloadMode::Queued,
-            },
-        )
-        .await
-    }
-
-    /// Download a blob, with additional options.
-    pub async fn download_with_opts(
-        &self,
-        hash: Hash,
-        opts: DownloadOptions,
-    ) -> Result<DownloadProgress> {
-        let DownloadOptions {
-            format,
-            nodes,
-            tag,
-            mode,
-        } = opts;
-        let stream = self
-            .rpc
-            .server_streaming(BlobDownloadRequest {
-                hash,
-                format,
-                nodes,
-                tag,
-                mode,
-            })
-            .await?;
-        Ok(DownloadProgress::new(
-            stream.map(|res| res.map_err(anyhow::Error::from)),
-        ))
-    }
-
-    /// Export a blob from the internal blob store to a path on the node's filesystem.
-    ///
-    /// `destination` should be an writeable, absolute path on the local node's filesystem.
-    ///
-    /// If `format` is set to [`ExportFormat::Collection`], and the `hash` refers to a collection,
-    /// all children of the collection will be exported. See [`ExportFormat`] for details.
-    ///
-    /// The `mode` argument defines if the blob should be copied to the target location or moved out of
-    /// the internal store into the target location. See [`ExportMode`] for details.
-    pub async fn export(
-        &self,
-        hash: Hash,
-        destination: PathBuf,
-        format: ExportFormat,
-        mode: ExportMode,
-    ) -> Result<ExportProgress> {
-        let req = ExportRequest {
-            hash,
-            path: destination,
-            format,
-            mode,
-        };
-        let stream = self.rpc.server_streaming(req).await?;
-        Ok(ExportProgress::new(
-            stream.map(|r| r.map_err(anyhow::Error::from)),
-        ))
-    }
-
-    /// List all complete blobs.
-    pub async fn list(&self) -> Result<impl Stream<Item = Result<BlobInfo>>> {
-        let stream = self.rpc.server_streaming(ListRequest).await?;
-        Ok(flatten(stream))
-    }
-
-    /// List all incomplete (partial) blobs.
-    pub async fn list_incomplete(&self) -> Result<impl Stream<Item = Result<IncompleteBlobInfo>>> {
-        let stream = self.rpc.server_streaming(ListIncompleteRequest).await?;
-        Ok(flatten(stream))
-    }
-
-    /// Read the content of a collection.
-    pub async fn get_collection(&self, hash: Hash) -> Result<Collection> {
-        Collection::load(hash, self).await
-    }
-
-    /// List all collections.
-    pub fn list_collections(&self) -> Result<impl Stream<Item = Result<CollectionInfo>>> {
-        let this = self.clone();
-        Ok(Gen::new(|co| async move {
-            if let Err(cause) = this.list_collections_impl(&co).await {
-                co.yield_(Err(cause)).await;
-            }
-        }))
-    }
-
-    async fn list_collections_impl(&self, co: &Co<Result<CollectionInfo>>) -> Result<()> {
-        let tags = self.tags_client();
-        let mut tags = tags.list_hash_seq().await?;
-        while let Some(tag) = tags.next().await {
-            let tag = tag?;
-            if let Ok(collection) = self.get_collection(tag.hash).await {
-                let info = CollectionInfo {
-                    tag: tag.name,
-                    hash: tag.hash,
-                    total_blobs_count: Some(collection.len() as u64 + 1),
-                    total_blobs_size: Some(0),
-                };
-                co.yield_(Ok(info)).await;
-            }
-        }
-        Ok(())
-    }
-
-    /// Delete a blob.
-    ///
-    /// **Warning**: this operation deletes the blob from the local store even
-    /// if it is tagged. You should usually not do this manually, but rely on the
-    /// node to remove data that is not tagged.
-    pub async fn delete_blob(&self, hash: Hash) -> Result<()> {
-        self.rpc.rpc(DeleteRequest { hash }).await??;
-        Ok(())
-    }
-
-    fn tags_client(&self) -> tags::Client<C> {
-        tags::Client::new(self.rpc.clone())
-    }
-}
-
-impl<C> SimpleStore for Client<C>
-where
-    C: Connector<RpcService>,
-{
-    async fn load(&self, hash: Hash) -> anyhow::Result<Bytes> {
-        self.read_to_bytes(hash).await
-    }
-}
-
-/// Defines the way to read bytes.
-#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy)]
-pub enum ReadAtLen {
-    /// Reads all available bytes.
-    #[default]
-    All,
-    /// Reads exactly this many bytes, erroring out on larger or smaller.
-    Exact(u64),
-    /// Reads at most this many bytes.
-    AtMost(u64),
-}
-
-impl ReadAtLen {
-    /// todo make private again
-    pub fn as_result_len(&self, size_remaining: u64) -> u64 {
-        match self {
-            ReadAtLen::All => size_remaining,
-            ReadAtLen::Exact(len) => *len,
-            ReadAtLen::AtMost(len) => std::cmp::min(*len, size_remaining),
-        }
-    }
-}
-
-/// Whether to wrap the added data in a collection.
-#[derive(Debug, Serialize, Deserialize, Default, Clone)]
-pub enum WrapOption {
-    /// Do not wrap the file or directory.
-    #[default]
-    NoWrap,
-    /// Wrap the file or directory in a collection.
-    Wrap {
-        /// Override the filename in the wrapping collection.
-        name: Option<String>,
-    },
-}
-
-/// Status information about a blob.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub enum BlobStatus {
-    /// The blob is not stored at all.
-    NotFound,
-    /// The blob is only stored partially.
-    Partial {
-        /// The size of the currently stored partial blob.
-        size: BaoBlobSize,
-    },
-    /// The blob is stored completely.
-    Complete {
-        /// The size of the blob.
-        size: u64,
-    },
-}
-
-/// Outcome of a blob add operation.
-#[derive(Debug, Clone)]
-pub struct AddOutcome {
-    /// The hash of the blob
-    pub hash: Hash,
-    /// The format the blob
-    pub format: BlobFormat,
-    /// The size of the blob
-    pub size: u64,
-    /// The tag of the blob
-    pub tag: Tag,
-}
-
-/// Information about a stored collection.
-#[derive(Debug, Serialize, Deserialize)]
-pub struct CollectionInfo {
-    /// Tag of the collection
-    pub tag: Tag,
-
-    /// Hash of the collection
-    pub hash: Hash,
-    /// Number of children in the collection
-    ///
-    /// This is an optional field, because the data is not always available.
-    pub total_blobs_count: Option<u64>,
-    /// Total size of the raw data referred to by all links
-    ///
-    /// This is an optional field, because the data is not always available.
-    pub total_blobs_size: Option<u64>,
-}
-
-/// Information about a complete blob.
-#[derive(Debug, Serialize, Deserialize)]
-pub struct BlobInfo {
-    /// Location of the blob
-    pub path: String,
-    /// The hash of the blob
-    pub hash: Hash,
-    /// The size of the blob
-    pub size: u64,
-}
-
-/// Information about an incomplete blob.
-#[derive(Debug, Serialize, Deserialize)]
-pub struct IncompleteBlobInfo {
-    /// The size we got
-    pub size: u64,
-    /// The size we expect
-    pub expected_size: u64,
-    /// The hash of the blob
-    pub hash: Hash,
-}
-
-/// Progress stream for blob add operations.
-#[derive(derive_more::Debug)]
-pub struct AddProgress {
-    #[debug(skip)]
-    stream:
-        Pin<Box<dyn Stream<Item = Result<crate::provider::AddProgress>> + Send + Unpin + 'static>>,
-    current_total_size: Arc<AtomicU64>,
-}
-
-impl AddProgress {
-    fn new(
-        stream: (impl Stream<
-            Item = Result<impl Into<crate::provider::AddProgress>, impl Into<anyhow::Error>>,
-        > + Send
-             + Unpin
-             + 'static),
-    ) -> Self {
-        let current_total_size = Arc::new(AtomicU64::new(0));
-        let total_size = current_total_size.clone();
-        let stream = stream.map(move |item| match item {
-            Ok(item) => {
-                let item = item.into();
-                if let crate::provider::AddProgress::Found { size, .. } = &item {
-                    total_size.fetch_add(*size, Ordering::Relaxed);
-                }
-                Ok(item)
-            }
-            Err(err) => Err(err.into()),
-        });
-        Self {
-            stream: Box::pin(stream),
-            current_total_size,
-        }
-    }
-    /// Finish writing the stream, ignoring all intermediate progress events.
-    ///
-    /// Returns a [`AddOutcome`] which contains a tag, format, hash and a size.
-    /// When importing a single blob, this is the hash and size of that blob.
-    /// When importing a collection, the hash is the hash of the collection and the size
-    /// is the total size of all imported blobs (but excluding the size of the collection blob
-    /// itself).
-    pub async fn finish(self) -> Result<AddOutcome> {
-        self.await
-    }
-}
-
-impl Stream for AddProgress {
-    type Item = Result<crate::provider::AddProgress>;
-    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        Pin::new(&mut self.stream).poll_next(cx)
-    }
-}
-
-impl Future for AddProgress {
-    type Output = Result<AddOutcome>;
-
-    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
-        loop {
-            match Pin::new(&mut self.stream).poll_next(cx) {
-                Poll::Pending => return Poll::Pending,
-                Poll::Ready(None) => {
-                    return Poll::Ready(Err(anyhow!("Response stream ended prematurely")))
-                }
-                Poll::Ready(Some(Err(err))) => return Poll::Ready(Err(err)),
-                Poll::Ready(Some(Ok(msg))) => match msg {
-                    crate::provider::AddProgress::AllDone { hash, format, tag } => {
-                        let outcome = AddOutcome {
-                            hash,
-                            format,
-                            tag,
-                            size: self.current_total_size.load(Ordering::Relaxed),
-                        };
-                        return Poll::Ready(Ok(outcome));
-                    }
-                    crate::provider::AddProgress::Abort(err) => {
-                        return Poll::Ready(Err(err.into()));
-                    }
-                    _ => {}
-                },
-            }
-        }
-    }
-}
-
-/// Outcome of a blob download operation.
-#[derive(Debug, Clone)]
-pub struct DownloadOutcome {
-    /// The size of the data we already had locally
-    pub local_size: u64,
-    /// The size of the data we downloaded from the network
-    pub downloaded_size: u64,
-    /// Statistics about the download
-    pub stats: crate::get::Stats,
-}
-
-/// Progress stream for blob download operations.
-#[derive(derive_more::Debug)]
-pub struct DownloadProgress {
-    #[debug(skip)]
-    stream: Pin<Box<dyn Stream<Item = Result<BytesDownloadProgress>> + Send + Unpin + 'static>>,
-    current_local_size: Arc<AtomicU64>,
-    current_network_size: Arc<AtomicU64>,
-}
-
-impl DownloadProgress {
-    /// Create a [`DownloadProgress`] that can help you easily poll the [`BytesDownloadProgress`] stream from your download until it is finished or errors.
-    pub fn new(
-        stream: (impl Stream<Item = Result<impl Into<BytesDownloadProgress>, impl Into<anyhow::Error>>>
-             + Send
-             + Unpin
-             + 'static),
-    ) -> Self {
-        let current_local_size = Arc::new(AtomicU64::new(0));
-        let current_network_size = Arc::new(AtomicU64::new(0));
-
-        let local_size = current_local_size.clone();
-        let network_size = current_network_size.clone();
-
-        let stream = stream.map(move |item| match item {
-            Ok(item) => {
-                let item = item.into();
-                match &item {
-                    BytesDownloadProgress::FoundLocal { size, .. } => {
-                        local_size.fetch_add(size.value(), Ordering::Relaxed);
-                    }
-                    BytesDownloadProgress::Found { size, .. } => {
-                        network_size.fetch_add(*size, Ordering::Relaxed);
-                    }
-                    _ => {}
-                }
-
-                Ok(item)
-            }
-            Err(err) => Err(err.into()),
-        });
-        Self {
-            stream: Box::pin(stream),
-            current_local_size,
-            current_network_size,
-        }
-    }
-
-    /// Finish writing the stream, ignoring all intermediate progress events.
-    ///
-    /// Returns a [`DownloadOutcome`] which contains the size of the content we downloaded and the size of the content we already had locally.
-    /// When importing a single blob, this is the size of that blob.
-    /// When importing a collection, this is the total size of all imported blobs (but excluding the size of the collection blob itself).
-    pub async fn finish(self) -> Result<DownloadOutcome> {
-        self.await
-    }
-}
-
-impl Stream for DownloadProgress {
-    type Item = Result<BytesDownloadProgress>;
-    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        Pin::new(&mut self.stream).poll_next(cx)
-    }
-}
-
-impl Future for DownloadProgress {
-    type Output = Result<DownloadOutcome>;
-
-    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
-        loop {
-            match Pin::new(&mut self.stream).poll_next(cx) {
-                Poll::Pending => return Poll::Pending,
-                Poll::Ready(None) => {
-                    return Poll::Ready(Err(anyhow!("Response stream ended prematurely")))
-                }
-                Poll::Ready(Some(Err(err))) => return Poll::Ready(Err(err)),
-                Poll::Ready(Some(Ok(msg))) => match msg {
-                    BytesDownloadProgress::AllDone(stats) => {
-                        let outcome = DownloadOutcome {
-                            local_size: self.current_local_size.load(Ordering::Relaxed),
-                            downloaded_size: self.current_network_size.load(Ordering::Relaxed),
-                            stats,
-                        };
-                        return Poll::Ready(Ok(outcome));
-                    }
-                    BytesDownloadProgress::Abort(err) => {
-                        return Poll::Ready(Err(err.into()));
-                    }
-                    _ => {}
-                },
-            }
-        }
-    }
-}
-
-/// Outcome of a blob export operation.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct ExportOutcome {
-    /// The total size of the exported data.
-    total_size: u64,
-}
-
-/// Progress stream for blob export operations.
-#[derive(derive_more::Debug)]
-pub struct ExportProgress {
-    #[debug(skip)]
-    stream: Pin<Box<dyn Stream<Item = Result<BytesExportProgress>> + Send + Unpin + 'static>>,
-    current_total_size: Arc<AtomicU64>,
-}
-
-impl ExportProgress {
-    /// Create a [`ExportProgress`] that can help you easily poll the [`BytesExportProgress`] stream from your
-    /// download until it is finished or errors.
-    pub fn new(
-        stream: (impl Stream<Item = Result<impl Into<BytesExportProgress>, impl Into<anyhow::Error>>>
-             + Send
-             + Unpin
-             + 'static),
-    ) -> Self {
-        let current_total_size = Arc::new(AtomicU64::new(0));
-        let total_size = current_total_size.clone();
-        let stream = stream.map(move |item| match item {
-            Ok(item) => {
-                let item = item.into();
-                if let BytesExportProgress::Found { size, .. } = &item {
-                    let size = size.value();
-                    total_size.fetch_add(size, Ordering::Relaxed);
-                }
-
-                Ok(item)
-            }
-            Err(err) => Err(err.into()),
-        });
-        Self {
-            stream: Box::pin(stream),
-            current_total_size,
-        }
-    }
-
-    /// Finish writing the stream, ignoring all intermediate progress events.
-    ///
-    /// Returns a [`ExportOutcome`] which contains the size of the content we exported.
-    pub async fn finish(self) -> Result<ExportOutcome> {
-        self.await
-    }
-}
-
-impl Stream for ExportProgress {
-    type Item = Result<BytesExportProgress>;
-    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        Pin::new(&mut self.stream).poll_next(cx)
-    }
-}
-
-impl Future for ExportProgress {
-    type Output = Result<ExportOutcome>;
-
-    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
-        loop {
-            match Pin::new(&mut self.stream).poll_next(cx) {
-                Poll::Pending => return Poll::Pending,
-                Poll::Ready(None) => {
-                    return Poll::Ready(Err(anyhow!("Response stream ended prematurely")))
-                }
-                Poll::Ready(Some(Err(err))) => return Poll::Ready(Err(err)),
-                Poll::Ready(Some(Ok(msg))) => match msg {
-                    BytesExportProgress::AllDone => {
-                        let outcome = ExportOutcome {
-                            total_size: self.current_total_size.load(Ordering::Relaxed),
-                        };
-                        return Poll::Ready(Ok(outcome));
-                    }
-                    BytesExportProgress::Abort(err) => {
-                        return Poll::Ready(Err(err.into()));
-                    }
-                    _ => {}
-                },
-            }
-        }
-    }
-}
-
-/// Data reader for a single blob.
-///
-/// Implements [`AsyncRead`].
-#[derive(derive_more::Debug)]
-pub struct Reader {
-    size: u64,
-    response_size: u64,
-    is_complete: bool,
-    #[debug("StreamReader")]
-    stream: tokio_util::io::StreamReader<BoxStreamSync<'static, io::Result<Bytes>>, Bytes>,
-}
-
-impl Reader {
-    fn new(
-        size: u64,
-        response_size: u64,
-        is_complete: bool,
-        stream: BoxStreamSync<'static, io::Result<Bytes>>,
-    ) -> Self {
-        Self {
-            size,
-            response_size,
-            is_complete,
-            stream: StreamReader::new(stream),
-        }
-    }
-
-    /// todo make private again
-    pub async fn from_rpc_read<C>(
-        rpc: &RpcClient<RpcService, C>,
-        hash: Hash,
-    ) -> anyhow::Result<Self>
-    where
-        C: Connector<RpcService>,
-    {
-        Self::from_rpc_read_at(rpc, hash, 0, ReadAtLen::All).await
-    }
-
-    async fn from_rpc_read_at<C>(
-        rpc: &RpcClient<RpcService, C>,
-        hash: Hash,
-        offset: u64,
-        len: ReadAtLen,
-    ) -> anyhow::Result<Self>
-    where
-        C: Connector<RpcService>,
-    {
-        let stream = rpc
-            .server_streaming(ReadAtRequest { hash, offset, len })
-            .await?;
-        let mut stream = flatten(stream);
-
-        let (size, is_complete) = match stream.next().await {
-            Some(Ok(ReadAtResponse::Entry { size, is_complete })) => (size, is_complete),
-            Some(Err(err)) => return Err(err),
-            Some(Ok(_)) => return Err(anyhow!("Expected header frame, but got data frame")),
-            None => return Err(anyhow!("Expected header frame, but RPC stream was dropped")),
-        };
-
-        let stream = stream.map(|item| match item {
-            Ok(ReadAtResponse::Data { chunk }) => Ok(chunk),
-            Ok(_) => Err(io::Error::new(io::ErrorKind::Other, "Expected data frame")),
-            Err(err) => Err(io::Error::new(io::ErrorKind::Other, format!("{err}"))),
-        });
-        let len = len.as_result_len(size.value() - offset);
-        Ok(Self::new(size.value(), len, is_complete, Box::pin(stream)))
-    }
-
-    /// Total size of this blob.
-    pub fn size(&self) -> u64 {
-        self.size
-    }
-
-    /// Whether this blob has been downloaded completely.
-    ///
-    /// Returns false for partial blobs for which some chunks are missing.
-    pub fn is_complete(&self) -> bool {
-        self.is_complete
-    }
-
-    /// Read all bytes of the blob.
-    pub async fn read_to_bytes(&mut self) -> anyhow::Result<Bytes> {
-        let mut buf = Vec::with_capacity(self.response_size as usize);
-        self.read_to_end(&mut buf).await?;
-        Ok(buf.into())
-    }
-}
-
-impl AsyncRead for Reader {
-    fn poll_read(
-        mut self: Pin<&mut Self>,
-        cx: &mut Context<'_>,
-        buf: &mut ReadBuf<'_>,
-    ) -> Poll<io::Result<()>> {
-        Pin::new(&mut self.stream).poll_read(cx, buf)
-    }
-}
-
-impl Stream for Reader {
-    type Item = io::Result<Bytes>;
-
-    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        Pin::new(&mut self.stream).get_pin_mut().poll_next(cx)
-    }
-
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        self.stream.get_ref().size_hint()
-    }
-}
-
-/// Options to configure a download request.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct DownloadOptions {
-    /// The format of the data to download.
-    pub format: BlobFormat,
-    /// Source nodes to download from.
-    ///
-    /// If set to more than a single node, they will all be tried. If `mode` is set to
-    /// [`DownloadMode::Direct`], they will be tried sequentially until a download succeeds.
-    /// If `mode` is set to [`DownloadMode::Queued`], the nodes may be dialed in parallel,
-    /// if the concurrency limits permit.
-    pub nodes: Vec<NodeAddr>,
-    /// Optional tag to tag the data with.
-    pub tag: SetTagOption,
-    /// Whether to directly start the download or add it to the download queue.
-    pub mode: DownloadMode,
-}
-
-fn chunked_bytes_stream(mut b: Bytes, c: usize) -> impl Stream<Item = Bytes> {
-    futures_lite::stream::iter(std::iter::from_fn(move || {
-        Some(b.split_to(b.len().min(c))).filter(|x| !x.is_empty())
-    }))
-}
-
-#[cfg(test)]
-mod tests {
-    use std::{path::Path, time::Duration};
-
-    use iroh::{test_utils::DnsPkarrServer, NodeId, RelayMode, SecretKey};
-    use node::Node;
-    use rand::RngCore;
-    use testresult::TestResult;
-    use tokio::{io::AsyncWriteExt, sync::mpsc};
-    use tracing_test::traced_test;
-
-    use super::*;
-    use crate::{hashseq::HashSeq, ticket::BlobTicket};
-
-    mod node {
-        //! An iroh node that just has the blobs transport
-        use std::path::Path;
-
-        use iroh::{protocol::Router, Endpoint, NodeAddr, NodeId};
-        use tokio_util::task::AbortOnDropHandle;
-
-        use super::RpcService;
-        use crate::{
-            net_protocol::Blobs,
-            provider::{CustomEventSender, EventSender},
-            rpc::client::{blobs, tags},
-        };
-
-        type RpcClient = quic_rpc::RpcClient<RpcService>;
-
-        /// An iroh node that just has the blobs transport
-        #[derive(Debug)]
-        pub struct Node {
-            router: iroh::protocol::Router,
-            client: RpcClient,
-            _rpc_task: AbortOnDropHandle<()>,
-        }
-
-        /// An iroh node builder
-        #[derive(Debug)]
-        pub struct Builder<S> {
-            store: S,
-            events: EventSender,
-            endpoint: Option<iroh::endpoint::Builder>,
-        }
-
-        impl<S: crate::store::Store> Builder<S> {
-            /// Sets the event sender
-            pub fn blobs_events(self, events: impl CustomEventSender) -> Self {
-                Self {
-                    events: events.into(),
-                    ..self
-                }
-            }
-
-            /// Set an endpoint builder
-            pub fn endpoint(self, endpoint: iroh::endpoint::Builder) -> Self {
-                Self {
-                    endpoint: Some(endpoint),
-                    ..self
-                }
-            }
-
-            /// Spawns the node
-            pub async fn spawn(self) -> anyhow::Result<Node> {
-                let store = self.store;
-                let events = self.events;
-                let endpoint = self
-                    .endpoint
-                    .unwrap_or_else(|| Endpoint::builder().discovery_n0())
-                    .bind()
-                    .await?;
-                let mut router = Router::builder(endpoint.clone());
-
-                // Setup blobs
-                let blobs = Blobs::builder(store.clone())
-                    .events(events)
-                    .build(&endpoint);
-                router = router.accept(crate::ALPN, blobs.clone());
-
-                // Build the router
-                let router = router.spawn();
-
-                // Setup RPC
-                let (internal_rpc, controller) = quic_rpc::transport::flume::channel(32);
-                let internal_rpc = quic_rpc::RpcServer::new(internal_rpc).boxed();
-                let _rpc_task = internal_rpc.spawn_accept_loop(move |msg, chan| {
-                    blobs.clone().handle_rpc_request(msg, chan)
-                });
-                let client = quic_rpc::RpcClient::new(controller).boxed();
-                Ok(Node {
-                    router,
-                    client,
-                    _rpc_task,
-                })
-            }
-        }
-
-        impl Node {
-            /// Creates a new node with memory storage
-            pub fn memory() -> Builder<crate::store::mem::Store> {
-                Builder {
-                    store: crate::store::mem::Store::new(),
-                    events: Default::default(),
-                    endpoint: None,
-                }
-            }
-
-            /// Creates a new node with persistent storage
-            pub async fn persistent(
-                path: impl AsRef<Path>,
-            ) -> anyhow::Result<Builder<crate::store::fs::Store>> {
-                Ok(Builder {
-                    store: crate::store::fs::Store::load(path).await?,
-                    events: Default::default(),
-                    endpoint: None,
-                })
-            }
-
-            /// Returns the node id
-            pub fn node_id(&self) -> NodeId {
-                self.router.endpoint().node_id()
-            }
-
-            /// Returns the node address
-            pub async fn node_addr(&self) -> anyhow::Result<NodeAddr> {
-                self.router.endpoint().node_addr().await
-            }
-
-            /// Shuts down the node
-            pub async fn shutdown(self) -> anyhow::Result<()> {
-                self.router.shutdown().await
-            }
-
-            /// Returns an in-memory blobs client
-            pub fn blobs(&self) -> blobs::Client {
-                blobs::Client::new(self.client.clone())
-            }
-
-            /// Returns an in-memory tags client
-            pub fn tags(&self) -> tags::Client {
-                tags::Client::new(self.client.clone())
-            }
-        }
-    }
-
-    #[tokio::test]
-    #[traced_test]
-    async fn test_blob_create_collection() -> Result<()> {
-        let node = node::Node::memory().spawn().await?;
-
-        // create temp file
-        let temp_dir = tempfile::tempdir().context("tempdir")?;
-
-        let in_root = temp_dir.path().join("in");
-        tokio::fs::create_dir_all(in_root.clone())
-            .await
-            .context("create dir all")?;
-
-        let mut paths = Vec::new();
-        for i in 0..5 {
-            let path = in_root.join(format!("test-{i}"));
-            let size = 100;
-            let mut buf = vec![0u8; size];
-            rand::thread_rng().fill_bytes(&mut buf);
-            let mut file = tokio::fs::File::create(path.clone())
-                .await
-                .context("create file")?;
-            file.write_all(&buf.clone()).await.context("write_all")?;
-            file.flush().await.context("flush")?;
-            paths.push(path);
-        }
-
-        let blobs = node.blobs();
-
-        let mut collection = Collection::default();
-        let mut tags = Vec::new();
-        // import files
-        for path in &paths {
-            let import_outcome = blobs
-                .add_from_path(
-                    path.to_path_buf(),
-                    false,
-                    SetTagOption::Auto,
-                    WrapOption::NoWrap,
-                )
-                .await
-                .context("import file")?
-                .finish()
-                .await
-                .context("import finish")?;
-
-            collection.push(
-                path.file_name().unwrap().to_str().unwrap().to_string(),
-                import_outcome.hash,
-            );
-            tags.push(import_outcome.tag);
-        }
-
-        let (hash, tag) = blobs
-            .create_collection(collection, SetTagOption::Auto, tags)
-            .await?;
-
-        let collections: Vec<_> = blobs.list_collections()?.try_collect().await?;
-
-        assert_eq!(collections.len(), 1);
-        {
-            let CollectionInfo {
-                tag,
-                hash,
-                total_blobs_count,
-                ..
-            } = &collections[0];
-            assert_eq!(tag, tag);
-            assert_eq!(hash, hash);
-            // 5 blobs + 1 meta
-            assert_eq!(total_blobs_count, &Some(5 + 1));
-        }
-
-        // check that "temp" tags have been deleted
-        let tags: Vec<_> = node.tags().list().await?.try_collect().await?;
-        assert_eq!(tags.len(), 1);
-        assert_eq!(tags[0].hash, hash);
-        assert_eq!(tags[0].name, tag);
-        assert_eq!(tags[0].format, BlobFormat::HashSeq);
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    #[traced_test]
-    async fn test_blob_read_at() -> Result<()> {
-        let node = node::Node::memory().spawn().await?;
-
-        // create temp file
-        let temp_dir = tempfile::tempdir().context("tempdir")?;
-
-        let in_root = temp_dir.path().join("in");
-        tokio::fs::create_dir_all(in_root.clone())
-            .await
-            .context("create dir all")?;
-
-        let path = in_root.join("test-blob");
-        let size = 1024 * 128;
-        let buf: Vec<u8> = (0..size).map(|i| i as u8).collect();
-        let mut file = tokio::fs::File::create(path.clone())
-            .await
-            .context("create file")?;
-        file.write_all(&buf.clone()).await.context("write_all")?;
-        file.flush().await.context("flush")?;
-
-        let blobs = node.blobs();
-
-        let import_outcome = blobs
-            .add_from_path(
-                path.to_path_buf(),
-                false,
-                SetTagOption::Auto,
-                WrapOption::NoWrap,
-            )
-            .await
-            .context("import file")?
-            .finish()
-            .await
-            .context("import finish")?;
-
-        let hash = import_outcome.hash;
-
-        // Read everything
-        let res = blobs.read_to_bytes(hash).await?;
-        assert_eq!(&res, &buf[..]);
-
-        // Read at smaller than blob_get_chunk_size
-        let res = blobs
-            .read_at_to_bytes(hash, 0, ReadAtLen::Exact(100))
-            .await?;
-        assert_eq!(res.len(), 100);
-        assert_eq!(&res[..], &buf[0..100]);
-
-        let res = blobs
-            .read_at_to_bytes(hash, 20, ReadAtLen::Exact(120))
-            .await?;
-        assert_eq!(res.len(), 120);
-        assert_eq!(&res[..], &buf[20..140]);
-
-        // Read at equal to blob_get_chunk_size
-        let res = blobs
-            .read_at_to_bytes(hash, 0, ReadAtLen::Exact(1024 * 64))
-            .await?;
-        assert_eq!(res.len(), 1024 * 64);
-        assert_eq!(&res[..], &buf[0..1024 * 64]);
-
-        let res = blobs
-            .read_at_to_bytes(hash, 20, ReadAtLen::Exact(1024 * 64))
-            .await?;
-        assert_eq!(res.len(), 1024 * 64);
-        assert_eq!(&res[..], &buf[20..(20 + 1024 * 64)]);
-
-        // Read at larger than blob_get_chunk_size
-        let res = blobs
-            .read_at_to_bytes(hash, 0, ReadAtLen::Exact(10 + 1024 * 64))
-            .await?;
-        assert_eq!(res.len(), 10 + 1024 * 64);
-        assert_eq!(&res[..], &buf[0..(10 + 1024 * 64)]);
-
-        let res = blobs
-            .read_at_to_bytes(hash, 20, ReadAtLen::Exact(10 + 1024 * 64))
-            .await?;
-        assert_eq!(res.len(), 10 + 1024 * 64);
-        assert_eq!(&res[..], &buf[20..(20 + 10 + 1024 * 64)]);
-
-        // full length
-        let res = blobs.read_at_to_bytes(hash, 20, ReadAtLen::All).await?;
-        assert_eq!(res.len(), 1024 * 128 - 20);
-        assert_eq!(&res[..], &buf[20..]);
-
-        // size should be total
-        let reader = blobs.read_at(hash, 0, ReadAtLen::Exact(20)).await?;
-        assert_eq!(reader.size(), 1024 * 128);
-        assert_eq!(reader.response_size, 20);
-
-        // last chunk - exact
-        let res = blobs
-            .read_at_to_bytes(hash, 1024 * 127, ReadAtLen::Exact(1024))
-            .await?;
-        assert_eq!(res.len(), 1024);
-        assert_eq!(res, &buf[1024 * 127..]);
-
-        // last chunk - open
-        let res = blobs
-            .read_at_to_bytes(hash, 1024 * 127, ReadAtLen::All)
-            .await?;
-        assert_eq!(res.len(), 1024);
-        assert_eq!(res, &buf[1024 * 127..]);
-
-        // last chunk - larger
-        let mut res = blobs
-            .read_at(hash, 1024 * 127, ReadAtLen::AtMost(2048))
-            .await?;
-        assert_eq!(res.size, 1024 * 128);
-        assert_eq!(res.response_size, 1024);
-        let res = res.read_to_bytes().await?;
-        assert_eq!(res.len(), 1024);
-        assert_eq!(res, &buf[1024 * 127..]);
-
-        // out of bounds - too long
-        let res = blobs
-            .read_at(hash, 0, ReadAtLen::Exact(1024 * 128 + 1))
-            .await;
-        let err = res.unwrap_err();
-        assert!(err.to_string().contains("out of bound"));
-
-        // out of bounds - offset larger than blob
-        let res = blobs.read_at(hash, 1024 * 128 + 1, ReadAtLen::All).await;
-        let err = res.unwrap_err();
-        assert!(err.to_string().contains("out of range"));
-
-        // out of bounds - offset + length too large
-        let res = blobs
-            .read_at(hash, 1024 * 127, ReadAtLen::Exact(1025))
-            .await;
-        let err = res.unwrap_err();
-        assert!(err.to_string().contains("out of bound"));
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    #[traced_test]
-    async fn test_blob_get_collection() -> Result<()> {
-        let node = node::Node::memory().spawn().await?;
-
-        // create temp file
-        let temp_dir = tempfile::tempdir().context("tempdir")?;
-
-        let in_root = temp_dir.path().join("in");
-        tokio::fs::create_dir_all(in_root.clone())
-            .await
-            .context("create dir all")?;
-
-        let mut paths = Vec::new();
-        for i in 0..5 {
-            let path = in_root.join(format!("test-{i}"));
-            let size = 100;
-            let mut buf = vec![0u8; size];
-            rand::thread_rng().fill_bytes(&mut buf);
-            let mut file = tokio::fs::File::create(path.clone())
-                .await
-                .context("create file")?;
-            file.write_all(&buf.clone()).await.context("write_all")?;
-            file.flush().await.context("flush")?;
-            paths.push(path);
-        }
-
-        let blobs = node.blobs();
-
-        let mut collection = Collection::default();
-        let mut tags = Vec::new();
-        // import files
-        for path in &paths {
-            let import_outcome = blobs
-                .add_from_path(
-                    path.to_path_buf(),
-                    false,
-                    SetTagOption::Auto,
-                    WrapOption::NoWrap,
-                )
-                .await
-                .context("import file")?
-                .finish()
-                .await
-                .context("import finish")?;
-
-            collection.push(
-                path.file_name().unwrap().to_str().unwrap().to_string(),
-                import_outcome.hash,
-            );
-            tags.push(import_outcome.tag);
-        }
-
-        let (hash, _tag) = blobs
-            .create_collection(collection, SetTagOption::Auto, tags)
-            .await?;
-
-        let collection = blobs.get_collection(hash).await?;
-
-        // 5 blobs
-        assert_eq!(collection.len(), 5);
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    #[traced_test]
-    async fn test_blob_share() -> Result<()> {
-        let node = node::Node::memory().spawn().await?;
-
-        // create temp file
-        let temp_dir = tempfile::tempdir().context("tempdir")?;
-
-        let in_root = temp_dir.path().join("in");
-        tokio::fs::create_dir_all(in_root.clone())
-            .await
-            .context("create dir all")?;
-
-        let path = in_root.join("test-blob");
-        let size = 1024 * 128;
-        let buf: Vec<u8> = (0..size).map(|i| i as u8).collect();
-        let mut file = tokio::fs::File::create(path.clone())
-            .await
-            .context("create file")?;
-        file.write_all(&buf.clone()).await.context("write_all")?;
-        file.flush().await.context("flush")?;
-
-        let blobs = node.blobs();
-
-        let import_outcome = blobs
-            .add_from_path(
-                path.to_path_buf(),
-                false,
-                SetTagOption::Auto,
-                WrapOption::NoWrap,
-            )
-            .await
-            .context("import file")?
-            .finish()
-            .await
-            .context("import finish")?;
-
-        // let ticket = blobs
-        //     .share(import_outcome.hash, BlobFormat::Raw, Default::default())
-        //     .await?;
-        // assert_eq!(ticket.hash(), import_outcome.hash);
-
-        let status = blobs.status(import_outcome.hash).await?;
-        assert_eq!(status, BlobStatus::Complete { size });
-
-        Ok(())
-    }
-
-    #[derive(Debug, Clone)]
-    struct BlobEvents {
-        sender: mpsc::Sender<crate::provider::Event>,
-    }
-
-    impl BlobEvents {
-        fn new(cap: usize) -> (Self, mpsc::Receiver<crate::provider::Event>) {
-            let (s, r) = mpsc::channel(cap);
-            (Self { sender: s }, r)
-        }
-    }
-
-    impl crate::provider::CustomEventSender for BlobEvents {
-        fn send(&self, event: crate::provider::Event) -> futures_lite::future::Boxed<()> {
-            let sender = self.sender.clone();
-            Box::pin(async move {
-                sender.send(event).await.ok();
-            })
-        }
-
-        fn try_send(&self, event: crate::provider::Event) {
-            self.sender.try_send(event).ok();
-        }
-    }
-
-    #[tokio::test]
-    #[traced_test]
-    async fn test_blob_provide_events() -> Result<()> {
-        let (node1_events, mut node1_events_r) = BlobEvents::new(16);
-        let node1 = node::Node::memory()
-            .blobs_events(node1_events)
-            .spawn()
-            .await?;
-
-        let (node2_events, mut node2_events_r) = BlobEvents::new(16);
-        let node2 = node::Node::memory()
-            .blobs_events(node2_events)
-            .spawn()
-            .await?;
-
-        let import_outcome = node1.blobs().add_bytes(&b"hello world"[..]).await?;
-
-        // Download in node2
-        let node1_addr = node1.node_addr().await?;
-        let res = node2
-            .blobs()
-            .download(import_outcome.hash, node1_addr)
-            .await?
-            .await?;
-        dbg!(&res);
-        assert_eq!(res.local_size, 0);
-        assert_eq!(res.downloaded_size, 11);
-
-        node1.shutdown().await?;
-        node2.shutdown().await?;
-
-        let mut ev1 = Vec::new();
-        while let Some(ev) = node1_events_r.recv().await {
-            ev1.push(ev);
-        }
-        // assert_eq!(ev1.len(), 3);
-        assert!(matches!(
-            ev1[0],
-            crate::provider::Event::ClientConnected { .. }
-        ));
-        assert!(matches!(
-            ev1[1],
-            crate::provider::Event::GetRequestReceived { .. }
-        ));
-        assert!(matches!(
-            ev1[2],
-            crate::provider::Event::TransferProgress { .. }
-        ));
-        assert!(matches!(
-            ev1[3],
-            crate::provider::Event::TransferCompleted { .. }
-        ));
-        dbg!(&ev1);
-
-        let mut ev2 = Vec::new();
-        while let Some(ev) = node2_events_r.recv().await {
-            ev2.push(ev);
-        }
-
-        // Node 2 did not provide anything
-        assert!(ev2.is_empty());
-        Ok(())
-    }
-    /// Download a existing blob from oneself
-    #[tokio::test]
-    #[traced_test]
-    async fn test_blob_get_self_existing() -> TestResult<()> {
-        let node = node::Node::memory().spawn().await?;
-        let node_id = node.node_id();
-        let blobs = node.blobs();
-
-        let AddOutcome { hash, size, .. } = blobs.add_bytes("foo").await?;
-
-        // Direct
-        let res = blobs
-            .download_with_opts(
-                hash,
-                DownloadOptions {
-                    format: BlobFormat::Raw,
-                    nodes: vec![node_id.into()],
-                    tag: SetTagOption::Auto,
-                    mode: DownloadMode::Direct,
-                },
-            )
-            .await?
-            .await?;
-
-        assert_eq!(res.local_size, size);
-        assert_eq!(res.downloaded_size, 0);
-
-        // Queued
-        let res = blobs
-            .download_with_opts(
-                hash,
-                DownloadOptions {
-                    format: BlobFormat::Raw,
-                    nodes: vec![node_id.into()],
-                    tag: SetTagOption::Auto,
-                    mode: DownloadMode::Queued,
-                },
-            )
-            .await?
-            .await?;
-
-        assert_eq!(res.local_size, size);
-        assert_eq!(res.downloaded_size, 0);
-
-        Ok(())
-    }
-
-    /// Download a missing blob from oneself
-    #[tokio::test]
-    #[traced_test]
-    async fn test_blob_get_self_missing() -> TestResult<()> {
-        let node = node::Node::memory().spawn().await?;
-        let node_id = node.node_id();
-        let blobs = node.blobs();
-
-        let hash = Hash::from_bytes([0u8; 32]);
-
-        // Direct
-        let res = blobs
-            .download_with_opts(
-                hash,
-                DownloadOptions {
-                    format: BlobFormat::Raw,
-                    nodes: vec![node_id.into()],
-                    tag: SetTagOption::Auto,
-                    mode: DownloadMode::Direct,
-                },
-            )
-            .await?
-            .await;
-        assert!(res.is_err());
-        assert_eq!(
-            res.err().unwrap().to_string().as_str(),
-            "No nodes to download from provided"
-        );
-
-        // Queued
-        let res = blobs
-            .download_with_opts(
-                hash,
-                DownloadOptions {
-                    format: BlobFormat::Raw,
-                    nodes: vec![node_id.into()],
-                    tag: SetTagOption::Auto,
-                    mode: DownloadMode::Queued,
-                },
-            )
-            .await?
-            .await;
-        assert!(res.is_err());
-        assert_eq!(
-            res.err().unwrap().to_string().as_str(),
-            "No provider nodes found"
-        );
-
-        Ok(())
-    }
-
-    /// Download a existing collection. Check that things succeed and no download is performed.
-    #[tokio::test]
-    #[traced_test]
-    async fn test_blob_get_existing_collection() -> TestResult<()> {
-        let node = node::Node::memory().spawn().await?;
-        // We use a nonexisting node id because we just want to check that this succeeds without
-        // hitting the network.
-        let node_id = NodeId::from_bytes(&[0u8; 32])?;
-        let blobs = node.blobs();
-
-        let mut collection = Collection::default();
-        let mut tags = Vec::new();
-        let mut size = 0;
-        for value in ["iroh", "is", "cool"] {
-            let import_outcome = blobs.add_bytes(value).await.context("add bytes")?;
-            collection.push(value.to_string(), import_outcome.hash);
-            tags.push(import_outcome.tag);
-            size += import_outcome.size;
-        }
-
-        let (hash, _tag) = blobs
-            .create_collection(collection, SetTagOption::Auto, tags)
-            .await?;
-
-        // load the hashseq and collection header manually to calculate our expected size
-        let hashseq_bytes = blobs.read_to_bytes(hash).await?;
-        size += hashseq_bytes.len() as u64;
-        let hashseq = HashSeq::try_from(hashseq_bytes)?;
-        let collection_header_bytes = blobs
-            .read_to_bytes(hashseq.into_iter().next().expect("header to exist"))
-            .await?;
-        size += collection_header_bytes.len() as u64;
-
-        // Direct
-        let res = blobs
-            .download_with_opts(
-                hash,
-                DownloadOptions {
-                    format: BlobFormat::HashSeq,
-                    nodes: vec![node_id.into()],
-                    tag: SetTagOption::Auto,
-                    mode: DownloadMode::Direct,
-                },
-            )
-            .await?
-            .await
-            .context("direct (download)")?;
-
-        assert_eq!(res.local_size, size);
-        assert_eq!(res.downloaded_size, 0);
-
-        // Queued
-        let res = blobs
-            .download_with_opts(
-                hash,
-                DownloadOptions {
-                    format: BlobFormat::HashSeq,
-                    nodes: vec![node_id.into()],
-                    tag: SetTagOption::Auto,
-                    mode: DownloadMode::Queued,
-                },
-            )
-            .await?
-            .await
-            .context("queued")?;
-
-        assert_eq!(res.local_size, size);
-        assert_eq!(res.downloaded_size, 0);
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    #[traced_test]
-    #[cfg_attr(target_os = "windows", ignore = "flaky")]
-    async fn test_blob_delete_mem() -> Result<()> {
-        let node = node::Node::memory().spawn().await?;
-
-        let res = node.blobs().add_bytes(&b"hello world"[..]).await?;
-
-        let hashes: Vec<_> = node.blobs().list().await?.try_collect().await?;
-        assert_eq!(hashes.len(), 1);
-        assert_eq!(hashes[0].hash, res.hash);
-
-        // delete
-        node.blobs().delete_blob(res.hash).await?;
-
-        let hashes: Vec<_> = node.blobs().list().await?.try_collect().await?;
-        assert!(hashes.is_empty());
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    #[traced_test]
-    async fn test_blob_delete_fs() -> Result<()> {
-        let dir = tempfile::tempdir()?;
-        let node = node::Node::persistent(dir.path()).await?.spawn().await?;
-
-        let res = node.blobs().add_bytes(&b"hello world"[..]).await?;
-
-        let hashes: Vec<_> = node.blobs().list().await?.try_collect().await?;
-        assert_eq!(hashes.len(), 1);
-        assert_eq!(hashes[0].hash, res.hash);
-
-        // delete
-        node.blobs().delete_blob(res.hash).await?;
-
-        let hashes: Vec<_> = node.blobs().list().await?.try_collect().await?;
-        assert!(hashes.is_empty());
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    #[traced_test]
-    async fn test_ticket_multiple_addrs() -> TestResult<()> {
-        let node = Node::memory().spawn().await?;
-        let hash = node
-            .blobs()
-            .add_bytes(Bytes::from_static(b"hello"))
-            .await?
-            .hash;
-
-        let addr = node.node_addr().await?;
-        let ticket = BlobTicket::new(addr, hash, BlobFormat::Raw)?;
-        println!("addrs: {:?}", ticket.node_addr());
-        assert!(!ticket.node_addr().direct_addresses.is_empty());
-        Ok(())
-    }
-
-    #[tokio::test]
-    #[traced_test]
-    async fn test_node_add_blob_stream() -> Result<()> {
-        use std::io::Cursor;
-        let node = Node::memory().spawn().await?;
-
-        let blobs = node.blobs();
-        let input = vec![2u8; 1024 * 256]; // 265kb so actually streaming, chunk size is 64kb
-        let reader = Cursor::new(input.clone());
-        let progress = blobs.add_reader(reader, SetTagOption::Auto).await?;
-        let outcome = progress.finish().await?;
-        let hash = outcome.hash;
-        let output = blobs.read_to_bytes(hash).await?;
-        assert_eq!(input, output.to_vec());
-        Ok(())
-    }
-
-    #[tokio::test]
-    #[traced_test]
-    async fn test_node_add_tagged_blob_event() -> Result<()> {
-        let node = Node::memory().spawn().await?;
-
-        let _got_hash = tokio::time::timeout(Duration::from_secs(10), async move {
-            let mut stream = node
-                .blobs()
-                .add_from_path(
-                    Path::new(env!("CARGO_MANIFEST_DIR")).join("README.md"),
-                    false,
-                    SetTagOption::Auto,
-                    WrapOption::NoWrap,
-                )
-                .await?;
-
-            while let Some(progress) = stream.next().await {
-                match progress? {
-                    crate::provider::AddProgress::AllDone { hash, .. } => {
-                        return Ok(hash);
-                    }
-                    crate::provider::AddProgress::Abort(e) => {
-                        anyhow::bail!("Error while adding data: {e}");
-                    }
-                    _ => {}
-                }
-            }
-            anyhow::bail!("stream ended without providing data");
-        })
-        .await
-        .context("timeout")?
-        .context("get failed")?;
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    #[traced_test]
-    async fn test_download_via_relay() -> Result<()> {
-        let (relay_map, relay_url, _guard) = iroh::test_utils::run_relay_server().await?;
-
-        let endpoint1 = iroh::Endpoint::builder()
-            .relay_mode(RelayMode::Custom(relay_map.clone()))
-            .insecure_skip_relay_cert_verify(true);
-        let node1 = Node::memory().endpoint(endpoint1).spawn().await?;
-        let endpoint2 = iroh::Endpoint::builder()
-            .relay_mode(RelayMode::Custom(relay_map.clone()))
-            .insecure_skip_relay_cert_verify(true);
-        let node2 = Node::memory().endpoint(endpoint2).spawn().await?;
-        let AddOutcome { hash, .. } = node1.blobs().add_bytes(b"foo".to_vec()).await?;
-
-        // create a node addr with only a relay URL, no direct addresses
-        let addr = NodeAddr::new(node1.node_id()).with_relay_url(relay_url);
-        node2.blobs().download(hash, addr).await?.await?;
-        assert_eq!(
-            node2
-                .blobs()
-                .read_to_bytes(hash)
-                .await
-                .context("get")?
-                .as_ref(),
-            b"foo"
-        );
-        Ok(())
-    }
-
-    #[tokio::test]
-    #[traced_test]
-    #[ignore = "flaky"]
-    async fn test_download_via_relay_with_discovery() -> Result<()> {
-        let (relay_map, _relay_url, _guard) = iroh::test_utils::run_relay_server().await?;
-        let dns_pkarr_server = DnsPkarrServer::run().await?;
-
-        let mut rng = rand::thread_rng();
-
-        let secret1 = SecretKey::generate(&mut rng);
-        let endpoint1 = iroh::Endpoint::builder()
-            .relay_mode(RelayMode::Custom(relay_map.clone()))
-            .insecure_skip_relay_cert_verify(true)
-            .dns_resolver(dns_pkarr_server.dns_resolver())
-            .secret_key(secret1.clone())
-            .discovery(dns_pkarr_server.discovery(secret1));
-        let node1 = Node::memory().endpoint(endpoint1).spawn().await?;
-        let secret2 = SecretKey::generate(&mut rng);
-        let endpoint2 = iroh::Endpoint::builder()
-            .relay_mode(RelayMode::Custom(relay_map.clone()))
-            .insecure_skip_relay_cert_verify(true)
-            .dns_resolver(dns_pkarr_server.dns_resolver())
-            .secret_key(secret2.clone())
-            .discovery(dns_pkarr_server.discovery(secret2));
-        let node2 = Node::memory().endpoint(endpoint2).spawn().await?;
-        let hash = node1.blobs().add_bytes(b"foo".to_vec()).await?.hash;
-
-        // create a node addr with node id only
-        let addr = NodeAddr::new(node1.node_id());
-        node2.blobs().download(hash, addr).await?.await?;
-        assert_eq!(
-            node2
-                .blobs()
-                .read_to_bytes(hash)
-                .await
-                .context("get")?
-                .as_ref(),
-            b"foo"
-        );
-        Ok(())
-    }
-}
diff --git a/src/rpc/client/blobs/batch.rs b/src/rpc/client/blobs/batch.rs
deleted file mode 100644
index 5a964441e..000000000
--- a/src/rpc/client/blobs/batch.rs
+++ /dev/null
@@ -1,472 +0,0 @@
-use std::{
-    io,
-    path::PathBuf,
-    sync::{Arc, Mutex},
-};
-
-use anyhow::{anyhow, Context, Result};
-use bytes::Bytes;
-use futures_buffered::BufferedStreamExt;
-use futures_lite::StreamExt;
-use futures_util::{sink::Buffer, FutureExt, SinkExt, Stream};
-use quic_rpc::{client::UpdateSink, Connector, RpcClient};
-use tokio::io::AsyncRead;
-use tokio_util::io::ReaderStream;
-use tracing::{debug, warn};
-
-use super::WrapOption;
-use crate::{
-    format::collection::Collection,
-    net_protocol::BatchId,
-    provider::BatchAddPathProgress,
-    rpc::proto::{
-        blobs::{
-            BatchAddPathRequest, BatchAddStreamRequest, BatchAddStreamResponse,
-            BatchAddStreamUpdate, BatchCreateTempTagRequest, BatchUpdate,
-        },
-        tags::{self, SyncMode},
-        RpcService,
-    },
-    store::ImportMode,
-    util::{SetTagOption, TagDrop},
-    BlobFormat, HashAndFormat, Tag, TempTag,
-};
-
-/// A scope in which blobs can be added.
-#[derive(derive_more::Debug)]
-struct BatchInner<C>
-where
-    C: Connector<RpcService>,
-{
-    /// The id of the scope.
-    batch: BatchId,
-    /// The rpc client.
-    rpc: RpcClient<RpcService, C>,
-    /// The stream to send drop
-    #[debug(skip)]
-    updates: Mutex<Buffer<UpdateSink<C, BatchUpdate>, BatchUpdate>>,
-}
-
-/// A batch for write operations.
-///
-/// This serves mostly as a scope for temporary tags.
-///
-/// It is not a transaction, so things in a batch are not atomic. Also, there is
-/// no isolation between batches.
-#[derive(derive_more::Debug)]
-pub struct Batch<C>(Arc<BatchInner<C>>)
-where
-    C: Connector<RpcService>;
-
-impl<C> TagDrop for BatchInner<C>
-where
-    C: Connector<RpcService>,
-{
-    fn on_drop(&self, content: &HashAndFormat) {
-        let mut updates = self.updates.lock().unwrap();
-        // make a spirited attempt to notify the server that we are dropping the content
-        //
-        // this will occasionally fail, but that's acceptable. The temp tags for the batch
-        // will be cleaned up as soon as the entire batch is dropped.
-        //
-        // E.g. a typical scenario is that you create a large array of temp tags, and then
-        // store them in a hash sequence and then drop the array. You will get many drops
-        // at the same time, and might get a send failure here.
-        //
-        // But that just means that the server will clean up the temp tags when the batch is
-        // dropped.
-        updates.feed(BatchUpdate::Drop(*content)).now_or_never();
-        updates.flush().now_or_never();
-    }
-}
-
-/// Options for adding a file as a blob
-#[derive(Debug, Clone, Copy, Default)]
-pub struct AddFileOpts {
-    /// The import mode
-    pub import_mode: ImportMode,
-    /// The format of the blob
-    pub format: BlobFormat,
-}
-
-/// Options for adding a directory as a collection
-#[derive(Debug, Clone)]
-pub struct AddDirOpts {
-    /// The import mode
-    pub import_mode: ImportMode,
-    /// Whether to preserve the directory name
-    pub wrap: WrapOption,
-    /// Io parallelism
-    pub io_parallelism: usize,
-}
-
-impl Default for AddDirOpts {
-    fn default() -> Self {
-        Self {
-            import_mode: ImportMode::TryReference,
-            wrap: WrapOption::NoWrap,
-            io_parallelism: 4,
-        }
-    }
-}
-
-/// Options for adding a directory as a collection
-#[derive(Debug, Clone)]
-pub struct AddReaderOpts {
-    /// The format of the blob
-    pub format: BlobFormat,
-    /// Size of the chunks to send
-    pub chunk_size: usize,
-}
-
-impl Default for AddReaderOpts {
-    fn default() -> Self {
-        Self {
-            format: BlobFormat::Raw,
-            chunk_size: 1024 * 64,
-        }
-    }
-}
-
-impl<C> Batch<C>
-where
-    C: Connector<RpcService>,
-{
-    pub(super) fn new(
-        batch: BatchId,
-        rpc: RpcClient<RpcService, C>,
-        updates: UpdateSink<C, BatchUpdate>,
-        buffer_size: usize,
-    ) -> Self {
-        let updates = updates.buffer(buffer_size);
-        Self(Arc::new(BatchInner {
-            batch,
-            rpc,
-            updates: updates.into(),
-        }))
-    }
-
-    /// Write a blob by passing bytes.
-    pub async fn add_bytes(&self, bytes: impl Into<Bytes>) -> Result<TempTag> {
-        self.add_bytes_with_opts(bytes, Default::default()).await
-    }
-
-    /// Import a blob from a filesystem path, using the default options.
-    ///
-    /// For more control, use [`Self::add_file_with_opts`].
-    pub async fn add_file(&self, path: PathBuf) -> Result<(TempTag, u64)> {
-        self.add_file_with_opts(path, AddFileOpts::default()).await
-    }
-
-    /// Add a directory as a hashseq in iroh collection format
-    pub async fn add_dir(&self, root: PathBuf) -> Result<TempTag> {
-        self.add_dir_with_opts(root, Default::default()).await
-    }
-
-    /// Write a blob by passing an async reader.
-    ///
-    /// This will consume the stream in 64KB chunks, and use a format of [BlobFormat::Raw].
-    ///
-    /// For more options, see [`Self::add_reader_with_opts`].
-    pub async fn add_reader(
-        &self,
-        reader: impl AsyncRead + Unpin + Send + 'static,
-    ) -> anyhow::Result<TempTag> {
-        self.add_reader_with_opts(reader, Default::default()).await
-    }
-
-    /// Write a blob by passing a stream of bytes.
-    pub async fn add_stream(
-        &self,
-        input: impl Stream<Item = io::Result<Bytes>> + Send + Unpin + 'static,
-    ) -> Result<TempTag> {
-        self.add_stream_with_opts(input, Default::default()).await
-    }
-
-    /// Creates a temp tag to protect some content (blob or hashseq) from being deleted.
-    ///
-    /// This is a lower-level API. The other functions in [`Batch`] already create [`TempTag`]s automatically.
-    ///
-    /// [`TempTag`]s allow you to protect some data from deletion while a download is ongoing,
-    /// even if you don't want to protect it permanently.
-    pub async fn temp_tag(&self, content: HashAndFormat) -> Result<TempTag> {
-        // Notify the server that we want one temp tag for the given content
-        self.0
-            .rpc
-            .rpc(BatchCreateTempTagRequest {
-                batch: self.0.batch,
-                content,
-            })
-            .await??;
-        // Only after success of the above call, we can create the corresponding local temp tag
-        Ok(self.local_temp_tag(content, None))
-    }
-
-    /// Write a blob by passing an async reader.
-    ///
-    /// This consumes the stream in chunks using `opts.chunk_size`. A good default is 64KB.
-    pub async fn add_reader_with_opts(
-        &self,
-        reader: impl AsyncRead + Unpin + Send + 'static,
-        opts: AddReaderOpts,
-    ) -> anyhow::Result<TempTag> {
-        let AddReaderOpts { format, chunk_size } = opts;
-        let input = ReaderStream::with_capacity(reader, chunk_size);
-        self.add_stream_with_opts(input, format).await
-    }
-
-    /// Write a blob by passing bytes.
-    pub async fn add_bytes_with_opts(
-        &self,
-        bytes: impl Into<Bytes>,
-        format: BlobFormat,
-    ) -> Result<TempTag> {
-        let input = futures_lite::stream::once(Ok(bytes.into()));
-        self.add_stream_with_opts(input, format).await
-    }
-
-    /// Import a blob from a filesystem path.
-    ///
-    /// `path` should be an absolute path valid for the file system on which
-    /// the node runs, which refers to a file.
-    ///
-    /// If you use [`ImportMode::TryReference`], Iroh will assume that the data will not
-    /// change and will share it in place without copying to the Iroh data directory
-    /// if appropriate. However, for tiny files, Iroh will copy the data.
-    ///
-    /// If you use [`ImportMode::Copy`], Iroh will always copy the data.
-    ///
-    /// Will return a temp tag for the added blob, as well as the size of the file.
-    pub async fn add_file_with_opts(
-        &self,
-        path: PathBuf,
-        opts: AddFileOpts,
-    ) -> Result<(TempTag, u64)> {
-        let AddFileOpts {
-            import_mode,
-            format,
-        } = opts;
-        anyhow::ensure!(
-            path.is_absolute(),
-            "Path must be absolute, but got: {:?}",
-            path
-        );
-        anyhow::ensure!(path.is_file(), "Path does not refer to a file: {:?}", path);
-        let mut stream = self
-            .0
-            .rpc
-            .server_streaming(BatchAddPathRequest {
-                path,
-                import_mode,
-                format,
-                batch: self.0.batch,
-            })
-            .await?;
-        let mut res_hash = None;
-        let mut res_size = None;
-        while let Some(item) = stream.next().await {
-            match item?.0 {
-                BatchAddPathProgress::Abort(cause) => {
-                    Err(cause)?;
-                }
-                BatchAddPathProgress::Done { hash } => {
-                    res_hash = Some(hash);
-                }
-                BatchAddPathProgress::Found { size } => {
-                    res_size = Some(size);
-                }
-                _ => {}
-            }
-        }
-        let hash = res_hash.context("Missing hash")?;
-        let size = res_size.context("Missing size")?;
-        Ok((
-            self.local_temp_tag(HashAndFormat { hash, format }, Some(size)),
-            size,
-        ))
-    }
-
-    /// Add a directory as a hashseq in iroh collection format
-    ///
-    /// This can also be used to add a single file as a collection, if
-    /// wrap is set to [WrapOption::Wrap].
-    ///
-    /// However, if you want to add a single file as a raw blob, use add_file instead.
-    pub async fn add_dir_with_opts(&self, root: PathBuf, opts: AddDirOpts) -> Result<TempTag> {
-        let AddDirOpts {
-            import_mode,
-            wrap,
-            io_parallelism,
-        } = opts;
-        anyhow::ensure!(root.is_absolute(), "Path must be absolute");
-
-        // let (send, recv) = flume::bounded(32);
-        // let import_progress = FlumeProgressSender::new(send);
-
-        // import all files below root recursively
-        let data_sources = crate::util::fs::scan_path(root, wrap)?;
-        let opts = AddFileOpts {
-            import_mode,
-            format: BlobFormat::Raw,
-        };
-        let result: Vec<_> = futures_lite::stream::iter(data_sources)
-            .map(|source| {
-                // let import_progress = import_progress.clone();
-                async move {
-                    let name = source.name().to_string();
-                    let (tag, size) = self
-                        .add_file_with_opts(source.path().to_owned(), opts)
-                        .await?;
-                    let hash = *tag.hash();
-                    anyhow::Ok((name, hash, size, tag))
-                }
-            })
-            .buffered_ordered(io_parallelism)
-            .try_collect()
-            .await?;
-
-        // create a collection
-        let (collection, child_tags): (Collection, Vec<_>) = result
-            .into_iter()
-            .map(|(name, hash, _, tag)| ((name, hash), tag))
-            .unzip();
-
-        let tag = self.add_collection(collection).await?;
-        drop(child_tags);
-        Ok(tag)
-    }
-
-    /// Write a blob by passing a stream of bytes.
-    ///
-    /// For convenient interop with common sources of data, this function takes a stream of `io::Result<Bytes>`.
-    /// If you have raw bytes, you need to wrap them in `io::Result::Ok`.
-    pub async fn add_stream_with_opts(
-        &self,
-        mut input: impl Stream<Item = io::Result<Bytes>> + Send + Unpin + 'static,
-        format: BlobFormat,
-    ) -> Result<TempTag> {
-        let (mut sink, mut stream) = self
-            .0
-            .rpc
-            .bidi(BatchAddStreamRequest {
-                batch: self.0.batch,
-                format,
-            })
-            .await?;
-        let mut size = 0u64;
-        while let Some(item) = input.next().await {
-            match item {
-                Ok(chunk) => {
-                    size += chunk.len() as u64;
-                    sink.send(BatchAddStreamUpdate::Chunk(chunk))
-                        .await
-                        .map_err(|err| anyhow!("Failed to send input stream to remote: {err:?}"))?;
-                }
-                Err(err) => {
-                    warn!("Abort send, reason: failed to read from source stream: {err:?}");
-                    sink.send(BatchAddStreamUpdate::Abort)
-                        .await
-                        .map_err(|err| anyhow!("Failed to send input stream to remote: {err:?}"))?;
-                    break;
-                }
-            }
-        }
-        // this is needed for the remote to notice that the stream is closed
-        drop(sink);
-        let mut res = None;
-        while let Some(item) = stream.next().await {
-            match item? {
-                BatchAddStreamResponse::Abort(cause) => {
-                    Err(cause)?;
-                }
-                BatchAddStreamResponse::Result { hash } => {
-                    res = Some(hash);
-                }
-                _ => {}
-            }
-        }
-        let hash = res.context("Missing answer")?;
-        Ok(self.local_temp_tag(HashAndFormat { hash, format }, Some(size)))
-    }
-
-    /// Add a collection.
-    ///
-    /// This is a convenience function that converts the collection into two blobs
-    /// (the metadata and the hash sequence) and adds them, returning a temp tag for
-    /// the hash sequence.
-    ///
-    /// Note that this does not guarantee that the data that the collection refers to
-    /// actually exists. It will just create 2 blobs, the metadata and the hash sequence
-    /// itself.
-    pub async fn add_collection(&self, collection: Collection) -> Result<TempTag> {
-        self.add_blob_seq(collection.to_blobs()).await
-    }
-
-    /// Add a sequence of blobs, where the last is a hash sequence.
-    ///
-    /// It is a common pattern in iroh to have a hash sequence with one or more
-    /// blobs of metadata, and the remaining blobs being the actual data. E.g.
-    /// a collection is a hash sequence where the first child is the metadata.
-    pub async fn add_blob_seq(&self, iter: impl Iterator<Item = Bytes>) -> Result<TempTag> {
-        let mut blobs = iter.peekable();
-        // put the tags somewhere
-        let mut tags = vec![];
-        loop {
-            let blob = blobs.next().context("Failed to get next blob")?;
-            if blobs.peek().is_none() {
-                return self.add_bytes_with_opts(blob, BlobFormat::HashSeq).await;
-            } else {
-                tags.push(self.add_bytes(blob).await?);
-            }
-        }
-    }
-
-    /// Upgrades a temp tag to a persistent tag.
-    pub async fn persist(&self, tt: TempTag) -> Result<Tag> {
-        let tag = self
-            .0
-            .rpc
-            .rpc(tags::CreateRequest {
-                value: tt.hash_and_format(),
-                batch: Some(self.0.batch),
-                sync: SyncMode::Full,
-            })
-            .await??;
-        Ok(tag)
-    }
-
-    /// Upgrades a temp tag to a persistent tag with a specific name.
-    pub async fn persist_to(&self, tt: TempTag, tag: Tag) -> Result<()> {
-        self.0
-            .rpc
-            .rpc(tags::SetRequest {
-                name: tag,
-                value: tt.hash_and_format(),
-                batch: Some(self.0.batch),
-                sync: SyncMode::Full,
-            })
-            .await??;
-        Ok(())
-    }
-
-    /// Upgrades a temp tag to a persistent tag with either a specific name or
-    /// an automatically generated name.
-    pub async fn persist_with_opts(&self, tt: TempTag, opts: SetTagOption) -> Result<Tag> {
-        match opts {
-            SetTagOption::Auto => self.persist(tt).await,
-            SetTagOption::Named(tag) => {
-                self.persist_to(tt, tag.clone()).await?;
-                Ok(tag)
-            }
-        }
-    }
-
-    /// Creates a temp tag for the given hash and format, without notifying the server.
-    ///
-    /// Caution: only do this for data for which you know the server side has created a temp tag.
-    fn local_temp_tag(&self, inner: HashAndFormat, _size: Option<u64>) -> TempTag {
-        let on_drop: Arc<dyn TagDrop> = self.0.clone();
-        let on_drop = Some(Arc::downgrade(&on_drop));
-        TempTag::new(inner, on_drop)
-    }
-}
diff --git a/src/rpc/client/tags.rs b/src/rpc/client/tags.rs
deleted file mode 100644
index cd9000ea6..000000000
--- a/src/rpc/client/tags.rs
+++ /dev/null
@@ -1,342 +0,0 @@
-//! API for tag management.
-//!
-//! The purpose of tags is to mark information as important to prevent it
-//! from being garbage-collected (if the garbage collector is turned on).
-//!
-//! A tag has a name that is an arbitrary byte string. In many cases this will be
-//! a valid UTF8 string, but there are also use cases where it is useful to have
-//! non string data like integer ids in the tag name.
-//!
-//! Tags point to a [`HashAndFormat`].
-//!
-//! A tag can point to a hash with format [`BlobFormat::Raw`]. In that case it will
-//! protect *just this blob* from being garbage-collected.
-//!
-//! It can also point to a hash in format [`BlobFormat::HashSeq`]. In that case it will
-//! protect the blob itself and all hashes in the blob (the blob must be just a sequence of hashes).
-//! Using this format it is possible to protect a large number of blobs with a single tag.
-//!
-//! Tags can be created, read, renamed and deleted. Tags *do not* have to correspond to
-//! already existing data. It is perfectly valid to create a tag for data you don't have yet.
-//!
-//! The main entry point is the [`Client`].
-use std::ops::{Bound, RangeBounds};
-
-use anyhow::Result;
-use futures_lite::{Stream, StreamExt};
-use quic_rpc::{client::BoxedConnector, Connector, RpcClient};
-use serde::{Deserialize, Serialize};
-
-use crate::{
-    rpc::proto::{
-        tags::{DeleteRequest, ListRequest, RenameRequest, SetRequest, SyncMode},
-        RpcService,
-    },
-    BlobFormat, Hash, HashAndFormat, Tag,
-};
-
-/// Iroh tags client.
-#[derive(Debug, Clone)]
-#[repr(transparent)]
-pub struct Client<C = BoxedConnector<RpcService>> {
-    pub(super) rpc: RpcClient<RpcService, C>,
-}
-
-/// Options for a list operation.
-#[derive(Debug, Clone)]
-pub struct ListOptions {
-    /// List tags to hash seqs
-    pub hash_seq: bool,
-    /// List tags to raw blobs
-    pub raw: bool,
-    /// Optional from tag (inclusive)
-    pub from: Option<Tag>,
-    /// Optional to tag (exclusive)
-    pub to: Option<Tag>,
-}
-
-fn tags_from_range<R, E>(range: R) -> (Option<Tag>, Option<Tag>)
-where
-    R: RangeBounds<E>,
-    E: AsRef<[u8]>,
-{
-    let from = match range.start_bound() {
-        Bound::Included(start) => Some(Tag::from(start.as_ref())),
-        Bound::Excluded(start) => Some(Tag::from(start.as_ref()).successor()),
-        Bound::Unbounded => None,
-    };
-    let to = match range.end_bound() {
-        Bound::Included(end) => Some(Tag::from(end.as_ref()).successor()),
-        Bound::Excluded(end) => Some(Tag::from(end.as_ref())),
-        Bound::Unbounded => None,
-    };
-    (from, to)
-}
-
-impl ListOptions {
-    /// List a range of tags
-    pub fn range<R, E>(range: R) -> Self
-    where
-        R: RangeBounds<E>,
-        E: AsRef<[u8]>,
-    {
-        let (from, to) = tags_from_range(range);
-        Self {
-            from,
-            to,
-            raw: true,
-            hash_seq: true,
-        }
-    }
-
-    /// List tags with a prefix
-    pub fn prefix(prefix: &[u8]) -> Self {
-        let from = Tag::from(prefix);
-        let to = from.next_prefix();
-        Self {
-            raw: true,
-            hash_seq: true,
-            from: Some(from),
-            to,
-        }
-    }
-
-    /// List a single tag
-    pub fn single(name: &[u8]) -> Self {
-        let from = Tag::from(name);
-        Self {
-            to: Some(from.successor()),
-            from: Some(from),
-            raw: true,
-            hash_seq: true,
-        }
-    }
-
-    /// List all tags
-    pub fn all() -> Self {
-        Self {
-            raw: true,
-            hash_seq: true,
-            from: None,
-            to: None,
-        }
-    }
-
-    /// List raw tags
-    pub fn raw() -> Self {
-        Self {
-            raw: true,
-            hash_seq: false,
-            from: None,
-            to: None,
-        }
-    }
-
-    /// List hash seq tags
-    pub fn hash_seq() -> Self {
-        Self {
-            raw: false,
-            hash_seq: true,
-            from: None,
-            to: None,
-        }
-    }
-}
-
-/// Options for a delete operation.
-#[derive(Debug, Clone)]
-pub struct DeleteOptions {
-    /// Optional from tag (inclusive)
-    pub from: Option<Tag>,
-    /// Optional to tag (exclusive)
-    pub to: Option<Tag>,
-}
-
-impl DeleteOptions {
-    /// Delete a single tag
-    pub fn single(name: &[u8]) -> Self {
-        let name = Tag::from(name);
-        Self {
-            to: Some(name.successor()),
-            from: Some(name),
-        }
-    }
-
-    /// Delete a range of tags
-    pub fn range<R, E>(range: R) -> Self
-    where
-        R: RangeBounds<E>,
-        E: AsRef<[u8]>,
-    {
-        let (from, to) = tags_from_range(range);
-        Self { from, to }
-    }
-
-    /// Delete tags with a prefix
-    pub fn prefix(prefix: &[u8]) -> Self {
-        let from = Tag::from(prefix);
-        let to = from.next_prefix();
-        Self {
-            from: Some(from),
-            to,
-        }
-    }
-}
-
-/// A client that uses the memory connector.
-pub type MemClient = Client<crate::rpc::MemConnector>;
-
-impl<C> Client<C>
-where
-    C: Connector<RpcService>,
-{
-    /// Creates a new client
-    pub fn new(rpc: RpcClient<RpcService, C>) -> Self {
-        Self { rpc }
-    }
-
-    /// List all tags with options.
-    ///
-    /// This is the most flexible way to list tags. All the other list methods are just convenience
-    /// methods that call this one with the appropriate options.
-    pub async fn list_with_opts(
-        &self,
-        options: ListOptions,
-    ) -> Result<impl Stream<Item = Result<TagInfo>>> {
-        let stream = self
-            .rpc
-            .server_streaming(ListRequest::from(options))
-            .await?;
-        Ok(stream.map(|res| res.map_err(anyhow::Error::from)))
-    }
-
-    /// Set the value for a single tag
-    pub async fn set(&self, name: impl AsRef<[u8]>, value: impl Into<HashAndFormat>) -> Result<()> {
-        self.rpc
-            .rpc(SetRequest {
-                name: Tag::from(name.as_ref()),
-                value: value.into(),
-                batch: None,
-                sync: SyncMode::Full,
-            })
-            .await??;
-        Ok(())
-    }
-
-    /// Get the value of a single tag
-    pub async fn get(&self, name: impl AsRef<[u8]>) -> Result<Option<TagInfo>> {
-        let mut stream = self
-            .list_with_opts(ListOptions::single(name.as_ref()))
-            .await?;
-        stream.next().await.transpose()
-    }
-
-    /// Rename a tag atomically
-    ///
-    /// If the tag does not exist, this will return an error.
-    pub async fn rename(&self, from: impl AsRef<[u8]>, to: impl AsRef<[u8]>) -> Result<()> {
-        self.rpc
-            .rpc(RenameRequest {
-                from: Tag::from(from.as_ref()),
-                to: Tag::from(to.as_ref()),
-            })
-            .await??;
-        Ok(())
-    }
-
-    /// List a range of tags
-    pub async fn list_range<R, E>(&self, range: R) -> Result<impl Stream<Item = Result<TagInfo>>>
-    where
-        R: RangeBounds<E>,
-        E: AsRef<[u8]>,
-    {
-        self.list_with_opts(ListOptions::range(range)).await
-    }
-
-    /// Lists all tags with the given prefix.
-    pub async fn list_prefix(
-        &self,
-        prefix: impl AsRef<[u8]>,
-    ) -> Result<impl Stream<Item = Result<TagInfo>>> {
-        self.list_with_opts(ListOptions::prefix(prefix.as_ref()))
-            .await
-    }
-
-    /// Lists all tags.
-    pub async fn list(&self) -> Result<impl Stream<Item = Result<TagInfo>>> {
-        self.list_with_opts(ListOptions::all()).await
-    }
-
-    /// Lists all tags with a hash_seq format.
-    pub async fn list_hash_seq(&self) -> Result<impl Stream<Item = Result<TagInfo>>> {
-        self.list_with_opts(ListOptions::hash_seq()).await
-    }
-
-    /// Deletes a tag.
-    pub async fn delete_with_opts(&self, options: DeleteOptions) -> Result<()> {
-        self.rpc.rpc(DeleteRequest::from(options)).await??;
-        Ok(())
-    }
-
-    /// Deletes a tag.
-    pub async fn delete(&self, name: impl AsRef<[u8]>) -> Result<()> {
-        self.delete_with_opts(DeleteOptions::single(name.as_ref()))
-            .await
-    }
-
-    /// Deletes a range of tags.
-    pub async fn delete_range<R, E>(&self, range: R) -> Result<()>
-    where
-        R: RangeBounds<E>,
-        E: AsRef<[u8]>,
-    {
-        self.delete_with_opts(DeleteOptions::range(range)).await
-    }
-
-    /// Delete all tags with the given prefix.
-    pub async fn delete_prefix(&self, prefix: impl AsRef<[u8]>) -> Result<()> {
-        self.delete_with_opts(DeleteOptions::prefix(prefix.as_ref()))
-            .await
-    }
-
-    /// Delete all tags. Use with care. After this, all data will be garbage collected.
-    pub async fn delete_all(&self) -> Result<()> {
-        self.delete_with_opts(DeleteOptions {
-            from: None,
-            to: None,
-        })
-        .await
-    }
-}
-
-/// Information about a tag.
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
-pub struct TagInfo {
-    /// Name of the tag
-    pub name: Tag,
-    /// Format of the data
-    pub format: BlobFormat,
-    /// Hash of the data
-    pub hash: Hash,
-}
-
-impl TagInfo {
-    /// Create a new tag info.
-    pub fn new(name: impl AsRef<[u8]>, value: impl Into<HashAndFormat>) -> Self {
-        let name = name.as_ref();
-        let value = value.into();
-        Self {
-            name: Tag::from(name),
-            hash: value.hash,
-            format: value.format,
-        }
-    }
-
-    /// Get the hash and format of the tag.
-    pub fn hash_and_format(&self) -> HashAndFormat {
-        HashAndFormat {
-            hash: self.hash,
-            format: self.format,
-        }
-    }
-}
diff --git a/src/rpc/proto.rs b/src/rpc/proto.rs
deleted file mode 100644
index 174b0a80c..000000000
--- a/src/rpc/proto.rs
+++ /dev/null
@@ -1,36 +0,0 @@
-//! RPC protocol for the iroh-blobs service
-use nested_enum_utils::enum_conversions;
-use serde::{Deserialize, Serialize};
-
-pub mod blobs;
-pub mod tags;
-
-/// quic-rpc service for iroh blobs
-#[derive(Debug, Clone)]
-pub struct RpcService;
-
-impl quic_rpc::Service for RpcService {
-    type Req = Request;
-    type Res = Response;
-}
-
-#[allow(missing_docs)]
-#[enum_conversions]
-#[derive(Debug, Serialize, Deserialize)]
-pub enum Request {
-    Blobs(blobs::Request),
-    Tags(tags::Request),
-}
-
-#[allow(missing_docs)]
-#[enum_conversions]
-#[derive(Debug, Serialize, Deserialize)]
-pub enum Response {
-    Blobs(blobs::Response),
-    Tags(tags::Response),
-}
-
-/// Error type for RPC operations
-pub type RpcError = serde_error::Error;
-/// Result type for RPC operations
-pub type RpcResult<T> = Result<T, RpcError>;
diff --git a/src/rpc/proto/blobs.rs b/src/rpc/proto/blobs.rs
deleted file mode 100644
index 75fdad1c7..000000000
--- a/src/rpc/proto/blobs.rs
+++ /dev/null
@@ -1,318 +0,0 @@
-//! RPC requests and responses for the blob service.
-use std::path::PathBuf;
-
-use bytes::Bytes;
-use nested_enum_utils::enum_conversions;
-use quic_rpc_derive::rpc_requests;
-use serde::{Deserialize, Serialize};
-
-use super::{RpcError, RpcResult, RpcService};
-use crate::{
-    export::ExportProgress,
-    format::collection::Collection,
-    get::db::DownloadProgress,
-    net_protocol::{BatchId, BlobDownloadRequest},
-    provider::{AddProgress, BatchAddPathProgress},
-    rpc::client::blobs::{BlobInfo, BlobStatus, IncompleteBlobInfo, ReadAtLen, WrapOption},
-    store::{
-        BaoBlobSize, ConsistencyCheckProgress, ExportFormat, ExportMode, ImportMode,
-        ValidateProgress,
-    },
-    util::SetTagOption,
-    BlobFormat, Hash, HashAndFormat, Tag,
-};
-
-#[allow(missing_docs)]
-#[derive(strum::Display, Debug, Serialize, Deserialize)]
-#[enum_conversions(super::Request)]
-#[rpc_requests(RpcService)]
-pub enum Request {
-    #[server_streaming(response = RpcResult<ReadAtResponse>)]
-    ReadAt(ReadAtRequest),
-    #[bidi_streaming(update = AddStreamUpdate, response = AddStreamResponse)]
-    AddStream(AddStreamRequest),
-    AddStreamUpdate(AddStreamUpdate),
-    #[server_streaming(response = AddPathResponse)]
-    AddPath(AddPathRequest),
-    #[server_streaming(response = DownloadResponse)]
-    Download(BlobDownloadRequest),
-    #[server_streaming(response = ExportResponse)]
-    Export(ExportRequest),
-    #[server_streaming(response = RpcResult<BlobInfo>)]
-    List(ListRequest),
-    #[server_streaming(response = RpcResult<IncompleteBlobInfo>)]
-    ListIncomplete(ListIncompleteRequest),
-    #[rpc(response = RpcResult<()>)]
-    Delete(DeleteRequest),
-    #[server_streaming(response = ValidateProgress)]
-    Validate(ValidateRequest),
-    #[server_streaming(response = ConsistencyCheckProgress)]
-    Fsck(ConsistencyCheckRequest),
-    #[rpc(response = RpcResult<CreateCollectionResponse>)]
-    CreateCollection(CreateCollectionRequest),
-    #[rpc(response = RpcResult<BlobStatusResponse>)]
-    BlobStatus(BlobStatusRequest),
-
-    #[bidi_streaming(update = BatchUpdate, response = BatchCreateResponse)]
-    BatchCreate(BatchCreateRequest),
-    BatchUpdate(BatchUpdate),
-    #[bidi_streaming(update = BatchAddStreamUpdate, response = BatchAddStreamResponse)]
-    BatchAddStream(BatchAddStreamRequest),
-    BatchAddStreamUpdate(BatchAddStreamUpdate),
-    #[server_streaming(response = BatchAddPathResponse)]
-    BatchAddPath(BatchAddPathRequest),
-    #[rpc(response = RpcResult<()>)]
-    BatchCreateTempTag(BatchCreateTempTagRequest),
-}
-
-#[allow(missing_docs)]
-#[derive(strum::Display, Debug, Serialize, Deserialize)]
-#[enum_conversions(super::Response)]
-pub enum Response {
-    ReadAt(RpcResult<ReadAtResponse>),
-    AddStream(AddStreamResponse),
-    AddPath(AddPathResponse),
-    List(RpcResult<BlobInfo>),
-    ListIncomplete(RpcResult<IncompleteBlobInfo>),
-    Download(DownloadResponse),
-    Fsck(ConsistencyCheckProgress),
-    Export(ExportResponse),
-    Validate(ValidateProgress),
-    CreateCollection(RpcResult<CreateCollectionResponse>),
-    BlobStatus(RpcResult<BlobStatusResponse>),
-    BatchCreate(BatchCreateResponse),
-    BatchAddStream(BatchAddStreamResponse),
-    BatchAddPath(BatchAddPathResponse),
-}
-
-/// A request to the node to provide the data at the given path
-///
-/// Will produce a stream of [`AddProgress`] messages.
-#[derive(Debug, Serialize, Deserialize)]
-pub struct AddPathRequest {
-    /// The path to the data to provide.
-    ///
-    /// This should be an absolute path valid for the file system on which
-    /// the node runs. Usually the cli will run on the same machine as the
-    /// node, so this should be an absolute path on the cli machine.
-    pub path: PathBuf,
-    /// True if the provider can assume that the data will not change, so it
-    /// can be shared in place.
-    pub in_place: bool,
-    /// Tag to tag the data with.
-    pub tag: SetTagOption,
-    /// Whether to wrap the added data in a collection
-    pub wrap: WrapOption,
-}
-
-/// Wrapper around [`AddProgress`].
-#[derive(Debug, Serialize, Deserialize, derive_more::Into)]
-pub struct AddPathResponse(pub AddProgress);
-
-/// Progress response for [`BlobDownloadRequest`]
-#[derive(Debug, Clone, Serialize, Deserialize, derive_more::From, derive_more::Into)]
-pub struct DownloadResponse(pub DownloadProgress);
-
-/// A request to the node to download and share the data specified by the hash.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ExportRequest {
-    /// The hash of the blob to export.
-    pub hash: Hash,
-    /// The filepath to where the data should be saved
-    ///
-    /// This should be an absolute path valid for the file system on which
-    /// the node runs.
-    pub path: PathBuf,
-    /// Set to [`ExportFormat::Collection`] if the `hash` refers to a [`Collection`] and you want
-    /// to export all children of the collection into individual files.
-    pub format: ExportFormat,
-    /// The mode of exporting.
-    ///
-    /// The default is [`ExportMode::Copy`]. See [`ExportMode`] for details.
-    pub mode: ExportMode,
-}
-
-/// Progress response for [`ExportRequest`]
-#[derive(Debug, Clone, Serialize, Deserialize, derive_more::From, derive_more::Into)]
-pub struct ExportResponse(pub ExportProgress);
-
-/// A request to the node to validate the integrity of all provided data
-#[derive(Debug, Serialize, Deserialize)]
-pub struct ConsistencyCheckRequest {
-    /// repair the store by dropping inconsistent blobs
-    pub repair: bool,
-}
-
-/// A request to the node to validate the integrity of all provided data
-#[derive(Debug, Serialize, Deserialize)]
-pub struct ValidateRequest {
-    /// repair the store by downgrading blobs from complete to partial
-    pub repair: bool,
-}
-
-/// List all blobs, including collections
-#[derive(Debug, Serialize, Deserialize)]
-pub struct ListRequest;
-
-/// List all blobs, including collections
-#[derive(Debug, Serialize, Deserialize)]
-pub struct ListIncompleteRequest;
-
-/// Get the bytes for a hash
-#[derive(Serialize, Deserialize, Debug)]
-pub struct ReadAtRequest {
-    /// Hash to get bytes for
-    pub hash: Hash,
-    /// Offset to start reading at
-    pub offset: u64,
-    /// Length of the data to get
-    pub len: ReadAtLen,
-}
-
-/// Response to [`ReadAtRequest`]
-#[derive(Serialize, Deserialize, Debug)]
-pub enum ReadAtResponse {
-    /// The entry header.
-    Entry {
-        /// The size of the blob
-        size: BaoBlobSize,
-        /// Whether the blob is complete
-        is_complete: bool,
-    },
-    /// Chunks of entry data.
-    Data {
-        /// The data chunk
-        chunk: Bytes,
-    },
-}
-
-/// Write a blob from a byte stream
-#[derive(Serialize, Deserialize, Debug)]
-pub struct AddStreamRequest {
-    /// Tag to tag the data with.
-    pub tag: SetTagOption,
-}
-
-/// Write a blob from a byte stream
-#[derive(Serialize, Deserialize, Debug)]
-pub enum AddStreamUpdate {
-    /// A chunk of stream data
-    Chunk(Bytes),
-    /// Abort the request due to an error on the client side
-    Abort,
-}
-
-/// Wrapper around [`AddProgress`].
-#[derive(Debug, Serialize, Deserialize, derive_more::Into)]
-pub struct AddStreamResponse(pub AddProgress);
-
-/// Delete a blob
-#[derive(Debug, Serialize, Deserialize)]
-pub struct DeleteRequest {
-    /// Name of the tag
-    pub hash: Hash,
-}
-
-/// Create a collection.
-#[derive(Debug, Serialize, Deserialize)]
-pub struct CreateCollectionRequest {
-    /// The collection
-    pub collection: Collection,
-    /// Tag option.
-    pub tag: SetTagOption,
-    /// Tags that should be deleted after creation.
-    pub tags_to_delete: Vec<Tag>,
-}
-
-/// A response to a create collection request
-#[derive(Debug, Serialize, Deserialize)]
-pub struct CreateCollectionResponse {
-    /// The resulting hash.
-    pub hash: Hash,
-    /// The resulting tag.
-    pub tag: Tag,
-}
-
-/// Request to get the status of a blob
-#[derive(Debug, Serialize, Deserialize)]
-pub struct BlobStatusRequest {
-    /// The hash of the blob
-    pub hash: Hash,
-}
-
-/// The response to a status request
-#[derive(Debug, Serialize, Deserialize, derive_more::From, derive_more::Into)]
-pub struct BlobStatusResponse(pub BlobStatus);
-
-/// Request to create a new scope for temp tags
-#[derive(Debug, Serialize, Deserialize)]
-pub struct BatchCreateRequest;
-
-/// Update to a temp tag scope
-#[derive(Debug, Serialize, Deserialize)]
-pub enum BatchUpdate {
-    /// Drop of a remote temp tag
-    Drop(HashAndFormat),
-    /// Message to check that the connection is still alive
-    Ping,
-}
-
-/// Response to a temp tag scope request
-#[derive(Debug, Serialize, Deserialize)]
-pub enum BatchCreateResponse {
-    /// We got the id of the scope
-    Id(BatchId),
-}
-
-/// Create a temp tag with a given hash and format
-#[derive(Debug, Serialize, Deserialize)]
-pub struct BatchCreateTempTagRequest {
-    /// Content to protect
-    pub content: HashAndFormat,
-    /// Batch to create the temp tag in
-    pub batch: BatchId,
-}
-
-/// Write a blob from a byte stream
-#[derive(Serialize, Deserialize, Debug)]
-pub struct BatchAddStreamRequest {
-    /// What format to use for the blob
-    pub format: BlobFormat,
-    /// Batch to create the temp tag in
-    pub batch: BatchId,
-}
-
-/// Write a blob from a byte stream
-#[derive(Serialize, Deserialize, Debug)]
-pub enum BatchAddStreamUpdate {
-    /// A chunk of stream data
-    Chunk(Bytes),
-    /// Abort the request due to an error on the client side
-    Abort,
-}
-
-/// Wrapper around [`AddProgress`].
-#[allow(missing_docs)]
-#[derive(Debug, Serialize, Deserialize)]
-pub enum BatchAddStreamResponse {
-    Abort(RpcError),
-    OutboardProgress { offset: u64 },
-    Result { hash: Hash },
-}
-
-/// Write a blob from a byte stream
-#[derive(Serialize, Deserialize, Debug)]
-pub struct BatchAddPathRequest {
-    /// The path to the data to provide.
-    pub path: PathBuf,
-    /// Add the data in place
-    pub import_mode: ImportMode,
-    /// What format to use for the blob
-    pub format: BlobFormat,
-    /// Batch to create the temp tag in
-    pub batch: BatchId,
-}
-
-/// Response to a batch add path request
-#[derive(Serialize, Deserialize, Debug)]
-pub struct BatchAddPathResponse(pub BatchAddPathProgress);
diff --git a/src/rpc/proto/tags.rs b/src/rpc/proto/tags.rs
deleted file mode 100644
index f30547fa7..000000000
--- a/src/rpc/proto/tags.rs
+++ /dev/null
@@ -1,124 +0,0 @@
-//! Tags RPC protocol
-use nested_enum_utils::enum_conversions;
-use quic_rpc_derive::rpc_requests;
-use serde::{Deserialize, Serialize};
-
-use super::{RpcResult, RpcService};
-use crate::{
-    net_protocol::BatchId,
-    rpc::client::tags::{DeleteOptions, ListOptions, TagInfo},
-    HashAndFormat, Tag,
-};
-
-#[allow(missing_docs)]
-#[derive(strum::Display, Debug, Serialize, Deserialize)]
-#[enum_conversions(super::Request)]
-#[rpc_requests(RpcService)]
-pub enum Request {
-    #[rpc(response = RpcResult<Tag>)]
-    Create(CreateRequest),
-    #[rpc(response = RpcResult<()>)]
-    Set(SetRequest),
-    #[rpc(response = RpcResult<()>)]
-    Rename(RenameRequest),
-    #[rpc(response = RpcResult<()>)]
-    DeleteTag(DeleteRequest),
-    #[server_streaming(response = TagInfo)]
-    ListTags(ListRequest),
-}
-
-#[allow(missing_docs)]
-#[derive(strum::Display, Debug, Serialize, Deserialize)]
-#[enum_conversions(super::Response)]
-pub enum Response {
-    Create(RpcResult<Tag>),
-    ListTags(TagInfo),
-    DeleteTag(RpcResult<()>),
-}
-
-/// Determine how to sync the db after a modification operation
-#[derive(Debug, Serialize, Deserialize, Default)]
-pub enum SyncMode {
-    /// Fully sync the db
-    #[default]
-    Full,
-    /// Do not sync the db
-    None,
-}
-
-/// Create a tag
-#[derive(Debug, Serialize, Deserialize)]
-pub struct CreateRequest {
-    /// Value of the tag
-    pub value: HashAndFormat,
-    /// Batch to use, none for global
-    pub batch: Option<BatchId>,
-    /// Sync mode
-    pub sync: SyncMode,
-}
-
-/// Set or delete a tag
-#[derive(Debug, Serialize, Deserialize)]
-pub struct SetRequest {
-    /// Name of the tag
-    pub name: Tag,
-    /// Value of the tag
-    pub value: HashAndFormat,
-    /// Batch to use, none for global
-    pub batch: Option<BatchId>,
-    /// Sync mode
-    pub sync: SyncMode,
-}
-
-/// List all collections
-///
-/// Lists all collections that have been explicitly added to the database.
-#[derive(Debug, Serialize, Deserialize)]
-pub struct ListRequest {
-    /// List raw tags
-    pub raw: bool,
-    /// List hash seq tags
-    pub hash_seq: bool,
-    /// From tag (inclusive)
-    pub from: Option<Tag>,
-    /// To tag (exclusive)
-    pub to: Option<Tag>,
-}
-
-impl From<ListOptions> for ListRequest {
-    fn from(options: ListOptions) -> Self {
-        Self {
-            raw: options.raw,
-            hash_seq: options.hash_seq,
-            from: options.from,
-            to: options.to,
-        }
-    }
-}
-
-/// Delete a tag
-#[derive(Debug, Serialize, Deserialize)]
-pub struct DeleteRequest {
-    /// From tag (inclusive)
-    pub from: Option<Tag>,
-    /// To tag (exclusive)
-    pub to: Option<Tag>,
-}
-
-impl From<DeleteOptions> for DeleteRequest {
-    fn from(options: DeleteOptions) -> Self {
-        Self {
-            from: options.from,
-            to: options.to,
-        }
-    }
-}
-
-/// Rename a tag atomically
-#[derive(Debug, Serialize, Deserialize)]
-pub struct RenameRequest {
-    /// Old tag name
-    pub from: Tag,
-    /// New tag name
-    pub to: Tag,
-}
diff --git a/src/store.rs b/src/store.rs
deleted file mode 100644
index 3030d55b3..000000000
--- a/src/store.rs
+++ /dev/null
@@ -1,96 +0,0 @@
-//! Implementations of blob stores
-use crate::{BlobFormat, Hash, HashAndFormat};
-
-#[cfg(feature = "fs-store")]
-mod bao_file;
-pub mod mem;
-mod mutable_mem_storage;
-pub mod readonly_mem;
-
-#[cfg(feature = "fs-store")]
-pub mod fs;
-
-mod traits;
-use tracing::warn;
-pub use traits::*;
-
-/// Create a 16 byte unique ID.
-fn new_uuid() -> [u8; 16] {
-    use rand::Rng;
-    rand::thread_rng().gen::<[u8; 16]>()
-}
-
-/// Create temp file name based on a 16 byte UUID.
-fn temp_name() -> String {
-    format!("{}.temp", hex::encode(new_uuid()))
-}
-
-#[derive(Debug, Default, Clone)]
-struct TempCounters {
-    /// number of raw temp tags for a hash
-    raw: u64,
-    /// number of hash seq temp tags for a hash
-    hash_seq: u64,
-}
-
-impl TempCounters {
-    fn counter(&mut self, format: BlobFormat) -> &mut u64 {
-        match format {
-            BlobFormat::Raw => &mut self.raw,
-            BlobFormat::HashSeq => &mut self.hash_seq,
-        }
-    }
-
-    fn inc(&mut self, format: BlobFormat) {
-        let counter = self.counter(format);
-        *counter = counter.checked_add(1).unwrap();
-    }
-
-    fn dec(&mut self, format: BlobFormat) {
-        let counter = self.counter(format);
-        *counter = counter.saturating_sub(1);
-    }
-
-    fn is_empty(&self) -> bool {
-        self.raw == 0 && self.hash_seq == 0
-    }
-}
-
-#[derive(Debug, Clone, Default)]
-struct TempCounterMap(std::collections::BTreeMap<Hash, TempCounters>);
-
-impl TempCounterMap {
-    fn inc(&mut self, value: &HashAndFormat) {
-        let HashAndFormat { hash, format } = value;
-        self.0.entry(*hash).or_default().inc(*format)
-    }
-
-    fn dec(&mut self, value: &HashAndFormat) {
-        let HashAndFormat { hash, format } = value;
-        let Some(counters) = self.0.get_mut(hash) else {
-            warn!("Decrementing non-existent temp tag");
-            return;
-        };
-        counters.dec(*format);
-        if counters.is_empty() {
-            self.0.remove(hash);
-        }
-    }
-
-    fn contains(&self, hash: &Hash) -> bool {
-        self.0.contains_key(hash)
-    }
-
-    fn keys(&self) -> impl Iterator<Item = HashAndFormat> {
-        let mut res = Vec::new();
-        for (k, v) in self.0.iter() {
-            if v.raw > 0 {
-                res.push(HashAndFormat::raw(*k));
-            }
-            if v.hash_seq > 0 {
-                res.push(HashAndFormat::hash_seq(*k));
-            }
-        }
-        res.into_iter()
-    }
-}
diff --git a/src/store/bao_file.rs b/src/store/bao_file.rs
deleted file mode 100644
index c06328669..000000000
--- a/src/store/bao_file.rs
+++ /dev/null
@@ -1,1042 +0,0 @@
-//! An implementation of a bao file, meaning some data blob with associated
-//! outboard.
-//!
-//! Compared to just a pair of (data, outboard), this implementation also works
-//! when both the data and the outboard is incomplete, and not even the size
-//! is fully known.
-//!
-//! There is a full in memory implementation, and an implementation that uses
-//! the file system for the data, outboard, and sizes file. There is also a
-//! combined implementation that starts in memory and switches to file when
-//! the memory limit is reached.
-use std::{
-    fs::{File, OpenOptions},
-    io,
-    ops::{Deref, DerefMut},
-    path::{Path, PathBuf},
-    sync::{Arc, RwLock, Weak},
-};
-
-use bao_tree::{
-    io::{
-        fsm::BaoContentItem,
-        outboard::PreOrderOutboard,
-        sync::{ReadAt, WriteAt},
-    },
-    BaoTree,
-};
-use bytes::{Bytes, BytesMut};
-use derive_more::Debug;
-use iroh_io::AsyncSliceReader;
-
-use super::mutable_mem_storage::{MutableMemStorage, SizeInfo};
-use crate::{
-    store::BaoBatchWriter,
-    util::{get_limited_slice, MemOrFile, SparseMemFile},
-    Hash, IROH_BLOCK_SIZE,
-};
-
-/// Data files are stored in 3 files. The data file, the outboard file,
-/// and a sizes file. The sizes file contains the size that the remote side told us
-/// when writing each data block.
-///
-/// For complete data files, the sizes file is not needed, since you can just
-/// use the size of the data file.
-///
-/// For files below the chunk size, the outboard file is not needed, since
-/// there is only one leaf, and the outboard file is empty.
-struct DataPaths {
-    /// The data file. Size is determined by the chunk with the highest offset
-    /// that has been written.
-    ///
-    /// Gaps will be filled with zeros.
-    data: PathBuf,
-    /// The outboard file. This is *without* the size header, since that is not
-    /// known for partial files.
-    ///
-    /// The size of the outboard file is therefore a multiple of a hash pair
-    /// (64 bytes).
-    ///
-    /// The naming convention is to use obao for pre order traversal and oboa
-    /// for post order traversal. The log2 of the chunk group size is appended,
-    /// so for the default chunk group size in iroh of 4, the file extension
-    /// is .obao4.
-    outboard: PathBuf,
-    /// The sizes file. This is a file with 8 byte sizes for each chunk group.
-    /// The naming convention is to prepend the log2 of the chunk group size,
-    /// so for the default chunk group size in iroh of 4, the file extension
-    /// is .sizes4.
-    ///
-    /// The traversal order is not relevant for the sizes file, since it is
-    /// about the data chunks, not the hash pairs.
-    sizes: PathBuf,
-}
-
-/// Storage for complete blobs. There is no longer any uncertainty about the
-/// size, so we don't need a sizes file.
-///
-/// Writing is not possible but also not needed, since the file is complete.
-/// This covers all combinations of data and outboard being in memory or on
-/// disk.
-///
-/// For the memory variant, it does reading in a zero copy way, since storage
-/// is already a `Bytes`.
-#[derive(Default, derive_more::Debug)]
-pub struct CompleteStorage {
-    /// data part, which can be in memory or on disk.
-    #[debug("{:?}", data.as_ref().map_mem(|x| x.len()))]
-    pub data: MemOrFile<Bytes, (File, u64)>,
-    /// outboard part, which can be in memory or on disk.
-    #[debug("{:?}", outboard.as_ref().map_mem(|x| x.len()))]
-    pub outboard: MemOrFile<Bytes, (File, u64)>,
-}
-
-impl CompleteStorage {
-    /// Read from the data file at the given offset, until end of file or max bytes.
-    pub fn read_data_at(&self, offset: u64, len: usize) -> Bytes {
-        match &self.data {
-            MemOrFile::Mem(mem) => get_limited_slice(mem, offset, len),
-            MemOrFile::File((file, _size)) => read_to_end(file, offset, len).unwrap(),
-        }
-    }
-
-    /// Read from the outboard file at the given offset, until end of file or max bytes.
-    pub fn read_outboard_at(&self, offset: u64, len: usize) -> Bytes {
-        match &self.outboard {
-            MemOrFile::Mem(mem) => get_limited_slice(mem, offset, len),
-            MemOrFile::File((file, _size)) => read_to_end(file, offset, len).unwrap(),
-        }
-    }
-
-    /// The size of the data file.
-    pub fn data_size(&self) -> u64 {
-        match &self.data {
-            MemOrFile::Mem(mem) => mem.len() as u64,
-            MemOrFile::File((_file, size)) => *size,
-        }
-    }
-
-    /// The size of the outboard file.
-    pub fn outboard_size(&self) -> u64 {
-        match &self.outboard {
-            MemOrFile::Mem(mem) => mem.len() as u64,
-            MemOrFile::File((_file, size)) => *size,
-        }
-    }
-}
-
-/// Create a file for reading and writing, but *without* truncating the existing
-/// file.
-fn create_read_write(path: impl AsRef<Path>) -> io::Result<File> {
-    OpenOptions::new()
-        .read(true)
-        .write(true)
-        .create(true)
-        .truncate(false)
-        .open(path)
-}
-
-/// Read from the given file at the given offset, until end of file or max bytes.
-fn read_to_end(file: impl ReadAt, offset: u64, max: usize) -> io::Result<Bytes> {
-    let mut res = BytesMut::new();
-    let mut buf = [0u8; 4096];
-    let mut remaining = max;
-    let mut offset = offset;
-    while remaining > 0 {
-        let end = buf.len().min(remaining);
-        let read = file.read_at(offset, &mut buf[..end])?;
-        if read == 0 {
-            // eof
-            break;
-        }
-        res.extend_from_slice(&buf[..read]);
-        offset += read as u64;
-        remaining -= read;
-    }
-    Ok(res.freeze())
-}
-
-fn max_offset(batch: &[BaoContentItem]) -> u64 {
-    batch
-        .iter()
-        .filter_map(|item| match item {
-            BaoContentItem::Leaf(leaf) => {
-                let len = leaf.data.len().try_into().unwrap();
-                let end = leaf
-                    .offset
-                    .checked_add(len)
-                    .expect("u64 overflow for leaf end");
-                Some(end)
-            }
-            _ => None,
-        })
-        .max()
-        .unwrap_or(0)
-}
-
-/// A file storage for an incomplete bao file.
-#[derive(Debug)]
-pub struct FileStorage {
-    data: std::fs::File,
-    outboard: std::fs::File,
-    sizes: std::fs::File,
-}
-
-impl FileStorage {
-    /// Split into data, outboard and sizes files.
-    pub fn into_parts(self) -> (File, File, File) {
-        (self.data, self.outboard, self.sizes)
-    }
-
-    fn current_size(&self) -> io::Result<u64> {
-        let len = self.sizes.metadata()?.len();
-        if len < 8 {
-            Ok(0)
-        } else {
-            // todo: use the last full u64 in case the sizes file is not a multiple of 8
-            // bytes. Not sure how that would happen, but we should handle it.
-            let mut buf = [0u8; 8];
-            self.sizes.read_exact_at(len - 8, &mut buf)?;
-            Ok(u64::from_le_bytes(buf))
-        }
-    }
-
-    fn write_batch(&mut self, size: u64, batch: &[BaoContentItem]) -> io::Result<()> {
-        let tree = BaoTree::new(size, IROH_BLOCK_SIZE);
-        for item in batch {
-            match item {
-                BaoContentItem::Parent(parent) => {
-                    if let Some(offset) = tree.pre_order_offset(parent.node) {
-                        let o0 = offset * 64;
-                        self.outboard
-                            .write_all_at(o0, parent.pair.0.as_bytes().as_slice())?;
-                        self.outboard
-                            .write_all_at(o0 + 32, parent.pair.1.as_bytes().as_slice())?;
-                    }
-                }
-                BaoContentItem::Leaf(leaf) => {
-                    let o0 = leaf.offset;
-                    // divide by chunk size, multiply by 8
-                    let index = (leaf.offset >> (tree.block_size().chunk_log() + 10)) << 3;
-                    tracing::trace!(
-                        "write_batch f={:?} o={} l={}",
-                        self.data,
-                        o0,
-                        leaf.data.len()
-                    );
-                    self.data.write_all_at(o0, leaf.data.as_ref())?;
-                    let size = tree.size();
-                    self.sizes.write_all_at(index, &size.to_le_bytes())?;
-                }
-            }
-        }
-        Ok(())
-    }
-
-    fn read_data_at(&self, offset: u64, len: usize) -> io::Result<Bytes> {
-        read_to_end(&self.data, offset, len)
-    }
-
-    fn read_outboard_at(&self, offset: u64, len: usize) -> io::Result<Bytes> {
-        read_to_end(&self.outboard, offset, len)
-    }
-}
-
-/// The storage for a bao file. This can be either in memory or on disk.
-#[derive(Debug)]
-pub(crate) enum BaoFileStorage {
-    /// The entry is incomplete and in memory.
-    ///
-    /// Since it is incomplete, it must be writeable.
-    ///
-    /// This is used mostly for tiny entries, <= 16 KiB. But in principle it
-    /// can be used for larger sizes.
-    ///
-    /// Incomplete mem entries are *not* persisted at all. So if the store
-    /// crashes they will be gone.
-    IncompleteMem(MutableMemStorage),
-    /// The entry is incomplete and on disk.
-    IncompleteFile(FileStorage),
-    /// The entry is complete. Outboard and data can come from different sources
-    /// (memory or file).
-    ///
-    /// Writing to this is a no-op, since it is already complete.
-    Complete(CompleteStorage),
-}
-
-impl Default for BaoFileStorage {
-    fn default() -> Self {
-        BaoFileStorage::Complete(Default::default())
-    }
-}
-
-impl BaoFileStorage {
-    /// Take the storage out, leaving an empty storage in its place.
-    ///
-    /// Be careful to put something back in its place, or you will lose data.
-    #[cfg(feature = "fs-store")]
-    pub fn take(&mut self) -> Self {
-        std::mem::take(self)
-    }
-
-    /// Create a new mutable mem storage.
-    pub fn incomplete_mem() -> Self {
-        Self::IncompleteMem(Default::default())
-    }
-
-    /// Call sync_all on all the files.
-    fn sync_all(&self) -> io::Result<()> {
-        match self {
-            Self::Complete(_) => Ok(()),
-            Self::IncompleteMem(_) => Ok(()),
-            Self::IncompleteFile(file) => {
-                file.data.sync_all()?;
-                file.outboard.sync_all()?;
-                file.sizes.sync_all()?;
-                Ok(())
-            }
-        }
-    }
-
-    /// True if the storage is in memory.
-    pub fn is_mem(&self) -> bool {
-        match self {
-            Self::IncompleteMem(_) => true,
-            Self::IncompleteFile(_) => false,
-            Self::Complete(c) => c.data.is_mem() && c.outboard.is_mem(),
-        }
-    }
-}
-
-/// A weak reference to a bao file handle.
-#[derive(Debug, Clone)]
-pub struct BaoFileHandleWeak(Weak<BaoFileHandleInner>);
-
-impl BaoFileHandleWeak {
-    /// Upgrade to a strong reference if possible.
-    pub fn upgrade(&self) -> Option<BaoFileHandle> {
-        self.0.upgrade().map(BaoFileHandle)
-    }
-
-    /// True if the handle is still live (has strong references)
-    pub fn is_live(&self) -> bool {
-        self.0.strong_count() > 0
-    }
-}
-
-/// The inner part of a bao file handle.
-#[derive(Debug)]
-pub struct BaoFileHandleInner {
-    pub(crate) storage: RwLock<BaoFileStorage>,
-    config: Arc<BaoFileConfig>,
-    hash: Hash,
-}
-
-/// A cheaply cloneable handle to a bao file, including the hash and the configuration.
-#[derive(Debug, Clone, derive_more::Deref)]
-pub struct BaoFileHandle(Arc<BaoFileHandleInner>);
-
-pub(crate) type CreateCb = Arc<dyn Fn(&Hash) -> io::Result<()> + Send + Sync>;
-
-/// Configuration for the deferred batch writer. It will start writing to memory,
-/// and then switch to a file when the memory limit is reached.
-#[derive(derive_more::Debug, Clone)]
-pub struct BaoFileConfig {
-    /// Directory to store files in. Only used when memory limit is reached.
-    dir: Arc<PathBuf>,
-    /// Maximum data size (inclusive) before switching to file mode.
-    max_mem: usize,
-    /// Callback to call when we switch to file mode.
-    ///
-    /// Todo: make this async.
-    #[debug("{:?}", on_file_create.as_ref().map(|_| ()))]
-    on_file_create: Option<CreateCb>,
-}
-
-impl BaoFileConfig {
-    /// Create a new deferred batch writer configuration.
-    pub fn new(dir: Arc<PathBuf>, max_mem: usize, on_file_create: Option<CreateCb>) -> Self {
-        Self {
-            dir,
-            max_mem,
-            on_file_create,
-        }
-    }
-
-    /// Get the paths for a hash.
-    fn paths(&self, hash: &Hash) -> DataPaths {
-        DataPaths {
-            data: self.dir.join(format!("{}.data", hash.to_hex())),
-            outboard: self.dir.join(format!("{}.obao4", hash.to_hex())),
-            sizes: self.dir.join(format!("{}.sizes4", hash.to_hex())),
-        }
-    }
-}
-
-/// A reader for a bao file, reading just the data.
-#[derive(Debug)]
-pub struct DataReader(Option<BaoFileHandle>);
-
-async fn with_storage<T, P, F>(opt: &mut Option<BaoFileHandle>, no_io: P, f: F) -> io::Result<T>
-where
-    P: Fn(&BaoFileStorage) -> bool + Send + 'static,
-    F: FnOnce(&BaoFileStorage) -> io::Result<T> + Send + 'static,
-    T: Send + 'static,
-{
-    let handle = opt
-        .take()
-        .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "deferred batch busy"))?;
-    // if we can get the lock immediately, and we are in memory mode, we can
-    // avoid spawning a task.
-    if let Ok(storage) = handle.storage.try_read() {
-        if no_io(&storage) {
-            let res = f(&storage);
-            // clone because for some reason even when we drop storage, the
-            // borrow checker still thinks handle is borrowed.
-            *opt = Some(handle.clone());
-            return res;
-        }
-    };
-    // otherwise, we have to spawn a task.
-    let (handle, res) = tokio::task::spawn_blocking(move || {
-        let storage = handle.storage.read().unwrap();
-        let res = f(storage.deref());
-        drop(storage);
-        (handle, res)
-    })
-    .await
-    .expect("spawn_blocking failed");
-    *opt = Some(handle);
-    res
-}
-
-impl AsyncSliceReader for DataReader {
-    async fn read_at(&mut self, offset: u64, len: usize) -> io::Result<Bytes> {
-        with_storage(
-            &mut self.0,
-            BaoFileStorage::is_mem,
-            move |storage| match storage {
-                BaoFileStorage::Complete(mem) => Ok(mem.read_data_at(offset, len)),
-                BaoFileStorage::IncompleteMem(mem) => Ok(mem.read_data_at(offset, len)),
-                BaoFileStorage::IncompleteFile(file) => file.read_data_at(offset, len),
-            },
-        )
-        .await
-    }
-
-    async fn size(&mut self) -> io::Result<u64> {
-        with_storage(
-            &mut self.0,
-            BaoFileStorage::is_mem,
-            move |storage| match storage {
-                BaoFileStorage::Complete(mem) => Ok(mem.data_size()),
-                BaoFileStorage::IncompleteMem(mem) => Ok(mem.data.len() as u64),
-                BaoFileStorage::IncompleteFile(file) => file.data.metadata().map(|m| m.len()),
-            },
-        )
-        .await
-    }
-}
-
-/// A reader for the outboard part of a bao file.
-#[derive(Debug)]
-pub struct OutboardReader(Option<BaoFileHandle>);
-
-impl AsyncSliceReader for OutboardReader {
-    async fn read_at(&mut self, offset: u64, len: usize) -> io::Result<Bytes> {
-        with_storage(
-            &mut self.0,
-            BaoFileStorage::is_mem,
-            move |storage| match storage {
-                BaoFileStorage::Complete(mem) => Ok(mem.read_outboard_at(offset, len)),
-                BaoFileStorage::IncompleteMem(mem) => Ok(mem.read_outboard_at(offset, len)),
-                BaoFileStorage::IncompleteFile(file) => file.read_outboard_at(offset, len),
-            },
-        )
-        .await
-    }
-
-    async fn size(&mut self) -> io::Result<u64> {
-        with_storage(
-            &mut self.0,
-            BaoFileStorage::is_mem,
-            move |storage| match storage {
-                BaoFileStorage::Complete(mem) => Ok(mem.outboard_size()),
-                BaoFileStorage::IncompleteMem(mem) => Ok(mem.outboard.len() as u64),
-                BaoFileStorage::IncompleteFile(file) => file.outboard.metadata().map(|m| m.len()),
-            },
-        )
-        .await
-    }
-}
-
-enum HandleChange {
-    None,
-    MemToFile,
-    // later: size verified
-}
-
-impl BaoFileHandle {
-    /// Create a new bao file handle.
-    ///
-    /// This will create a new file handle with an empty memory storage.
-    /// Since there are very likely to be many of these, we use an arc rwlock
-    pub fn incomplete_mem(config: Arc<BaoFileConfig>, hash: Hash) -> Self {
-        let storage = BaoFileStorage::incomplete_mem();
-        Self(Arc::new(BaoFileHandleInner {
-            storage: RwLock::new(storage),
-            config,
-            hash,
-        }))
-    }
-
-    /// Create a new bao file handle with a partial file.
-    pub fn incomplete_file(config: Arc<BaoFileConfig>, hash: Hash) -> io::Result<Self> {
-        let paths = config.paths(&hash);
-        let storage = BaoFileStorage::IncompleteFile(FileStorage {
-            data: create_read_write(&paths.data)?,
-            outboard: create_read_write(&paths.outboard)?,
-            sizes: create_read_write(&paths.sizes)?,
-        });
-        Ok(Self(Arc::new(BaoFileHandleInner {
-            storage: RwLock::new(storage),
-            config,
-            hash,
-        })))
-    }
-
-    /// Create a new complete bao file handle.
-    pub fn new_complete(
-        config: Arc<BaoFileConfig>,
-        hash: Hash,
-        data: MemOrFile<Bytes, (File, u64)>,
-        outboard: MemOrFile<Bytes, (File, u64)>,
-    ) -> Self {
-        let storage = BaoFileStorage::Complete(CompleteStorage { data, outboard });
-        Self(Arc::new(BaoFileHandleInner {
-            storage: RwLock::new(storage),
-            config,
-            hash,
-        }))
-    }
-
-    /// Transform the storage in place. If the transform fails, the storage will
-    /// be an immutable empty storage.
-    #[cfg(feature = "fs-store")]
-    pub(crate) fn transform(
-        &self,
-        f: impl FnOnce(BaoFileStorage) -> io::Result<BaoFileStorage>,
-    ) -> io::Result<()> {
-        let mut lock = self.storage.write().unwrap();
-        let storage = lock.take();
-        *lock = f(storage)?;
-        Ok(())
-    }
-
-    /// True if the file is complete.
-    pub fn is_complete(&self) -> bool {
-        matches!(
-            self.storage.read().unwrap().deref(),
-            BaoFileStorage::Complete(_)
-        )
-    }
-
-    /// An AsyncSliceReader for the data file.
-    ///
-    /// Caution: this is a reader for the unvalidated data file. Reading this
-    /// can produce data that does not match the hash.
-    pub fn data_reader(&self) -> DataReader {
-        DataReader(Some(self.clone()))
-    }
-
-    /// An AsyncSliceReader for the outboard file.
-    ///
-    /// The outboard file is used to validate the data file. It is not guaranteed
-    /// to be complete.
-    pub fn outboard_reader(&self) -> OutboardReader {
-        OutboardReader(Some(self.clone()))
-    }
-
-    /// The most precise known total size of the data file.
-    pub fn current_size(&self) -> io::Result<u64> {
-        match self.storage.read().unwrap().deref() {
-            BaoFileStorage::Complete(mem) => Ok(mem.data_size()),
-            BaoFileStorage::IncompleteMem(mem) => Ok(mem.current_size()),
-            BaoFileStorage::IncompleteFile(file) => file.current_size(),
-        }
-    }
-
-    /// The outboard for the file.
-    pub fn outboard(&self) -> io::Result<PreOrderOutboard<OutboardReader>> {
-        let root = self.hash.into();
-        let tree = BaoTree::new(self.current_size()?, IROH_BLOCK_SIZE);
-        let outboard = self.outboard_reader();
-        Ok(PreOrderOutboard {
-            root,
-            tree,
-            data: outboard,
-        })
-    }
-
-    /// The hash of the file.
-    pub fn hash(&self) -> Hash {
-        self.hash
-    }
-
-    /// Create a new writer from the handle.
-    pub fn writer(&self) -> BaoFileWriter {
-        BaoFileWriter(Some(self.clone()))
-    }
-
-    /// This is the synchronous impl for writing a batch.
-    fn write_batch(&self, size: u64, batch: &[BaoContentItem]) -> io::Result<HandleChange> {
-        let mut storage = self.storage.write().unwrap();
-        match storage.deref_mut() {
-            BaoFileStorage::IncompleteMem(mem) => {
-                // check if we need to switch to file mode, otherwise write to memory
-                if max_offset(batch) <= self.config.max_mem as u64 {
-                    mem.write_batch(size, batch)?;
-                    Ok(HandleChange::None)
-                } else {
-                    // create the paths. This allocates 3 pathbufs, so we do it
-                    // only when we need to.
-                    let paths = self.config.paths(&self.hash);
-                    // *first* switch to file mode, *then* write the batch.
-                    //
-                    // otherwise we might allocate a lot of memory if we get
-                    // a write at the end of a very large file.
-                    let mut file_batch = mem.persist(paths)?;
-                    file_batch.write_batch(size, batch)?;
-                    *storage = BaoFileStorage::IncompleteFile(file_batch);
-                    Ok(HandleChange::MemToFile)
-                }
-            }
-            BaoFileStorage::IncompleteFile(file) => {
-                // already in file mode, just write the batch
-                file.write_batch(size, batch)?;
-                Ok(HandleChange::None)
-            }
-            BaoFileStorage::Complete(_) => {
-                // we are complete, so just ignore the write
-                // unless there is a bug, this would just write the exact same data
-                Ok(HandleChange::None)
-            }
-        }
-    }
-
-    /// Downgrade to a weak reference.
-    pub fn downgrade(&self) -> BaoFileHandleWeak {
-        BaoFileHandleWeak(Arc::downgrade(&self.0))
-    }
-}
-
-impl SizeInfo {
-    /// Persist into a file where each chunk has its own slot.
-    pub fn persist(&self, mut target: impl WriteAt) -> io::Result<()> {
-        let size_offset = (self.offset >> IROH_BLOCK_SIZE.chunk_log()) << 3;
-        target.write_all_at(size_offset, self.size.to_le_bytes().as_slice())?;
-        Ok(())
-    }
-
-    /// Convert to a vec in slot format.
-    pub fn to_vec(&self) -> Vec<u8> {
-        let mut res = Vec::new();
-        self.persist(&mut res).expect("io error writing to vec");
-        res
-    }
-}
-
-impl MutableMemStorage {
-    /// Persist the batch to disk, creating a FileBatch.
-    fn persist(&self, paths: DataPaths) -> io::Result<FileStorage> {
-        let mut data = create_read_write(&paths.data)?;
-        let mut outboard = create_read_write(&paths.outboard)?;
-        let mut sizes = create_read_write(&paths.sizes)?;
-        self.data.persist(&mut data)?;
-        self.outboard.persist(&mut outboard)?;
-        self.sizes.persist(&mut sizes)?;
-        data.sync_all()?;
-        outboard.sync_all()?;
-        sizes.sync_all()?;
-        Ok(FileStorage {
-            data,
-            outboard,
-            sizes,
-        })
-    }
-
-    /// Get the parts data, outboard and sizes
-    pub fn into_parts(self) -> (SparseMemFile, SparseMemFile, SizeInfo) {
-        (self.data, self.outboard, self.sizes)
-    }
-}
-
-/// This is finally the thing for which we can implement BaoPairMut.
-///
-/// It is a BaoFileHandle wrapped in an Option, so that we can take it out
-/// in the future.
-#[derive(Debug)]
-pub struct BaoFileWriter(Option<BaoFileHandle>);
-
-impl BaoBatchWriter for BaoFileWriter {
-    async fn write_batch(&mut self, size: u64, batch: Vec<BaoContentItem>) -> std::io::Result<()> {
-        let Some(handle) = self.0.take() else {
-            return Err(io::Error::new(io::ErrorKind::Other, "deferred batch busy"));
-        };
-        let (handle, change) = tokio::task::spawn_blocking(move || {
-            let change = handle.write_batch(size, &batch);
-            (handle, change)
-        })
-        .await
-        .expect("spawn_blocking failed");
-        match change? {
-            HandleChange::None => {}
-            HandleChange::MemToFile => {
-                if let Some(cb) = handle.config.on_file_create.as_ref() {
-                    cb(&handle.hash)?;
-                }
-            }
-        }
-        self.0 = Some(handle);
-        Ok(())
-    }
-
-    async fn sync(&mut self) -> io::Result<()> {
-        let Some(handle) = self.0.take() else {
-            return Err(io::Error::new(io::ErrorKind::Other, "deferred batch busy"));
-        };
-        let (handle, res) = tokio::task::spawn_blocking(move || {
-            let res = handle.storage.write().unwrap().sync_all();
-            (handle, res)
-        })
-        .await
-        .expect("spawn_blocking failed");
-        self.0 = Some(handle);
-        res
-    }
-}
-
-#[cfg(test)]
-pub mod test_support {
-    use std::{future::Future, io::Cursor, ops::Range};
-
-    use bao_tree::{
-        io::{
-            fsm::{ResponseDecoder, ResponseDecoderNext},
-            outboard::PostOrderMemOutboard,
-            round_up_to_chunks,
-            sync::encode_ranges_validated,
-        },
-        BlockSize, ChunkRanges,
-    };
-    use futures_lite::{Stream, StreamExt};
-    use iroh_io::AsyncStreamReader;
-    use rand::RngCore;
-    use range_collections::RangeSet2;
-
-    use super::*;
-    use crate::util::limited_range;
-
-    pub const IROH_BLOCK_SIZE: BlockSize = BlockSize::from_chunk_log(4);
-
-    /// Decode a response into a batch file writer.
-    pub async fn decode_response_into_batch<R, W>(
-        root: Hash,
-        block_size: BlockSize,
-        ranges: ChunkRanges,
-        mut encoded: R,
-        mut target: W,
-    ) -> io::Result<()>
-    where
-        R: AsyncStreamReader,
-        W: BaoBatchWriter,
-    {
-        let size = encoded.read::<8>().await?;
-        let size = u64::from_le_bytes(size);
-        let mut reading =
-            ResponseDecoder::new(root.into(), ranges, BaoTree::new(size, block_size), encoded);
-        let mut stack = Vec::new();
-        loop {
-            let item = match reading.next().await {
-                ResponseDecoderNext::Done(_reader) => break,
-                ResponseDecoderNext::More((next, item)) => {
-                    reading = next;
-                    item?
-                }
-            };
-            match item {
-                BaoContentItem::Parent(_) => {
-                    stack.push(item);
-                }
-                BaoContentItem::Leaf(_) => {
-                    // write a batch every time we see a leaf
-                    // the last item will be a leaf.
-                    stack.push(item);
-                    target.write_batch(size, std::mem::take(&mut stack)).await?;
-                }
-            }
-        }
-        assert!(stack.is_empty(), "last item should be a leaf");
-        Ok(())
-    }
-
-    pub fn random_test_data(size: usize) -> Vec<u8> {
-        let mut rand = rand::thread_rng();
-        let mut res = vec![0u8; size];
-        rand.fill_bytes(&mut res);
-        res
-    }
-
-    /// Take some data and encode it
-    pub fn simulate_remote(data: &[u8]) -> (Hash, Cursor<Bytes>) {
-        let outboard = bao_tree::io::outboard::PostOrderMemOutboard::create(data, IROH_BLOCK_SIZE);
-        let size = data.len() as u64;
-        let mut encoded = size.to_le_bytes().to_vec();
-        bao_tree::io::sync::encode_ranges_validated(
-            data,
-            &outboard,
-            &ChunkRanges::all(),
-            &mut encoded,
-        )
-        .unwrap();
-        let hash = outboard.root;
-        (hash.into(), Cursor::new(encoded.into()))
-    }
-
-    pub fn to_ranges(ranges: &[Range<u64>]) -> RangeSet2<u64> {
-        let mut range_set = RangeSet2::empty();
-        for range in ranges.as_ref().iter().cloned() {
-            range_set |= RangeSet2::from(range);
-        }
-        range_set
-    }
-
-    /// Simulate the send side, when asked to send bao encoded data for the given ranges.
-    pub fn make_wire_data(
-        data: &[u8],
-        ranges: impl AsRef<[Range<u64>]>,
-    ) -> (Hash, ChunkRanges, Vec<u8>) {
-        // compute a range set from the given ranges
-        let range_set = to_ranges(ranges.as_ref());
-        // round up to chunks
-        let chunk_ranges = round_up_to_chunks(&range_set);
-        // compute the outboard
-        let outboard = PostOrderMemOutboard::create(data, IROH_BLOCK_SIZE).flip();
-        let size = data.len() as u64;
-        let mut encoded = size.to_le_bytes().to_vec();
-        encode_ranges_validated(data, &outboard, &chunk_ranges, &mut encoded).unwrap();
-        (outboard.root.into(), chunk_ranges, encoded)
-    }
-
-    pub async fn validate(handle: &BaoFileHandle, original: &[u8], ranges: &[Range<u64>]) {
-        let mut r = handle.data_reader();
-        for range in ranges {
-            let start = range.start;
-            let len = (range.end - range.start).try_into().unwrap();
-            let data = &original[limited_range(start, len, original.len())];
-            let read = r.read_at(start, len).await.unwrap();
-            assert_eq!(data.len(), read.as_ref().len());
-            assert_eq!(data, read.as_ref());
-        }
-    }
-
-    /// Helper to simulate a slow request.
-    pub fn trickle(
-        data: &[u8],
-        mtu: usize,
-        delay: std::time::Duration,
-    ) -> impl Stream<Item = Bytes> {
-        let parts = data
-            .chunks(mtu)
-            .map(Bytes::copy_from_slice)
-            .collect::<Vec<_>>();
-        futures_lite::stream::iter(parts).then(move |part| async move {
-            tokio::time::sleep(delay).await;
-            part
-        })
-    }
-
-    pub async fn local<F>(f: F) -> F::Output
-    where
-        F: Future,
-    {
-        tokio::task::LocalSet::new().run_until(f).await
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::io::Write;
-
-    use bao_tree::{blake3, ChunkNum, ChunkRanges};
-    use futures_lite::StreamExt;
-    use iroh_io::TokioStreamReader;
-    use tests::test_support::{
-        decode_response_into_batch, local, make_wire_data, random_test_data, trickle, validate,
-    };
-    use tokio::task::JoinSet;
-
-    use super::*;
-    use crate::util::local_pool::LocalPool;
-
-    #[tokio::test]
-    async fn partial_downloads() {
-        local(async move {
-            let n = 1024 * 64u64;
-            let test_data = random_test_data(n as usize);
-            let temp_dir = tempfile::tempdir().unwrap();
-            let hash = blake3::hash(&test_data);
-            let handle = BaoFileHandle::incomplete_mem(
-                Arc::new(BaoFileConfig::new(
-                    Arc::new(temp_dir.as_ref().to_owned()),
-                    1024 * 16,
-                    None,
-                )),
-                hash.into(),
-            );
-            let mut tasks = JoinSet::new();
-            for i in 1..3 {
-                let file = handle.writer();
-                let range = (i * (n / 4))..((i + 1) * (n / 4));
-                println!("range: {:?}", range);
-                let (hash, chunk_ranges, wire_data) = make_wire_data(&test_data, &[range]);
-                let trickle = trickle(&wire_data, 1200, std::time::Duration::from_millis(10))
-                    .map(io::Result::Ok)
-                    .boxed();
-                let trickle = TokioStreamReader::new(tokio_util::io::StreamReader::new(trickle));
-                let _task = tasks.spawn_local(async move {
-                    decode_response_into_batch(hash, IROH_BLOCK_SIZE, chunk_ranges, trickle, file)
-                        .await
-                });
-            }
-            while let Some(res) = tasks.join_next().await {
-                res.unwrap().unwrap();
-            }
-            println!(
-                "len {:?} {:?}",
-                handle,
-                handle.data_reader().size().await.unwrap()
-            );
-            #[allow(clippy::single_range_in_vec_init)]
-            let ranges = [1024 * 16..1024 * 48];
-            validate(&handle, &test_data, &ranges).await;
-
-            // let ranges =
-            // let full_chunks = bao_tree::io::full_chunk_groups();
-            let mut encoded = Vec::new();
-            let ob = handle.outboard().unwrap();
-            encoded
-                .write_all(ob.tree.size().to_le_bytes().as_slice())
-                .unwrap();
-            bao_tree::io::fsm::encode_ranges_validated(
-                handle.data_reader(),
-                ob,
-                &ChunkRanges::from(ChunkNum(16)..ChunkNum(48)),
-                encoded,
-            )
-            .await
-            .unwrap();
-        })
-        .await;
-    }
-
-    #[tokio::test]
-    async fn concurrent_downloads() {
-        let n = 1024 * 32u64;
-        let test_data = random_test_data(n as usize);
-        let temp_dir = tempfile::tempdir().unwrap();
-        let hash = blake3::hash(&test_data);
-        let handle = BaoFileHandle::incomplete_mem(
-            Arc::new(BaoFileConfig::new(
-                Arc::new(temp_dir.as_ref().to_owned()),
-                1024 * 16,
-                None,
-            )),
-            hash.into(),
-        );
-        let local = LocalPool::default();
-        let mut tasks = Vec::new();
-        for i in 0..4 {
-            let file = handle.writer();
-            let range = (i * (n / 4))..((i + 1) * (n / 4));
-            println!("range: {:?}", range);
-            let (hash, chunk_ranges, wire_data) = make_wire_data(&test_data, &[range]);
-            let trickle = trickle(&wire_data, 1200, std::time::Duration::from_millis(10))
-                .map(io::Result::Ok)
-                .boxed();
-            let trickle = TokioStreamReader::new(tokio_util::io::StreamReader::new(trickle));
-            let task = local.spawn(move || async move {
-                decode_response_into_batch(hash, IROH_BLOCK_SIZE, chunk_ranges, trickle, file).await
-            });
-            tasks.push(task);
-        }
-        for task in tasks {
-            task.await.unwrap().unwrap();
-        }
-        println!(
-            "len {:?} {:?}",
-            handle,
-            handle.data_reader().size().await.unwrap()
-        );
-        #[allow(clippy::single_range_in_vec_init)]
-        let ranges = [0..n];
-        validate(&handle, &test_data, &ranges).await;
-
-        let mut encoded = Vec::new();
-        let ob = handle.outboard().unwrap();
-        encoded
-            .write_all(ob.tree.size().to_le_bytes().as_slice())
-            .unwrap();
-        bao_tree::io::fsm::encode_ranges_validated(
-            handle.data_reader(),
-            ob,
-            &ChunkRanges::all(),
-            encoded,
-        )
-        .await
-        .unwrap();
-    }
-
-    #[tokio::test]
-    async fn stay_in_mem() {
-        let test_data = random_test_data(1024 * 17);
-        #[allow(clippy::single_range_in_vec_init)]
-        let ranges = [0..test_data.len().try_into().unwrap()];
-        let (hash, chunk_ranges, wire_data) = make_wire_data(&test_data, &ranges);
-        println!("file len is {:?}", chunk_ranges);
-        let temp_dir = tempfile::tempdir().unwrap();
-        let handle = BaoFileHandle::incomplete_mem(
-            Arc::new(BaoFileConfig::new(
-                Arc::new(temp_dir.as_ref().to_owned()),
-                1024 * 16,
-                None,
-            )),
-            hash,
-        );
-        decode_response_into_batch(
-            hash,
-            IROH_BLOCK_SIZE,
-            chunk_ranges,
-            wire_data.as_slice(),
-            handle.writer(),
-        )
-        .await
-        .unwrap();
-        validate(&handle, &test_data, &ranges).await;
-
-        let mut encoded = Vec::new();
-        let ob = handle.outboard().unwrap();
-        encoded
-            .write_all(ob.tree.size().to_le_bytes().as_slice())
-            .unwrap();
-        bao_tree::io::fsm::encode_ranges_validated(
-            handle.data_reader(),
-            ob,
-            &ChunkRanges::all(),
-            encoded,
-        )
-        .await
-        .unwrap();
-        println!("{:?}", handle);
-    }
-}
diff --git a/src/store/fs.rs b/src/store/fs.rs
index 0628dc183..ff8bc900b 100644
--- a/src/store/fs.rs
+++ b/src/store/fs.rs
@@ -1,2678 +1,2091 @@
-//! redb backed storage
+//! # File based blob store.
 //!
-//! Data can get into the store in two ways:
+//! A file based blob store needs a writeable directory to work with.
 //!
-//! 1. import from local data
-//! 2. sync from a remote
+//! General design:
 //!
-//! These two cases are very different. In the first case, we have the data
-//! completely and don't know the hash yet. We compute the outboard and hash,
-//! and only then move/reference the data into the store.
+//! The file store consists of two actors.
 //!
-//! The entry for the hash comes into existence already complete.
+//! # The main actor
 //!
-//! In the second case, we know the hash, but don't have the data yet. We create
-//! a partial entry, and then request the data from the remote. This is the more
-//! complex case.
+//! The purpose of the main actor is to handle user commands and own a map of
+//! handles for hashes that are currently being worked on.
 //!
-//! Partial entries always start as pure in memory entries without a database
-//! entry. Only once we receive enough data, we convert them into a persistent
-//! partial entry. This is necessary because we can't trust the size given
-//! by the remote side before receiving data. It is also an optimization,
-//! because for small blobs it is not worth it to create a partial entry.
+//! It also owns tasks for ongoing import and export operations, as well as the
+//! database actor.
 //!
-//! A persistent partial entry is always stored as three files in the file
-//! system: The data file, the outboard file, and a sizes file that contains
-//! the most up to date information about the size of the data.
+//! Handling a command almost always involves either forwarding it to the
+//! database actor or creating a hash context and spawning a task.
 //!
-//! The redb database entry for a persistent partial entry does not contain
-//! any information about the size of the data until the size is exactly known.
+//! # The database actor
 //!
-//! Updating this information on each write would be too costly.
+//! The database actor is responsible for storing metadata about each hash,
+//! as well as inlined data and outboard data for small files.
 //!
-//! Marking a partial entry as complete is done from the outside. At this point
-//! the size is taken as validated. Depending on the size we decide whether to
-//! store data and outboard inline or to keep storing it in external files.
+//! In addition to the metadata, the database actor also stores tags.
 //!
-//! Data can get out of the store in two ways:
+//! # Tasks
 //!
-//! 1. the data and outboard of both partial and complete entries can be read at any time and
-//!    shared over the network. Only data that is complete will be shared, everything else will
-//!    lead to validation errors.
+//! Tasks do not return a result. They are responsible for sending an error
+//! to the requester if possible. Otherwise, just dropping the sender will
+//! also fail the receiver, but without a descriptive error message.
 //!
-//! 2. entries can be exported to the file system. This currently only works for complete entries.
+//! Tasks are usually implemented as an impl fn that does return a result,
+//! and a wrapper (named `..._task`) that just forwards the error, if any.
 //!
-//! Tables:
+//! That way you can use `?` syntax in the task implementation. The impl fns
+//! are also easier to test.
 //!
-//! The blobs table contains a mapping from hash to rough entry state.
-//! The inline_data table contains the actual data for complete entries.
-//! The inline_outboard table contains the actual outboard for complete entries.
-//! The tags table contains a mapping from tag to hash.
+//! # Context
 //!
-//! Design:
+//! The main actor holds a TaskContext that is needed for almost all tasks,
+//! such as the config and a way to interact with the database.
 //!
-//! The redb store is accessed in a single threaded way by an actor that runs
-//! on its own std thread. Communication with this actor is via a flume channel,
-//! with oneshot channels for the return values if needed.
+//! For tasks that are specific to a hash, a HashContext combines the task
+//! context with a slot from the table of the main actor that can be used
+//! to obtain an unique handle for the hash.
 //!
-//! Errors:
+//! # Runtime
 //!
-//! ActorError is an enum containing errors that can happen inside message
-//! handlers of the actor. This includes various redb related errors and io
-//! errors when reading or writing non-inlined data or outboard files.
+//! The fs store owns and manages its own tokio runtime. Dropping the store
+//! will clean up the database and shut down the runtime. However, some parts
+//! of the persistent state won't make it to disk, so operations involving large
+//! partial blobs will have a large initial delay on the next startup.
 //!
-//! OuterError is an enum containing all the actor errors and in addition
-//! errors when communicating with the actor.
+//! It is also not guaranteed that all write operations will make it to disk.
+//! The on-disk store will be in a consistent state, but might miss some writes
+//! in the last seconds before shutdown.
+//!
+//! To avoid this, you can use the [`crate::api::Store::shutdown`] method to
+//! cleanly shut down the store and save ephemeral state to disk.
+//!
+//! Note that if you use the store inside a [`iroh::protocol::Router`] and shut
+//! down the router using [`iroh::protocol::Router::shutdown`], the store will be
+//! safely shut down as well. Any store refs you are holding will be inoperable
+//! after this.
 use std::{
-    collections::{BTreeMap, BTreeSet},
+    collections::{HashMap, HashSet},
+    fmt, fs,
     future::Future,
-    io,
-    ops::Bound,
+    io::Write,
+    num::NonZeroU64,
+    ops::Deref,
     path::{Path, PathBuf},
-    sync::{Arc, RwLock},
-    time::{Duration, SystemTime},
+    sync::Arc,
 };
 
-use bao_tree::io::{
-    fsm::Outboard,
-    sync::{ReadAt, Size},
+use bao_tree::{
+    io::{
+        mixed::{traverse_ranges_validated, EncodedItem, ReadBytesAt},
+        sync::ReadAt,
+        BaoContentItem, Leaf,
+    },
+    ChunkNum, ChunkRanges,
 };
 use bytes::Bytes;
-use futures_lite::{Stream, StreamExt};
-use genawaiter::rc::{Co, Gen};
-use iroh_io::AsyncSliceReader;
-use redb::{AccessGuard, DatabaseError, ReadableTable, StorageError};
-use serde::{Deserialize, Serialize};
-use smallvec::SmallVec;
-use tokio::io::AsyncWriteExt;
-use tracing::trace_span;
-mod tables;
-#[doc(hidden)]
-pub mod test_support;
-#[cfg(test)]
-mod tests;
-mod util;
-mod validate;
+use delete_set::{BaoFilePart, ProtectHandle};
+use entry_state::{DataLocation, OutboardLocation};
+use gc::run_gc;
+use import::{ImportEntry, ImportSource};
+use irpc::channel::mpsc;
+use meta::{list_blobs, Snapshot};
+use n0_future::{future::yield_now, io};
+use nested_enum_utils::enum_conversions;
+use range_collections::range_set::RangeSetRange;
+use tokio::task::{Id, JoinError, JoinSet};
+use tracing::{error, instrument, trace};
 
-use tables::{ReadOnlyTables, ReadableTables, Tables};
-
-use self::{tables::DeleteSet, test_support::EntryData, util::PeekableFlumeReceiver};
-use super::{
-    bao_file::{BaoFileConfig, BaoFileHandle, BaoFileHandleWeak, CreateCb},
-    temp_name, BaoBatchWriter, BaoBlobSize, ConsistencyCheckProgress, EntryStatus, ExportMode,
-    ExportProgressCb, ImportMode, ImportProgress, Map, ReadableStore, TempCounterMap,
-};
 use crate::{
-    store::{
-        bao_file::{BaoFileStorage, CompleteStorage},
-        fs::{
-            tables::BaoFilePart,
-            util::{overwrite_and_sync, read_and_remove},
+    api::{
+        proto::{
+            self, bitfield::is_validated, BatchMsg, BatchResponse, Bitfield, Command,
+            CreateTempTagMsg, ExportBaoMsg, ExportBaoRequest, ExportPathMsg, ExportPathRequest,
+            ExportRangesItem, ExportRangesMsg, ExportRangesRequest, HashSpecific, ImportBaoMsg,
+            ImportBaoRequest, ObserveMsg, Scope,
         },
-        GcMarkEvent, GcSweepEvent,
+        ApiClient,
+    },
+    store::{
+        util::{BaoTreeSender, FixedSize, MemOrFile, ValueOrPoisioned},
+        Hash,
     },
     util::{
-        compute_outboard,
-        progress::{
-            BoxedProgressSender, IdGenerator, IgnoreProgressSender, ProgressSendError,
-            ProgressSender,
-        },
-        raw_outboard_size, MemOrFile, TagCounter, TagDrop,
+        channel::oneshot,
+        temp_tag::{TagDrop, TempTag, TempTagScope, TempTags},
+        ChunkRangesExt,
     },
-    BlobFormat, Hash, HashAndFormat, Tag, TempTag,
+};
+mod bao_file;
+use bao_file::{BaoFileHandle, BaoFileHandleWeak};
+mod delete_set;
+mod entry_state;
+mod import;
+mod meta;
+pub mod options;
+pub(crate) mod util;
+use entry_state::EntryState;
+use import::{import_byte_stream, import_bytes, import_path, ImportEntryMsg};
+use options::Options;
+use tracing::Instrument;
+mod gc;
+
+use super::HashAndFormat;
+use crate::api::{
+    self,
+    blobs::{AddProgressItem, ExportMode, ExportProgressItem},
+    Store,
 };
 
-/// Location of the data.
-///
-/// Data can be inlined in the database, a file conceptually owned by the store,
-/// or a number of external files conceptually owned by the user.
-///
-/// Only complete data can be inlined.
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
-pub(crate) enum DataLocation<I = (), E = ()> {
-    /// Data is in the inline_data table.
-    Inline(I),
-    /// Data is in the canonical location in the data directory.
-    Owned(E),
-    /// Data is in several external locations. This should be a non-empty list.
-    External(Vec<PathBuf>, E),
+/// Create a 16 byte unique ID.
+fn new_uuid() -> [u8; 16] {
+    use rand::RngCore;
+    let mut rng = rand::thread_rng();
+    let mut bytes = [0u8; 16];
+    rng.fill_bytes(&mut bytes);
+    bytes
 }
 
-impl<X> DataLocation<X, u64> {
-    fn union(self, that: DataLocation<X, u64>) -> ActorResult<Self> {
-        Ok(match (self, that) {
-            (
-                DataLocation::External(mut paths, a_size),
-                DataLocation::External(b_paths, b_size),
-            ) => {
-                if a_size != b_size {
-                    return Err(ActorError::Inconsistent(format!(
-                        "complete size mismatch {} {}",
-                        a_size, b_size
-                    )));
-                }
-                paths.extend(b_paths);
-                paths.sort();
-                paths.dedup();
-                DataLocation::External(paths, a_size)
-            }
-            (_, b @ DataLocation::Owned(_)) => {
-                // owned needs to win, since it has an associated file. Choosing
-                // external would orphan the file.
-                b
-            }
-            (a @ DataLocation::Owned(_), _) => {
-                // owned needs to win, since it has an associated file. Choosing
-                // external would orphan the file.
-                a
-            }
-            (_, b @ DataLocation::Inline(_)) => {
-                // inline needs to win, since it has associated data. Choosing
-                // external would orphan the file.
-                b
-            }
-            (a @ DataLocation::Inline(_), _) => {
-                // inline needs to win, since it has associated data. Choosing
-                // external would orphan the file.
-                a
-            }
-        })
-    }
+/// Create temp file name based on a 16 byte UUID.
+fn temp_name() -> String {
+    format!("{}.temp", hex::encode(new_uuid()))
 }
 
-impl<I, E> DataLocation<I, E> {
-    fn discard_inline_data(self) -> DataLocation<(), E> {
-        match self {
-            DataLocation::Inline(_) => DataLocation::Inline(()),
-            DataLocation::Owned(x) => DataLocation::Owned(x),
-            DataLocation::External(paths, x) => DataLocation::External(paths, x),
-        }
-    }
+#[derive(Debug)]
+#[enum_conversions()]
+pub(crate) enum InternalCommand {
+    Dump(meta::Dump),
+    FinishImport(ImportEntryMsg),
+    ClearScope(ClearScope),
 }
 
-/// Location of the outboard.
-///
-/// Outboard can be inlined in the database or a file conceptually owned by the store.
-/// Outboards are implementation specific to the store and as such are always owned.
-///
-/// Only complete outboards can be inlined.
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
-pub(crate) enum OutboardLocation<I = ()> {
-    /// Outboard is in the inline_outboard table.
-    Inline(I),
-    /// Outboard is in the canonical location in the data directory.
-    Owned,
-    /// Outboard is not needed
-    NotNeeded,
+#[derive(Debug)]
+pub(crate) struct ClearScope {
+    pub scope: Scope,
 }
 
-impl<I> OutboardLocation<I> {
-    fn discard_extra_data(self) -> OutboardLocation<()> {
+impl InternalCommand {
+    pub fn parent_span(&self) -> tracing::Span {
         match self {
-            Self::Inline(_) => OutboardLocation::Inline(()),
-            Self::Owned => OutboardLocation::Owned,
-            Self::NotNeeded => OutboardLocation::NotNeeded,
+            Self::Dump(_) => tracing::Span::current(),
+            Self::ClearScope(_) => tracing::Span::current(),
+            Self::FinishImport(cmd) => cmd
+                .parent_span_opt()
+                .cloned()
+                .unwrap_or_else(tracing::Span::current),
         }
     }
 }
 
-/// The information about an entry that we keep in the entry table for quick access.
-///
-/// The exact info to store here is TBD, so usually you should use the accessor methods.
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
-pub(crate) enum EntryState<I = ()> {
-    /// For a complete entry we always know the size. It does not make much sense
-    /// to write to a complete entry, so they are much easier to share.
-    Complete {
-        /// Location of the data.
-        data_location: DataLocation<I, u64>,
-        /// Location of the outboard.
-        outboard_location: OutboardLocation<I>,
-    },
-    /// Partial entries are entries for which we know the hash, but don't have
-    /// all the data. They are created when syncing from somewhere else by hash.
-    ///
-    /// As such they are always owned. There is also no inline storage for them.
-    /// Non short lived partial entries always live in the file system, and for
-    /// short lived ones we never create a database entry in the first place.
-    Partial {
-        /// Once we get the last chunk of a partial entry, we have validated
-        /// the size of the entry despite it still being incomplete.
-        ///
-        /// E.g. a giant file where we just requested the last chunk.
-        size: Option<u64>,
-    },
+/// Context needed by most tasks
+#[derive(Debug)]
+struct TaskContext {
+    // Store options such as paths and inline thresholds, in an Arc to cheaply share with tasks.
+    pub options: Arc<Options>,
+    // Metadata database, basically a mpsc sender with some extra functionality.
+    pub db: meta::Db,
+    // Handle to send internal commands
+    pub internal_cmd_tx: tokio::sync::mpsc::Sender<InternalCommand>,
+    /// The file handle for the empty hash.
+    pub empty: BaoFileHandle,
+    /// Handle to protect files from deletion.
+    pub protect: ProtectHandle,
 }
 
-impl Default for EntryState {
-    fn default() -> Self {
-        Self::Partial { size: None }
+impl TaskContext {
+    pub async fn clear_scope(&self, scope: Scope) {
+        self.internal_cmd_tx
+            .send(ClearScope { scope }.into())
+            .await
+            .ok();
     }
 }
 
-impl EntryState {
-    fn union(self, that: Self) -> ActorResult<Self> {
-        match (self, that) {
-            (
-                Self::Complete {
-                    data_location,
-                    outboard_location,
-                },
-                Self::Complete {
-                    data_location: b_data_location,
-                    ..
-                },
-            ) => Ok(Self::Complete {
-                // combine external paths if needed
-                data_location: data_location.union(b_data_location)?,
-                outboard_location,
-            }),
-            (a @ Self::Complete { .. }, Self::Partial { .. }) =>
-            // complete wins over partial
-            {
-                Ok(a)
-            }
-            (Self::Partial { .. }, b @ Self::Complete { .. }) =>
-            // complete wins over partial
-            {
-                Ok(b)
-            }
-            (Self::Partial { size: a_size }, Self::Partial { size: b_size }) =>
-            // keep known size from either entry
-            {
-                let size = match (a_size, b_size) {
-                    (Some(a_size), Some(b_size)) => {
-                        // validated sizes are different. this means that at
-                        // least one validation was wrong, which would be a bug
-                        // in bao-tree.
-                        if a_size != b_size {
-                            return Err(ActorError::Inconsistent(format!(
-                                "validated size mismatch {} {}",
-                                a_size, b_size
-                            )));
-                        }
-                        Some(a_size)
-                    }
-                    (Some(a_size), None) => Some(a_size),
-                    (None, Some(b_size)) => Some(b_size),
-                    (None, None) => None,
-                };
-                Ok(Self::Partial { size })
-            }
-        }
-    }
+#[derive(Debug)]
+struct Actor {
+    // Context that can be cheaply shared with tasks.
+    context: Arc<TaskContext>,
+    // Receiver for incoming user commands.
+    cmd_rx: tokio::sync::mpsc::Receiver<Command>,
+    // Receiver for incoming file store specific commands.
+    fs_cmd_rx: tokio::sync::mpsc::Receiver<InternalCommand>,
+    // Tasks for import and export operations.
+    tasks: JoinSet<()>,
+    // Running tasks
+    running: HashSet<Id>,
+    // handles
+    handles: HashMap<Hash, Slot>,
+    // temp tags
+    temp_tags: TempTags,
+    // our private tokio runtime. It has to live somewhere.
+    _rt: RtWrapper,
 }
 
-impl redb::Value for EntryState {
-    type SelfType<'a> = EntryState;
+/// Wraps a slot and the task context.
+///
+/// This contains everything a hash-specific task should need.
+struct HashContext {
+    slot: Slot,
+    ctx: Arc<TaskContext>,
+}
 
-    type AsBytes<'a> = SmallVec<[u8; 128]>;
+impl HashContext {
+    pub fn db(&self) -> &meta::Db {
+        &self.ctx.db
+    }
 
-    fn fixed_width() -> Option<usize> {
-        None
+    pub fn options(&self) -> &Arc<Options> {
+        &self.ctx.options
     }
 
-    fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a>
-    where
-        Self: 'a,
-    {
-        postcard::from_bytes(data).unwrap()
+    pub async fn lock(&self) -> tokio::sync::MutexGuard<'_, Option<BaoFileHandleWeak>> {
+        self.slot.0.lock().await
     }
 
-    fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a>
-    where
-        Self: 'a,
-        Self: 'b,
-    {
-        postcard::to_extend(value, SmallVec::new()).unwrap()
+    pub fn protect(&self, hash: Hash, parts: impl IntoIterator<Item = BaoFilePart>) {
+        self.ctx.protect.protect(hash, parts);
     }
 
-    fn type_name() -> redb::TypeName {
-        redb::TypeName::new("EntryState")
+    /// Update the entry state in the database, and wait for completion.
+    pub async fn update(&self, hash: Hash, state: EntryState<Bytes>) -> io::Result<()> {
+        let (tx, rx) = oneshot::channel();
+        self.db()
+            .send(
+                meta::Update {
+                    hash,
+                    state,
+                    tx: Some(tx),
+                    span: tracing::Span::current(),
+                }
+                .into(),
+            )
+            .await?;
+        rx.await.map_err(|_e| io::Error::other(""))??;
+        Ok(())
     }
-}
 
-/// Options for inlining small complete data or outboards.
-#[derive(Debug, Clone)]
-pub struct InlineOptions {
-    /// Maximum data size to inline.
-    pub max_data_inlined: u64,
-    /// Maximum outboard size to inline.
-    pub max_outboard_inlined: u64,
-}
+    pub async fn get_entry_state(&self, hash: Hash) -> io::Result<Option<EntryState<Bytes>>> {
+        if hash == Hash::EMPTY {
+            return Ok(Some(EntryState::Complete {
+                data_location: DataLocation::Inline(Bytes::new()),
+                outboard_location: OutboardLocation::NotNeeded,
+            }));
+        }
+        let (tx, rx) = oneshot::channel();
+        self.db()
+            .send(
+                meta::Get {
+                    hash,
+                    tx,
+                    span: tracing::Span::current(),
+                }
+                .into(),
+            )
+            .await
+            .ok();
+        let res = rx.await.map_err(io::Error::other)?;
+        Ok(res.state?)
+    }
 
-impl InlineOptions {
-    /// Do not inline anything, ever.
-    pub const NO_INLINE: Self = Self {
-        max_data_inlined: 0,
-        max_outboard_inlined: 0,
-    };
-    /// Always inline everything
-    pub const ALWAYS_INLINE: Self = Self {
-        max_data_inlined: u64::MAX,
-        max_outboard_inlined: u64::MAX,
-    };
-}
+    /// Update the entry state in the database, and wait for completion.
+    pub async fn set(&self, hash: Hash, state: EntryState<Bytes>) -> io::Result<()> {
+        let (tx, rx) = oneshot::channel();
+        self.db()
+            .send(
+                meta::Set {
+                    hash,
+                    state,
+                    tx,
+                    span: tracing::Span::current(),
+                }
+                .into(),
+            )
+            .await
+            .map_err(io::Error::other)?;
+        rx.await.map_err(|_e| io::Error::other(""))??;
+        Ok(())
+    }
 
-impl Default for InlineOptions {
-    fn default() -> Self {
-        Self {
-            max_data_inlined: 1024 * 16,
-            max_outboard_inlined: 1024 * 16,
+    pub async fn get_or_create(&self, hash: Hash) -> api::Result<BaoFileHandle> {
+        if hash == Hash::EMPTY {
+            return Ok(self.ctx.empty.clone());
         }
+        let res = self
+            .slot
+            .get_or_create(|| async {
+                let res = self.db().get(hash).await.map_err(io::Error::other)?;
+                let res = match res {
+                    Some(state) => open_bao_file(&hash, state, &self.ctx).await,
+                    None => Ok(BaoFileHandle::new_partial_mem(
+                        hash,
+                        self.ctx.options.clone(),
+                    )),
+                };
+                Ok((res?, ()))
+            })
+            .await
+            .map_err(api::Error::from);
+        trace!("{res:?}");
+        let (res, _) = res?;
+        Ok(res)
     }
 }
 
-/// Options for directories used by the file store.
-#[derive(Debug, Clone)]
-pub struct PathOptions {
-    /// Path to the directory where data and outboard files are stored.
-    pub data_path: PathBuf,
-    /// Path to the directory where temp files are stored.
-    /// This *must* be on the same device as `data_path`, since we need to
-    /// atomically move temp files into place.
-    pub temp_path: PathBuf,
+async fn open_bao_file(
+    hash: &Hash,
+    state: EntryState<Bytes>,
+    ctx: &TaskContext,
+) -> io::Result<BaoFileHandle> {
+    let options = &ctx.options;
+    Ok(match state {
+        EntryState::Complete {
+            data_location,
+            outboard_location,
+        } => {
+            let data = match data_location {
+                DataLocation::Inline(data) => MemOrFile::Mem(data),
+                DataLocation::Owned(size) => {
+                    let path = options.path.data_path(hash);
+                    let file = fs::File::open(&path)?;
+                    MemOrFile::File(FixedSize::new(file, size))
+                }
+                DataLocation::External(paths, size) => {
+                    let Some(path) = paths.into_iter().next() else {
+                        return Err(io::Error::other("no external data path"));
+                    };
+                    let file = fs::File::open(&path)?;
+                    MemOrFile::File(FixedSize::new(file, size))
+                }
+            };
+            let outboard = match outboard_location {
+                OutboardLocation::NotNeeded => MemOrFile::empty(),
+                OutboardLocation::Inline(data) => MemOrFile::Mem(data),
+                OutboardLocation::Owned => {
+                    let path = options.path.outboard_path(hash);
+                    let file = fs::File::open(&path)?;
+                    MemOrFile::File(file)
+                }
+            };
+            BaoFileHandle::new_complete(*hash, data, outboard, options.clone())
+        }
+        EntryState::Partial { .. } => BaoFileHandle::new_partial_file(*hash, ctx).await?,
+    })
 }
 
-impl PathOptions {
-    fn new(root: &Path) -> Self {
-        Self {
-            data_path: root.join("data"),
-            temp_path: root.join("temp"),
-        }
-    }
+/// An entry for each hash, containing a weak reference to a BaoFileHandle
+/// wrapped in a tokio mutex so handle creation is sequential.
+#[derive(Debug, Clone, Default)]
+pub(crate) struct Slot(Arc<tokio::sync::Mutex<Option<BaoFileHandleWeak>>>);
 
-    fn owned_data_path(&self, hash: &Hash) -> PathBuf {
-        self.data_path.join(format!("{}.data", hash.to_hex()))
+impl Slot {
+    pub async fn is_live(&self) -> bool {
+        let slot = self.0.lock().await;
+        slot.as_ref().map(|weak| !weak.is_dead()).unwrap_or(false)
     }
 
-    fn owned_outboard_path(&self, hash: &Hash) -> PathBuf {
-        self.data_path.join(format!("{}.obao4", hash.to_hex()))
+    /// Get the handle if it exists and is still alive, otherwise load it from the database.
+    /// If there is nothing in the database, create a new in-memory handle.
+    ///
+    /// `make` will be called if the a live handle does not exist.
+    pub async fn get_or_create<F, Fut, T>(&self, make: F) -> io::Result<(BaoFileHandle, T)>
+    where
+        F: FnOnce() -> Fut,
+        Fut: std::future::Future<Output = io::Result<(BaoFileHandle, T)>>,
+        T: Default,
+    {
+        let mut slot = self.0.lock().await;
+        if let Some(weak) = &*slot {
+            if let Some(handle) = weak.upgrade() {
+                return Ok((handle, Default::default()));
+            }
+        }
+        let handle = make().await;
+        if let Ok((handle, _)) = &handle {
+            *slot = Some(handle.downgrade());
+        }
+        handle
     }
+}
 
-    fn owned_sizes_path(&self, hash: &Hash) -> PathBuf {
-        self.data_path.join(format!("{}.sizes4", hash.to_hex()))
+impl Actor {
+    fn db(&self) -> &meta::Db {
+        &self.context.db
     }
 
-    fn temp_file_name(&self) -> PathBuf {
-        self.temp_path.join(temp_name())
+    fn context(&self) -> Arc<TaskContext> {
+        self.context.clone()
     }
-}
 
-/// Options for transaction batching.
-#[derive(Debug, Clone)]
-pub struct BatchOptions {
-    /// Maximum number of actor messages to batch before creating a new read transaction.
-    pub max_read_batch: usize,
-    /// Maximum duration to wait before committing a read transaction.
-    pub max_read_duration: Duration,
-    /// Maximum number of actor messages to batch before committing write transaction.
-    pub max_write_batch: usize,
-    /// Maximum duration to wait before committing a write transaction.
-    pub max_write_duration: Duration,
-}
+    fn spawn(&mut self, fut: impl Future<Output = ()> + Send + 'static) {
+        let span = tracing::Span::current();
+        let id = self.tasks.spawn(fut.instrument(span)).id();
+        self.running.insert(id);
+    }
 
-impl Default for BatchOptions {
-    fn default() -> Self {
-        Self {
-            max_read_batch: 10000,
-            max_read_duration: Duration::from_secs(1),
-            max_write_batch: 1000,
-            max_write_duration: Duration::from_millis(500),
+    fn log_task_result(&mut self, res: Result<(Id, ()), JoinError>) {
+        match res {
+            Ok((id, _)) => {
+                // println!("task {id} finished");
+                self.running.remove(&id);
+                // println!("{:?}", self.running);
+            }
+            Err(e) => {
+                error!("task failed: {e}");
+            }
         }
     }
-}
 
-/// Options for the file store.
-#[derive(Debug, Clone)]
-pub struct Options {
-    /// Path options.
-    pub path: PathOptions,
-    /// Inline storage options.
-    pub inline: InlineOptions,
-    /// Transaction batching options.
-    pub batch: BatchOptions,
-}
+    async fn create_temp_tag(&mut self, cmd: CreateTempTagMsg) {
+        let CreateTempTagMsg { tx, inner, .. } = cmd;
+        let mut tt = self.temp_tags.create(inner.scope, inner.value);
+        if tx.is_rpc() {
+            tt.leak();
+        }
+        tx.send(tt).await.ok();
+    }
 
-#[derive(derive_more::Debug)]
-pub(crate) enum ImportSource {
-    TempFile(PathBuf),
-    External(PathBuf),
-    Memory(#[debug(skip)] Bytes),
-}
+    async fn clear_dead_handles(&mut self) {
+        let mut to_remove = Vec::new();
+        for (hash, slot) in &self.handles {
+            if !slot.is_live().await {
+                to_remove.push(*hash);
+            }
+        }
+        for hash in to_remove {
+            if let Some(slot) = self.handles.remove(&hash) {
+                // do a quick check if the handle has become alive in the meantime, and reinsert it
+                let guard = slot.0.lock().await;
+                let is_live = guard.as_ref().map(|x| !x.is_dead()).unwrap_or_default();
+                if is_live {
+                    drop(guard);
+                    self.handles.insert(hash, slot);
+                }
+            }
+        }
+    }
 
-impl ImportSource {
-    fn content(&self) -> MemOrFile<&[u8], &Path> {
-        match self {
-            Self::TempFile(path) => MemOrFile::File(path.as_path()),
-            Self::External(path) => MemOrFile::File(path.as_path()),
-            Self::Memory(data) => MemOrFile::Mem(data.as_ref()),
+    async fn handle_command(&mut self, cmd: Command) {
+        let span = cmd.parent_span();
+        let _entered = span.enter();
+        match cmd {
+            Command::SyncDb(cmd) => {
+                trace!("{cmd:?}");
+                self.db().send(cmd.into()).await.ok();
+            }
+            Command::Shutdown(cmd) => {
+                trace!("{cmd:?}");
+                self.db().send(cmd.into()).await.ok();
+            }
+            Command::CreateTag(cmd) => {
+                trace!("{cmd:?}");
+                self.db().send(cmd.into()).await.ok();
+            }
+            Command::SetTag(cmd) => {
+                trace!("{cmd:?}");
+                self.db().send(cmd.into()).await.ok();
+            }
+            Command::ListTags(cmd) => {
+                trace!("{cmd:?}");
+                self.db().send(cmd.into()).await.ok();
+            }
+            Command::DeleteTags(cmd) => {
+                trace!("{cmd:?}");
+                self.db().send(cmd.into()).await.ok();
+            }
+            Command::RenameTag(cmd) => {
+                trace!("{cmd:?}");
+                self.db().send(cmd.into()).await.ok();
+            }
+            Command::ClearProtected(cmd) => {
+                trace!("{cmd:?}");
+                self.clear_dead_handles().await;
+                self.db().send(cmd.into()).await.ok();
+            }
+            Command::BlobStatus(cmd) => {
+                trace!("{cmd:?}");
+                self.db().send(cmd.into()).await.ok();
+            }
+            Command::ListBlobs(cmd) => {
+                trace!("{cmd:?}");
+                let (tx, rx) = tokio::sync::oneshot::channel();
+                self.db()
+                    .send(
+                        Snapshot {
+                            tx,
+                            span: cmd.span.clone(),
+                        }
+                        .into(),
+                    )
+                    .await
+                    .ok();
+                if let Ok(snapshot) = rx.await {
+                    self.spawn(list_blobs(snapshot, cmd));
+                }
+            }
+            Command::DeleteBlobs(cmd) => {
+                trace!("{cmd:?}");
+                self.db().send(cmd.into()).await.ok();
+            }
+            Command::Batch(cmd) => {
+                trace!("{cmd:?}");
+                let (id, scope) = self.temp_tags.create_scope();
+                self.spawn(handle_batch(cmd, id, scope, self.context()));
+            }
+            Command::CreateTempTag(cmd) => {
+                trace!("{cmd:?}");
+                self.create_temp_tag(cmd).await;
+            }
+            Command::ListTempTags(cmd) => {
+                trace!("{cmd:?}");
+                let tts = self.temp_tags.list();
+                cmd.tx.send(tts).await.ok();
+            }
+            Command::ImportBytes(cmd) => {
+                trace!("{cmd:?}");
+                self.spawn(import_bytes(cmd, self.context()));
+            }
+            Command::ImportByteStream(cmd) => {
+                trace!("{cmd:?}");
+                self.spawn(import_byte_stream(cmd, self.context()));
+            }
+            Command::ImportPath(cmd) => {
+                trace!("{cmd:?}");
+                self.spawn(import_path(cmd, self.context()));
+            }
+            Command::ExportPath(cmd) => {
+                trace!("{cmd:?}");
+                let ctx = self.hash_context(cmd.hash);
+                self.spawn(export_path(cmd, ctx));
+            }
+            Command::ExportBao(cmd) => {
+                trace!("{cmd:?}");
+                let ctx = self.hash_context(cmd.hash);
+                self.spawn(export_bao(cmd, ctx));
+            }
+            Command::ExportRanges(cmd) => {
+                trace!("{cmd:?}");
+                let ctx = self.hash_context(cmd.hash);
+                self.spawn(export_ranges(cmd, ctx));
+            }
+            Command::ImportBao(cmd) => {
+                trace!("{cmd:?}");
+                let ctx = self.hash_context(cmd.hash);
+                self.spawn(import_bao(cmd, ctx));
+            }
+            Command::Observe(cmd) => {
+                trace!("{cmd:?}");
+                let ctx = self.hash_context(cmd.hash);
+                self.spawn(observe(cmd, ctx));
+            }
         }
     }
 
-    fn len(&self) -> io::Result<u64> {
-        match self {
-            Self::TempFile(path) => std::fs::metadata(path).map(|m| m.len()),
-            Self::External(path) => std::fs::metadata(path).map(|m| m.len()),
-            Self::Memory(data) => Ok(data.len() as u64),
+    /// Create a hash context for a given hash.
+    fn hash_context(&mut self, hash: Hash) -> HashContext {
+        HashContext {
+            slot: self.handles.entry(hash).or_default().clone(),
+            ctx: self.context.clone(),
         }
     }
-}
 
-/// Use BaoFileHandle as the entry type for the map.
-pub type Entry = BaoFileHandle;
+    async fn handle_fs_command(&mut self, cmd: InternalCommand) {
+        let span = cmd.parent_span();
+        let _entered = span.enter();
+        match cmd {
+            InternalCommand::Dump(cmd) => {
+                trace!("{cmd:?}");
+                self.db().send(cmd.into()).await.ok();
+            }
+            InternalCommand::ClearScope(cmd) => {
+                trace!("{cmd:?}");
+                self.temp_tags.end_scope(cmd.scope);
+            }
+            InternalCommand::FinishImport(cmd) => {
+                trace!("{cmd:?}");
+                if cmd.hash == Hash::EMPTY {
+                    cmd.tx
+                        .send(AddProgressItem::Done(TempTag::leaking_empty(cmd.format)))
+                        .await
+                        .ok();
+                } else {
+                    let tt = self.temp_tags.create(
+                        cmd.scope,
+                        HashAndFormat {
+                            hash: cmd.hash,
+                            format: cmd.format,
+                        },
+                    );
+                    let ctx = self.hash_context(cmd.hash);
+                    self.spawn(finish_import(cmd, tt, ctx));
+                }
+            }
+        }
+    }
 
-impl super::MapEntry for Entry {
-    fn hash(&self) -> Hash {
-        self.hash()
+    async fn run(mut self) {
+        loop {
+            tokio::select! {
+                cmd = self.cmd_rx.recv() => {
+                    let Some(cmd) = cmd else {
+                        break;
+                    };
+                    self.handle_command(cmd).await;
+                }
+                Some(cmd) = self.fs_cmd_rx.recv() => {
+                    self.handle_fs_command(cmd).await;
+                }
+                Some(res) = self.tasks.join_next_with_id(), if !self.tasks.is_empty() => {
+                    self.log_task_result(res);
+                }
+            }
+        }
     }
 
-    fn size(&self) -> BaoBlobSize {
-        let size = self.current_size().unwrap();
-        tracing::trace!("redb::Entry::size() = {}", size);
-        BaoBlobSize::new(size, self.is_complete())
+    async fn new(
+        db_path: PathBuf,
+        rt: RtWrapper,
+        cmd_rx: tokio::sync::mpsc::Receiver<Command>,
+        fs_commands_rx: tokio::sync::mpsc::Receiver<InternalCommand>,
+        fs_commands_tx: tokio::sync::mpsc::Sender<InternalCommand>,
+        options: Arc<Options>,
+    ) -> anyhow::Result<Self> {
+        trace!(
+            "creating data directory: {}",
+            options.path.data_path.display()
+        );
+        fs::create_dir_all(&options.path.data_path)?;
+        trace!(
+            "creating temp directory: {}",
+            options.path.temp_path.display()
+        );
+        fs::create_dir_all(&options.path.temp_path)?;
+        trace!(
+            "creating parent directory for db file{}",
+            db_path.parent().unwrap().display()
+        );
+        fs::create_dir_all(db_path.parent().unwrap())?;
+        let (db_send, db_recv) = tokio::sync::mpsc::channel(100);
+        let (protect, ds) = delete_set::pair(Arc::new(options.path.clone()));
+        let db_actor = meta::Actor::new(db_path, db_recv, ds, options.batch.clone())?;
+        let slot_context = Arc::new(TaskContext {
+            options: options.clone(),
+            db: meta::Db::new(db_send),
+            internal_cmd_tx: fs_commands_tx,
+            empty: BaoFileHandle::new_complete(
+                Hash::EMPTY,
+                MemOrFile::empty(),
+                MemOrFile::empty(),
+                options,
+            ),
+            protect,
+        });
+        rt.spawn(db_actor.run());
+        Ok(Self {
+            context: slot_context,
+            cmd_rx,
+            fs_cmd_rx: fs_commands_rx,
+            tasks: JoinSet::new(),
+            running: HashSet::new(),
+            handles: Default::default(),
+            temp_tags: Default::default(),
+            _rt: rt,
+        })
     }
+}
+
+struct RtWrapper(Option<tokio::runtime::Runtime>);
 
-    fn is_complete(&self) -> bool {
-        self.is_complete()
+impl From<tokio::runtime::Runtime> for RtWrapper {
+    fn from(rt: tokio::runtime::Runtime) -> Self {
+        Self(Some(rt))
     }
+}
 
-    async fn outboard(&self) -> io::Result<impl Outboard> {
-        self.outboard()
+impl fmt::Debug for RtWrapper {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        ValueOrPoisioned(self.0.as_ref()).fmt(f)
     }
+}
 
-    async fn data_reader(&self) -> io::Result<impl AsyncSliceReader> {
-        Ok(self.data_reader())
+impl Deref for RtWrapper {
+    type Target = tokio::runtime::Runtime;
+
+    fn deref(&self) -> &Self::Target {
+        self.0.as_ref().unwrap()
     }
 }
 
-impl super::MapEntryMut for Entry {
-    async fn batch_writer(&self) -> io::Result<impl BaoBatchWriter> {
-        Ok(self.writer())
+impl Drop for RtWrapper {
+    fn drop(&mut self) {
+        if let Some(rt) = self.0.take() {
+            trace!("dropping tokio runtime");
+            tokio::task::block_in_place(|| {
+                drop(rt);
+            });
+            trace!("dropped tokio runtime");
+        }
     }
 }
 
-#[derive(derive_more::Debug)]
-pub(crate) struct Import {
-    /// The hash and format of the data to import
-    content_id: HashAndFormat,
-    /// The source of the data to import, can be a temp file, external file, or memory
-    source: ImportSource,
-    /// Data size
-    data_size: u64,
-    /// Outboard without length prefix
-    #[debug("{:?}", outboard.as_ref().map(|x| x.len()))]
-    outboard: Option<Vec<u8>>,
+async fn handle_batch(cmd: BatchMsg, id: Scope, scope: Arc<TempTagScope>, ctx: Arc<TaskContext>) {
+    if let Err(cause) = handle_batch_impl(cmd, id, &scope).await {
+        error!("batch failed: {cause}");
+    }
+    ctx.clear_scope(id).await;
 }
 
-#[derive(derive_more::Debug)]
-pub(crate) struct Export {
-    /// A temp tag to keep the entry alive while exporting. This also
-    /// contains the hash to be exported.
-    temp_tag: TempTag,
-    /// The target path for the export.
-    target: PathBuf,
-    /// The export mode to use.
-    mode: ExportMode,
-    /// The progress callback to use.
-    #[debug(skip)]
-    progress: ExportProgressCb,
+async fn handle_batch_impl(cmd: BatchMsg, id: Scope, scope: &Arc<TempTagScope>) -> api::Result<()> {
+    let BatchMsg { tx, mut rx, .. } = cmd;
+    trace!("created scope {}", id);
+    tx.send(id).await.map_err(api::Error::other)?;
+    while let Some(msg) = rx.recv().await? {
+        match msg {
+            BatchResponse::Drop(msg) => scope.on_drop(&msg),
+            BatchResponse::Ping => {}
+        }
+    }
+    Ok(())
 }
 
-#[derive(derive_more::Debug)]
-pub(crate) enum ActorMessage {
-    // Query method: get a file handle for a hash, if it exists.
-    // This will produce a file handle even for entries that are not yet in redb at all.
-    Get {
-        hash: Hash,
-        tx: oneshot::Sender<ActorResult<Option<BaoFileHandle>>>,
-    },
-    /// Query method: get the rough entry status for a hash. Just complete, partial or not found.
-    EntryStatus {
-        hash: Hash,
-        tx: oneshot::Sender<ActorResult<EntryStatus>>,
-    },
-    #[cfg(test)]
-    /// Query method: get the full entry state for a hash, both in memory and in redb.
-    /// This is everything we got about the entry, including the actual inline outboard and data.
-    EntryState {
-        hash: Hash,
-        tx: oneshot::Sender<ActorResult<test_support::EntryStateResponse>>,
-    },
-    /// Query method: get the full entry state for a hash.
-    GetFullEntryState {
-        hash: Hash,
-        tx: oneshot::Sender<ActorResult<Option<EntryData>>>,
-    },
-    /// Modification method: set the full entry state for a hash.
-    SetFullEntryState {
-        hash: Hash,
-        entry: Option<EntryData>,
-        tx: oneshot::Sender<ActorResult<()>>,
-    },
-    /// Modification method: get or create a file handle for a hash.
-    ///
-    /// If the entry exists in redb, either partial or complete, the corresponding
-    /// data will be returned. If it does not yet exist, a new partial file handle
-    /// will be created, but not yet written to redb.
-    GetOrCreate {
-        hash: Hash,
-        tx: oneshot::Sender<ActorResult<BaoFileHandle>>,
-    },
-    /// Modification method: inline size was exceeded for a partial entry.
-    /// If the entry is complete, this is a no-op. If the entry is partial and in
-    /// memory, it will be written to a file and created in redb.
-    OnMemSizeExceeded { hash: Hash },
-    /// Modification method: marks a partial entry as complete.
-    /// Calling this on a complete entry is a no-op.
-    OnComplete { handle: BaoFileHandle },
-    /// Modification method: import data into a redb store
-    ///
-    /// At this point the size, hash and outboard must already be known.
-    Import {
-        cmd: Import,
-        tx: oneshot::Sender<ActorResult<(TempTag, u64)>>,
-    },
-    /// Modification method: export data from a redb store
-    ///
-    /// In most cases this will not modify the store. Only when using
-    /// [`ExportMode::TryReference`] and the entry is large enough to not be
-    /// inlined.
-    Export {
-        cmd: Export,
-        tx: oneshot::Sender<ActorResult<()>>,
-    },
-    /// Update inline options
-    UpdateInlineOptions {
-        /// The new inline options
-        inline_options: InlineOptions,
-        /// Whether to reapply the new options to existing entries
-        reapply: bool,
-        tx: oneshot::Sender<()>,
-    },
-    /// Bulk query method: get entries from the blobs table
-    Blobs {
-        #[debug(skip)]
-        filter: FilterPredicate<Hash, EntryState>,
-        #[allow(clippy::type_complexity)]
-        tx: oneshot::Sender<
-            ActorResult<Vec<std::result::Result<(Hash, EntryState), StorageError>>>,
-        >,
-    },
-    /// Bulk query method: get the entire tags table
-    Tags {
-        from: Option<Tag>,
-        to: Option<Tag>,
-        #[allow(clippy::type_complexity)]
-        tx: oneshot::Sender<
-            ActorResult<Vec<std::result::Result<(Tag, HashAndFormat), StorageError>>>,
-        >,
-    },
-    /// Modification method: set a tag to a value.
-    SetTag {
-        tag: Tag,
-        value: HashAndFormat,
-        tx: oneshot::Sender<ActorResult<()>>,
-    },
-    /// Modification method: set a tag to a value.
-    DeleteTags {
-        from: Option<Tag>,
-        to: Option<Tag>,
-        tx: oneshot::Sender<ActorResult<()>>,
-    },
-    /// Modification method: create a new unique tag and set it to a value.
-    CreateTag {
-        hash: HashAndFormat,
-        tx: oneshot::Sender<ActorResult<Tag>>,
-    },
-    /// Modification method: rename a tag atomically.
-    RenameTag {
-        from: Tag,
-        to: Tag,
-        tx: oneshot::Sender<ActorResult<()>>,
-    },
-    /// Modification method: unconditional delete the data for a number of hashes
-    Delete {
-        hashes: Vec<Hash>,
-        tx: oneshot::Sender<ActorResult<()>>,
-    },
-    /// Modification method: delete the data for a number of hashes, only if not protected
-    GcDelete {
-        hashes: Vec<Hash>,
-        tx: oneshot::Sender<ActorResult<()>>,
-    },
-    /// Sync the entire database to disk.
-    ///
-    /// This just makes sure that there is no write transaction open.
-    Sync { tx: oneshot::Sender<()> },
-    /// Internal method: dump the entire database to stdout.
-    Dump,
-    /// Internal method: validate the entire database.
-    ///
-    /// Note that this will block the actor until it is done, so don't use it
-    /// on a node under load.
-    Fsck {
-        repair: bool,
-        progress: BoxedProgressSender<ConsistencyCheckProgress>,
-        tx: oneshot::Sender<ActorResult<()>>,
-    },
-    /// Internal method: notify the actor that a new gc epoch has started.
-    ///
-    /// This will be called periodically and can be used to do misc cleanups.
-    GcStart { tx: oneshot::Sender<()> },
-    /// Internal method: shutdown the actor.
-    ///
-    /// Can have an optional oneshot sender to signal when the actor has shut down.
-    Shutdown { tx: Option<oneshot::Sender<()>> },
+#[instrument(skip_all, fields(hash = %cmd.hash_short()))]
+async fn finish_import(cmd: ImportEntryMsg, mut tt: TempTag, ctx: HashContext) {
+    let res = match finish_import_impl(cmd.inner, ctx).await {
+        Ok(()) => {
+            // for a remote call, we can't have the on_drop callback, so we have to leak the temp tag
+            // it will be cleaned up when either the process exits or scope ends
+            if cmd.tx.is_rpc() {
+                trace!("leaking temp tag {}", tt.hash_and_format());
+                tt.leak();
+            }
+            AddProgressItem::Done(tt)
+        }
+        Err(cause) => AddProgressItem::Error(cause),
+    };
+    cmd.tx.send(res).await.ok();
 }
 
-impl ActorMessage {
-    fn category(&self) -> MessageCategory {
-        match self {
-            Self::Get { .. }
-            | Self::GetOrCreate { .. }
-            | Self::EntryStatus { .. }
-            | Self::Blobs { .. }
-            | Self::Tags { .. }
-            | Self::GcStart { .. }
-            | Self::GetFullEntryState { .. }
-            | Self::Dump => MessageCategory::ReadOnly,
-            Self::Import { .. }
-            | Self::Export { .. }
-            | Self::OnMemSizeExceeded { .. }
-            | Self::OnComplete { .. }
-            | Self::SetTag { .. }
-            | Self::CreateTag { .. }
-            | Self::SetFullEntryState { .. }
-            | Self::Delete { .. }
-            | Self::DeleteTags { .. }
-            | Self::RenameTag { .. }
-            | Self::GcDelete { .. } => MessageCategory::ReadWrite,
-            Self::UpdateInlineOptions { .. }
-            | Self::Sync { .. }
-            | Self::Shutdown { .. }
-            | Self::Fsck { .. } => MessageCategory::TopLevel,
-            #[cfg(test)]
-            Self::EntryState { .. } => MessageCategory::ReadOnly,
+async fn finish_import_impl(import_data: ImportEntry, ctx: HashContext) -> io::Result<()> {
+    let ImportEntry {
+        source,
+        hash,
+        outboard,
+        ..
+    } = import_data;
+    let options = ctx.options();
+    match &source {
+        ImportSource::Memory(data) => {
+            debug_assert!(options.is_inlined_data(data.len() as u64));
+        }
+        ImportSource::External(_, _, size) => {
+            debug_assert!(!options.is_inlined_data(*size));
+        }
+        ImportSource::TempFile(_, _, size) => {
+            debug_assert!(!options.is_inlined_data(*size));
+        }
+    }
+    let guard = ctx.lock().await;
+    let handle = guard.as_ref().and_then(|x| x.upgrade());
+    // if I do have an existing handle, I have to possibly deal with observers.
+    // if I don't have an existing handle, there are 2 cases:
+    //   the entry exists in the db, but we don't have a handle
+    //   the entry does not exist at all.
+    // convert the import source to a data location and drop the open files
+    ctx.protect(hash, [BaoFilePart::Data, BaoFilePart::Outboard]);
+    let data_location = match source {
+        ImportSource::Memory(data) => DataLocation::Inline(data),
+        ImportSource::External(path, _file, size) => DataLocation::External(vec![path], size),
+        ImportSource::TempFile(path, _file, size) => {
+            // this will always work on any unix, but on windows there might be an issue if the target file is open!
+            // possibly open with FILE_SHARE_DELETE on windows?
+            let target = ctx.options().path.data_path(&hash);
+            trace!(
+                "moving temp file to owned data location: {} -> {}",
+                path.display(),
+                target.display()
+            );
+            if let Err(cause) = fs::rename(&path, &target) {
+                error!(
+                    "failed to move temp file {} to owned data location {}: {cause}",
+                    path.display(),
+                    target.display()
+                );
+            }
+            DataLocation::Owned(size)
+        }
+    };
+    let outboard_location = match outboard {
+        MemOrFile::Mem(bytes) if bytes.is_empty() => OutboardLocation::NotNeeded,
+        MemOrFile::Mem(bytes) => OutboardLocation::Inline(bytes),
+        MemOrFile::File(path) => {
+            // the same caveat as above applies here
+            let target = ctx.options().path.outboard_path(&hash);
+            trace!(
+                "moving temp file to owned outboard location: {} -> {}",
+                path.display(),
+                target.display()
+            );
+            if let Err(cause) = fs::rename(&path, &target) {
+                error!(
+                    "failed to move temp file {} to owned outboard location {}: {cause}",
+                    path.display(),
+                    target.display()
+                );
+            }
+            OutboardLocation::Owned
         }
+    };
+    if let Some(handle) = handle {
+        let data = match &data_location {
+            DataLocation::Inline(data) => MemOrFile::Mem(data.clone()),
+            DataLocation::Owned(size) => {
+                let path = ctx.options().path.data_path(&hash);
+                let file = fs::File::open(&path)?;
+                MemOrFile::File(FixedSize::new(file, *size))
+            }
+            DataLocation::External(paths, size) => {
+                let Some(path) = paths.iter().next() else {
+                    return Err(io::Error::other("no external data path"));
+                };
+                let file = fs::File::open(path)?;
+                MemOrFile::File(FixedSize::new(file, *size))
+            }
+        };
+        let outboard = match &outboard_location {
+            OutboardLocation::NotNeeded => MemOrFile::empty(),
+            OutboardLocation::Inline(data) => MemOrFile::Mem(data.clone()),
+            OutboardLocation::Owned => {
+                let path = ctx.options().path.outboard_path(&hash);
+                let file = fs::File::open(&path)?;
+                MemOrFile::File(file)
+            }
+        };
+        handle.complete(data, outboard);
     }
+    let state = EntryState::Complete {
+        data_location,
+        outboard_location,
+    };
+    ctx.update(hash, state).await?;
+    Ok(())
 }
 
-enum MessageCategory {
-    ReadOnly,
-    ReadWrite,
-    TopLevel,
+#[instrument(skip_all, fields(hash = %cmd.hash_short()))]
+async fn import_bao(cmd: ImportBaoMsg, ctx: HashContext) {
+    trace!("{cmd:?}");
+    let ImportBaoMsg {
+        inner: ImportBaoRequest { size, hash },
+        rx,
+        tx,
+        ..
+    } = cmd;
+    let res = match ctx.get_or_create(hash).await {
+        Ok(handle) => import_bao_impl(size, rx, handle, ctx).await,
+        Err(cause) => Err(cause),
+    };
+    trace!("{res:?}");
+    tx.send(res).await.ok();
 }
 
-/// Predicate for filtering entries in a redb table.
-pub(crate) type FilterPredicate<K, V> =
-    Box<dyn Fn(u64, AccessGuard<K>, AccessGuard<V>) -> Option<(K, V)> + Send + Sync>;
-
-/// Storage that is using a redb database for small files and files for
-/// large files.
-#[derive(Debug, Clone)]
-pub struct Store(Arc<StoreInner>);
+fn chunk_range(leaf: &Leaf) -> ChunkRanges {
+    let start = ChunkNum::chunks(leaf.offset);
+    let end = ChunkNum::chunks(leaf.offset + leaf.data.len() as u64);
+    (start..end).into()
+}
 
-impl Store {
-    /// Load or create a new store.
-    pub async fn load(root: impl AsRef<Path>) -> io::Result<Self> {
-        let path = root.as_ref();
-        let db_path = path.join("blobs.db");
-        let options = Options {
-            path: PathOptions::new(path),
-            inline: Default::default(),
-            batch: Default::default(),
-        };
-        Self::new(db_path, options).await
+async fn import_bao_impl(
+    size: NonZeroU64,
+    mut rx: mpsc::Receiver<BaoContentItem>,
+    handle: BaoFileHandle,
+    ctx: HashContext,
+) -> api::Result<()> {
+    trace!(
+        "importing bao: {} {} bytes",
+        handle.hash().fmt_short(),
+        size
+    );
+    let mut batch = Vec::<BaoContentItem>::new();
+    let mut ranges = ChunkRanges::empty();
+    while let Some(item) = rx.recv().await? {
+        // if the batch is not empty, the last item is a leaf and the current item is a parent, write the batch
+        if !batch.is_empty() && batch[batch.len() - 1].is_leaf() && item.is_parent() {
+            let bitfield = Bitfield::new_unchecked(ranges, size.into());
+            handle.write_batch(&batch, &bitfield, &ctx.ctx).await?;
+            batch.clear();
+            ranges = ChunkRanges::empty();
+        }
+        if let BaoContentItem::Leaf(leaf) = &item {
+            let leaf_range = chunk_range(leaf);
+            if is_validated(size, &leaf_range) && size.get() != leaf.offset + leaf.data.len() as u64
+            {
+                return Err(api::Error::io(io::ErrorKind::InvalidData, "invalid size"));
+            }
+            ranges |= leaf_range;
+        }
+        batch.push(item);
     }
-
-    /// Create a new store with custom options.
-    pub async fn new(path: PathBuf, options: Options) -> io::Result<Self> {
-        // spawn_blocking because StoreInner::new creates directories
-        let rt = tokio::runtime::Handle::try_current()
-            .map_err(|_| io::Error::new(io::ErrorKind::Other, "no tokio runtime"))?;
-        let inner =
-            tokio::task::spawn_blocking(move || StoreInner::new_sync(path, options, rt)).await??;
-        Ok(Self(Arc::new(inner)))
+    if !batch.is_empty() {
+        let bitfield = Bitfield::new_unchecked(ranges, size.into());
+        handle.write_batch(&batch, &bitfield, &ctx.ctx).await?;
     }
+    Ok(())
+}
 
-    /// Update the inline options.
-    ///
-    /// When reapply is true, the new options will be applied to all existing
-    /// entries.
-    pub async fn update_inline_options(
-        &self,
-        inline_options: InlineOptions,
-        reapply: bool,
-    ) -> io::Result<()> {
-        Ok(self
-            .0
-            .update_inline_options(inline_options, reapply)
-            .await?)
-    }
-
-    /// Dump the entire content of the database to stdout.
-    pub async fn dump(&self) -> io::Result<()> {
-        Ok(self.0.dump().await?)
-    }
+#[instrument(skip_all, fields(hash = %cmd.hash_short()))]
+async fn observe(cmd: ObserveMsg, ctx: HashContext) {
+    let Ok(handle) = ctx.get_or_create(cmd.hash).await else {
+        return;
+    };
+    handle.subscribe().forward(cmd.tx).await.ok();
 }
 
-#[derive(Debug)]
-struct StoreInner {
-    tx: async_channel::Sender<ActorMessage>,
-    temp: Arc<RwLock<TempCounterMap>>,
-    handle: Option<std::thread::JoinHandle<()>>,
-    path_options: Arc<PathOptions>,
+#[instrument(skip_all, fields(hash = %cmd.hash_short()))]
+async fn export_ranges(mut cmd: ExportRangesMsg, ctx: HashContext) {
+    match ctx.get_or_create(cmd.hash).await {
+        Ok(handle) => {
+            if let Err(cause) = export_ranges_impl(cmd.inner, &mut cmd.tx, handle).await {
+                cmd.tx
+                    .send(ExportRangesItem::Error(cause.into()))
+                    .await
+                    .ok();
+            }
+        }
+        Err(cause) => {
+            cmd.tx.send(ExportRangesItem::Error(cause)).await.ok();
+        }
+    }
 }
 
-impl TagDrop for RwLock<TempCounterMap> {
-    fn on_drop(&self, content: &HashAndFormat) {
-        self.write().unwrap().dec(content);
+async fn export_ranges_impl(
+    cmd: ExportRangesRequest,
+    tx: &mut mpsc::Sender<ExportRangesItem>,
+    handle: BaoFileHandle,
+) -> io::Result<()> {
+    let ExportRangesRequest { ranges, hash } = cmd;
+    trace!(
+        "exporting ranges: {hash} {ranges:?} size={}",
+        handle.current_size()?
+    );
+    debug_assert!(handle.hash() == hash, "hash mismatch");
+    let bitfield = handle.bitfield()?;
+    let data = handle.data_reader();
+    let size = bitfield.size();
+    for range in ranges.iter() {
+        let range = match range {
+            RangeSetRange::Range(range) => size.min(*range.start)..size.min(*range.end),
+            RangeSetRange::RangeFrom(range) => size.min(*range.start)..size,
+        };
+        let requested = ChunkRanges::bytes(range.start..range.end);
+        if !bitfield.ranges.is_superset(&requested) {
+            return Err(io::Error::other(format!(
+                "missing range: {requested:?}, present: {bitfield:?}",
+            )));
+        }
+        let bs = 1024;
+        let mut offset = range.start;
+        loop {
+            let end: u64 = (offset + bs).min(range.end);
+            let size = (end - offset) as usize;
+            tx.send(ExportRangesItem::Data(Leaf {
+                offset,
+                data: data.read_bytes_at(offset, size)?,
+            }))
+            .await?;
+            offset = end;
+            if offset >= range.end {
+                break;
+            }
+        }
     }
+    Ok(())
 }
 
-impl TagCounter for RwLock<TempCounterMap> {
-    fn on_create(&self, content: &HashAndFormat) {
-        self.write().unwrap().inc(content);
+#[instrument(skip_all, fields(hash = %cmd.hash_short()))]
+async fn export_bao(mut cmd: ExportBaoMsg, ctx: HashContext) {
+    match ctx.get_or_create(cmd.hash).await {
+        Ok(handle) => {
+            if let Err(cause) = export_bao_impl(cmd.inner, &mut cmd.tx, handle).await {
+                cmd.tx
+                    .send(bao_tree::io::EncodeError::Io(io::Error::other(cause)).into())
+                    .await
+                    .ok();
+            }
+        }
+        Err(cause) => {
+            let cause = anyhow::anyhow!("failed to open file: {cause}");
+            cmd.tx
+                .send(bao_tree::io::EncodeError::Io(io::Error::other(cause)).into())
+                .await
+                .ok();
+        }
     }
 }
 
-impl StoreInner {
-    fn new_sync(path: PathBuf, options: Options, rt: tokio::runtime::Handle) -> io::Result<Self> {
-        tracing::trace!(
-            "creating data directory: {}",
-            options.path.data_path.display()
-        );
-        std::fs::create_dir_all(&options.path.data_path)?;
-        tracing::trace!(
-            "creating temp directory: {}",
-            options.path.temp_path.display()
-        );
-        std::fs::create_dir_all(&options.path.temp_path)?;
-        tracing::trace!(
-            "creating parent directory for db file{}",
-            path.parent().unwrap().display()
-        );
-        std::fs::create_dir_all(path.parent().unwrap())?;
-        let temp: Arc<RwLock<TempCounterMap>> = Default::default();
-        let (actor, tx) = Actor::new(&path, options.clone(), temp.clone(), rt.clone())?;
-        let handle = std::thread::Builder::new()
-            .name("redb-actor".to_string())
-            .spawn(move || {
-                rt.block_on(async move {
-                    if let Err(cause) = actor.run_batched().await {
-                        tracing::error!("redb actor failed: {}", cause);
-                    }
-                });
-            })
-            .expect("failed to spawn thread");
-        Ok(Self {
-            tx,
-            temp,
-            handle: Some(handle),
-            path_options: Arc::new(options.path),
-        })
-    }
-
-    pub async fn get(&self, hash: Hash) -> OuterResult<Option<BaoFileHandle>> {
-        let (tx, rx) = oneshot::channel();
-        self.tx.send(ActorMessage::Get { hash, tx }).await?;
-        Ok(rx.await??)
-    }
-
-    async fn get_or_create(&self, hash: Hash) -> OuterResult<BaoFileHandle> {
-        let (tx, rx) = oneshot::channel();
-        self.tx.send(ActorMessage::GetOrCreate { hash, tx }).await?;
-        Ok(rx.await??)
-    }
-
-    async fn blobs(&self) -> OuterResult<Vec<io::Result<Hash>>> {
-        let (tx, rx) = oneshot::channel();
-        let filter: FilterPredicate<Hash, EntryState> = Box::new(|_i, k, v| {
-            let v = v.value();
-            if let EntryState::Complete { .. } = &v {
-                Some((k.value(), v))
-            } else {
-                None
-            }
-        });
-        self.tx.send(ActorMessage::Blobs { filter, tx }).await?;
-        let blobs = rx.await?;
-        let res = blobs?
-            .into_iter()
-            .map(|r| {
-                r.map(|(hash, _)| hash)
-                    .map_err(|e| ActorError::from(e).into())
-            })
-            .collect::<Vec<_>>();
-        Ok(res)
-    }
-
-    async fn partial_blobs(&self) -> OuterResult<Vec<io::Result<Hash>>> {
-        let (tx, rx) = oneshot::channel();
-        let filter: FilterPredicate<Hash, EntryState> = Box::new(|_i, k, v| {
-            let v = v.value();
-            if let EntryState::Partial { .. } = &v {
-                Some((k.value(), v))
-            } else {
-                None
-            }
-        });
-        self.tx.send(ActorMessage::Blobs { filter, tx }).await?;
-        let blobs = rx.await?;
-        let res = blobs?
-            .into_iter()
-            .map(|r| {
-                r.map(|(hash, _)| hash)
-                    .map_err(|e| ActorError::from(e).into())
-            })
-            .collect::<Vec<_>>();
-        Ok(res)
-    }
-
-    async fn tags(
-        &self,
-        from: Option<Tag>,
-        to: Option<Tag>,
-    ) -> OuterResult<Vec<io::Result<(Tag, HashAndFormat)>>> {
-        let (tx, rx) = oneshot::channel();
-        self.tx.send(ActorMessage::Tags { from, to, tx }).await?;
-        let tags = rx.await?;
-        // transform the internal error type into io::Error
-        let tags = tags?
-            .into_iter()
-            .map(|r| r.map_err(|e| ActorError::from(e).into()))
-            .collect();
-        Ok(tags)
-    }
-
-    async fn set_tag(&self, tag: Tag, value: HashAndFormat) -> OuterResult<()> {
-        let (tx, rx) = oneshot::channel();
-        self.tx
-            .send(ActorMessage::SetTag { tag, value, tx })
-            .await?;
-        Ok(rx.await??)
-    }
-
-    async fn delete_tags(&self, from: Option<Tag>, to: Option<Tag>) -> OuterResult<()> {
-        let (tx, rx) = oneshot::channel();
-        self.tx
-            .send(ActorMessage::DeleteTags { from, to, tx })
-            .await?;
-        Ok(rx.await??)
-    }
-
-    async fn create_tag(&self, hash: HashAndFormat) -> OuterResult<Tag> {
-        let (tx, rx) = oneshot::channel();
-        self.tx.send(ActorMessage::CreateTag { hash, tx }).await?;
-        Ok(rx.await??)
-    }
-
-    async fn rename_tag(&self, from: Tag, to: Tag) -> OuterResult<()> {
-        let (tx, rx) = oneshot::channel();
-        self.tx
-            .send(ActorMessage::RenameTag { from, to, tx })
-            .await?;
-        Ok(rx.await??)
-    }
-
-    async fn delete(&self, hashes: Vec<Hash>) -> OuterResult<()> {
-        let (tx, rx) = oneshot::channel();
-        self.tx.send(ActorMessage::Delete { hashes, tx }).await?;
-        Ok(rx.await??)
-    }
-
-    async fn gc_delete(&self, hashes: Vec<Hash>) -> OuterResult<()> {
-        let (tx, rx) = oneshot::channel();
-        self.tx.send(ActorMessage::GcDelete { hashes, tx }).await?;
-        Ok(rx.await??)
-    }
-
-    async fn gc_start(&self) -> OuterResult<()> {
-        let (tx, rx) = oneshot::channel();
-        self.tx.send(ActorMessage::GcStart { tx }).await?;
-        Ok(rx.await?)
-    }
-
-    async fn entry_status(&self, hash: &Hash) -> OuterResult<EntryStatus> {
-        let (tx, rx) = oneshot::channel();
-        self.tx
-            .send(ActorMessage::EntryStatus { hash: *hash, tx })
-            .await?;
-        Ok(rx.await??)
-    }
-
-    fn entry_status_sync(&self, hash: &Hash) -> OuterResult<EntryStatus> {
-        let (tx, rx) = oneshot::channel();
-        self.tx
-            .send_blocking(ActorMessage::EntryStatus { hash: *hash, tx })?;
-        Ok(rx.recv()??)
-    }
+async fn export_bao_impl(
+    cmd: ExportBaoRequest,
+    tx: &mut mpsc::Sender<EncodedItem>,
+    handle: BaoFileHandle,
+) -> anyhow::Result<()> {
+    let ExportBaoRequest { ranges, hash } = cmd;
+    debug_assert!(handle.hash() == hash, "hash mismatch");
+    let outboard = handle.outboard()?;
+    let size = outboard.tree.size();
+    if size == 0 && hash != Hash::EMPTY {
+        // we have no data whatsoever, so we stop here
+        return Ok(());
+    }
+    trace!("exporting bao: {hash} {ranges:?} size={size}",);
+    let data = handle.data_reader();
+    let tx = BaoTreeSender::new(tx);
+    traverse_ranges_validated(data, outboard, &ranges, tx).await?;
+    Ok(())
+}
 
-    async fn complete(&self, entry: Entry) -> OuterResult<()> {
-        self.tx
-            .send(ActorMessage::OnComplete { handle: entry })
-            .await?;
-        Ok(())
+#[instrument(skip_all, fields(hash = %cmd.hash_short()))]
+async fn export_path(cmd: ExportPathMsg, ctx: HashContext) {
+    let ExportPathMsg { inner, mut tx, .. } = cmd;
+    if let Err(cause) = export_path_impl(inner, &mut tx, ctx).await {
+        tx.send(cause.into()).await.ok();
     }
+}
 
-    async fn export(
-        &self,
-        hash: Hash,
-        target: PathBuf,
-        mode: ExportMode,
-        progress: ExportProgressCb,
-    ) -> OuterResult<()> {
-        tracing::debug!(
-            "exporting {} to {} using mode {:?}",
-            hash.to_hex(),
-            target.display(),
-            mode
-        );
-        if !target.is_absolute() {
-            return Err(io::Error::new(
+async fn export_path_impl(
+    cmd: ExportPathRequest,
+    tx: &mut mpsc::Sender<ExportProgressItem>,
+    ctx: HashContext,
+) -> api::Result<()> {
+    let ExportPathRequest { mode, target, .. } = cmd;
+    if !target.is_absolute() {
+        return Err(api::Error::io(
+            io::ErrorKind::InvalidInput,
+            "path is not absolute",
+        ));
+    }
+    if let Some(parent) = target.parent() {
+        fs::create_dir_all(parent)?;
+    }
+    let _guard = ctx.lock().await;
+    let state = ctx.get_entry_state(cmd.hash).await?;
+    let (data_location, outboard_location) = match state {
+        Some(EntryState::Complete {
+            data_location,
+            outboard_location,
+        }) => (data_location, outboard_location),
+        Some(EntryState::Partial { .. }) => {
+            return Err(api::Error::io(
                 io::ErrorKind::InvalidInput,
-                "target path must be absolute",
-            )
-            .into());
+                "cannot export partial entry",
+            ));
         }
-        let parent = target.parent().ok_or_else(|| {
-            OuterError::from(io::Error::new(
-                io::ErrorKind::InvalidInput,
-                "target path has no parent directory",
-            ))
-        })?;
-        std::fs::create_dir_all(parent)?;
-        let temp_tag = self.temp.temp_tag(HashAndFormat::raw(hash));
-        let (tx, rx) = oneshot::channel();
-        self.tx
-            .send(ActorMessage::Export {
-                cmd: Export {
-                    temp_tag,
-                    target,
-                    mode,
-                    progress,
-                },
-                tx,
-            })
-            .await?;
-        Ok(rx.await??)
-    }
-
-    async fn consistency_check(
-        &self,
-        repair: bool,
-        progress: BoxedProgressSender<ConsistencyCheckProgress>,
-    ) -> OuterResult<()> {
-        let (tx, rx) = oneshot::channel();
-        self.tx
-            .send(ActorMessage::Fsck {
-                repair,
-                progress,
-                tx,
-            })
-            .await?;
-        Ok(rx.await??)
-    }
-
-    async fn update_inline_options(
-        &self,
-        inline_options: InlineOptions,
-        reapply: bool,
-    ) -> OuterResult<()> {
-        let (tx, rx) = oneshot::channel();
-        self.tx
-            .send(ActorMessage::UpdateInlineOptions {
-                inline_options,
-                reapply,
-                tx,
-            })
-            .await?;
-        Ok(rx.await?)
-    }
-
-    async fn dump(&self) -> OuterResult<()> {
-        self.tx.send(ActorMessage::Dump).await?;
-        Ok(())
-    }
-
-    async fn sync(&self) -> OuterResult<()> {
-        let (tx, rx) = oneshot::channel();
-        self.tx.send(ActorMessage::Sync { tx }).await?;
-        Ok(rx.await?)
-    }
-
-    fn import_file_sync(
-        &self,
-        path: PathBuf,
-        mode: ImportMode,
-        format: BlobFormat,
-        progress: impl ProgressSender<Msg = ImportProgress> + IdGenerator,
-    ) -> OuterResult<(TempTag, u64)> {
-        if !path.is_absolute() {
-            return Err(
-                io::Error::new(io::ErrorKind::InvalidInput, "path must be absolute").into(),
-            );
+        None => {
+            return Err(api::Error::io(io::ErrorKind::NotFound, "no entry found"));
         }
-        if !path.is_file() && !path.is_symlink() {
-            return Err(io::Error::new(
-                io::ErrorKind::InvalidInput,
-                "path is not a file or symlink",
-            )
-            .into());
-        }
-        let id = progress.new_id();
-        progress.blocking_send(ImportProgress::Found {
-            id,
-            name: path.to_string_lossy().to_string(),
-        })?;
-        let file = match mode {
-            ImportMode::TryReference => ImportSource::External(path),
-            ImportMode::Copy => {
-                if std::fs::metadata(&path)?.len() < 16 * 1024 {
-                    // we don't know if the data will be inlined since we don't
-                    // have the inline options here. But still for such a small file
-                    // it does not seem worth it do to the temp file ceremony.
-                    let data = std::fs::read(&path)?;
-                    ImportSource::Memory(data.into())
-                } else {
-                    let temp_path = self.temp_file_name();
-                    // copy the data, since it is not stable
-                    progress.try_send(ImportProgress::CopyProgress { id, offset: 0 })?;
-                    if reflink_copy::reflink_or_copy(&path, &temp_path)?.is_none() {
-                        tracing::debug!("reflinked {} to {}", path.display(), temp_path.display());
-                    } else {
-                        tracing::debug!("copied {} to {}", path.display(), temp_path.display());
+    };
+    trace!("exporting {} to {}", cmd.hash.to_hex(), target.display());
+    let data = match data_location {
+        DataLocation::Inline(data) => MemOrFile::Mem(data),
+        DataLocation::Owned(size) => {
+            MemOrFile::File((ctx.options().path.data_path(&cmd.hash), size))
+        }
+        DataLocation::External(paths, size) => MemOrFile::File((
+            paths
+                .into_iter()
+                .next()
+                .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "no external data path"))?,
+            size,
+        )),
+    };
+    let size = match &data {
+        MemOrFile::Mem(data) => data.len() as u64,
+        MemOrFile::File((_, size)) => *size,
+    };
+    tx.send(ExportProgressItem::Size(size))
+        .await
+        .map_err(api::Error::other)?;
+    match data {
+        MemOrFile::Mem(data) => {
+            let mut target = fs::File::create(&target)?;
+            target.write_all(&data)?;
+        }
+        MemOrFile::File((source_path, size)) => match mode {
+            ExportMode::Copy => {
+                let source = fs::File::open(&source_path)?;
+                let mut target = fs::File::create(&target)?;
+                copy_with_progress(&source, size, &mut target, tx).await?
+            }
+            ExportMode::TryReference => {
+                match std::fs::rename(&source_path, &target) {
+                    Ok(()) => {}
+                    Err(cause) => {
+                        const ERR_CROSS: i32 = 18;
+                        if cause.raw_os_error() == Some(ERR_CROSS) {
+                            let source = fs::File::open(&source_path)?;
+                            let mut target = fs::File::create(&target)?;
+                            copy_with_progress(&source, size, &mut target, tx).await?;
+                        } else {
+                            return Err(cause.into());
+                        }
                     }
-                    // copy progress for size will be called in finalize_import_sync
-                    ImportSource::TempFile(temp_path)
                 }
+                ctx.set(
+                    cmd.hash,
+                    EntryState::Complete {
+                        data_location: DataLocation::External(vec![target], size),
+                        outboard_location,
+                    },
+                )
+                .await?;
             }
-        };
-        let (tag, size) = self.finalize_import_sync(file, format, id, progress)?;
-        Ok((tag, size))
-    }
-
-    fn import_bytes_sync(&self, data: Bytes, format: BlobFormat) -> OuterResult<TempTag> {
-        let id = 0;
-        let file = ImportSource::Memory(data);
-        let progress = IgnoreProgressSender::default();
-        let (tag, _size) = self.finalize_import_sync(file, format, id, progress)?;
-        Ok(tag)
-    }
-
-    fn finalize_import_sync(
-        &self,
-        file: ImportSource,
-        format: BlobFormat,
-        id: u64,
-        progress: impl ProgressSender<Msg = ImportProgress> + IdGenerator,
-    ) -> OuterResult<(TempTag, u64)> {
-        let data_size = file.len()?;
-        tracing::debug!("finalize_import_sync {:?} {}", file, data_size);
-        progress.blocking_send(ImportProgress::Size {
-            id,
-            size: data_size,
-        })?;
-        let progress2 = progress.clone();
-        let (hash, outboard) = match file.content() {
-            MemOrFile::File(path) => {
-                let span = trace_span!("outboard.compute", path = %path.display());
-                let _guard = span.enter();
-                let file = std::fs::File::open(path)?;
-                compute_outboard(file, data_size, move |offset| {
-                    Ok(progress2.try_send(ImportProgress::OutboardProgress { id, offset })?)
-                })?
-            }
-            MemOrFile::Mem(bytes) => {
-                // todo: progress? usually this is will be small enough that progress might not be needed.
-                compute_outboard(bytes, data_size, |_| Ok(()))?
-            }
-        };
-        progress.blocking_send(ImportProgress::OutboardDone { id, hash })?;
-        // from here on, everything related to the hash is protected by the temp tag
-        let tag = self.temp.temp_tag(HashAndFormat { hash, format });
-        let hash = *tag.hash();
-        // blocking send for the import
-        let (tx, rx) = oneshot::channel();
-        self.tx.send_blocking(ActorMessage::Import {
-            cmd: Import {
-                content_id: HashAndFormat { hash, format },
-                source: file,
-                outboard,
-                data_size,
-            },
-            tx,
-        })?;
-        Ok(rx.recv()??)
-    }
-
-    fn temp_file_name(&self) -> PathBuf {
-        self.path_options.temp_file_name()
+        },
     }
+    tx.send(ExportProgressItem::Done)
+        .await
+        .map_err(api::Error::other)?;
+    Ok(())
+}
 
-    async fn shutdown(&self) {
-        let (tx, rx) = oneshot::channel();
-        self.tx
-            .send(ActorMessage::Shutdown { tx: Some(tx) })
+async fn copy_with_progress(
+    file: impl ReadAt,
+    size: u64,
+    target: &mut impl Write,
+    tx: &mut mpsc::Sender<ExportProgressItem>,
+) -> io::Result<()> {
+    let mut offset = 0;
+    let mut buf = vec![0u8; 1024 * 1024];
+    while offset < size {
+        let remaining = buf.len().min((size - offset) as usize);
+        let buf: &mut [u8] = &mut buf[..remaining];
+        file.read_exact_at(offset, buf)?;
+        target.write_all(buf)?;
+        tx.try_send(ExportProgressItem::CopyProgress(offset))
             .await
-            .ok();
-        rx.await.ok();
+            .map_err(|_e| io::Error::other(""))?;
+        yield_now().await;
+        offset += buf.len() as u64;
     }
+    Ok(())
 }
 
-impl Drop for StoreInner {
-    fn drop(&mut self) {
-        if let Some(handle) = self.handle.take() {
-            self.tx
-                .send_blocking(ActorMessage::Shutdown { tx: None })
-                .ok();
-            handle.join().ok();
+impl FsStore {
+    /// Load or create a new store.
+    pub async fn load(root: impl AsRef<Path>) -> anyhow::Result<Self> {
+        let path = root.as_ref();
+        let db_path = path.join("blobs.db");
+        let options = Options::new(path);
+        Self::load_with_opts(db_path, options).await
+    }
+
+    /// Load or create a new store with custom options, returning an additional sender for file store specific commands.
+    pub async fn load_with_opts(db_path: PathBuf, options: Options) -> anyhow::Result<FsStore> {
+        let rt = tokio::runtime::Builder::new_multi_thread()
+            .thread_name("iroh-blob-store")
+            .enable_time()
+            .build()?;
+        let handle = rt.handle().clone();
+        let (commands_tx, commands_rx) = tokio::sync::mpsc::channel(100);
+        let (fs_commands_tx, fs_commands_rx) = tokio::sync::mpsc::channel(100);
+        let gc_config = options.gc.clone();
+        let actor = handle
+            .spawn(Actor::new(
+                db_path,
+                rt.into(),
+                commands_rx,
+                fs_commands_rx,
+                fs_commands_tx.clone(),
+                Arc::new(options),
+            ))
+            .await??;
+        handle.spawn(actor.run());
+        let store = FsStore::new(commands_tx.into(), fs_commands_tx);
+        if let Some(config) = gc_config {
+            handle.spawn(run_gc(store.deref().clone(), config));
         }
+        Ok(store)
     }
 }
 
-struct ActorState {
-    handles: BTreeMap<Hash, BaoFileHandleWeak>,
-    protected: BTreeSet<Hash>,
-    temp: Arc<RwLock<TempCounterMap>>,
-    msgs_rx: async_channel::Receiver<ActorMessage>,
-    create_options: Arc<BaoFileConfig>,
-    options: Options,
-    rt: tokio::runtime::Handle,
-}
-
-/// The actor for the redb store.
+/// A file based store.
 ///
-/// It is split into the database and the rest of the state to allow for split
-/// borrows in the message handlers.
-struct Actor {
-    db: redb::Database,
-    state: ActorState,
-}
-
-/// Error type for message handler functions of the redb actor.
+/// A store can be created using [`load`](FsStore::load) or [`load_with_opts`](FsStore::load_with_opts).
+/// Load will use the default options and create the required directories, while load_with_opts allows
+/// you to customize the options and the location of the database. Both variants will create the database
+/// if it does not exist, and load an existing database if one is found at the configured location.
 ///
-/// What can go wrong are various things with redb, as well as io errors related
-/// to files other than redb.
-#[derive(Debug, thiserror::Error)]
-pub(crate) enum ActorError {
-    #[error("table error: {0}")]
-    Table(#[from] redb::TableError),
-    #[error("database error: {0}")]
-    Database(#[from] redb::DatabaseError),
-    #[error("transaction error: {0}")]
-    Transaction(#[from] redb::TransactionError),
-    #[error("commit error: {0}")]
-    Commit(#[from] redb::CommitError),
-    #[error("storage error: {0}")]
-    Storage(#[from] redb::StorageError),
-    #[error("io error: {0}")]
-    Io(#[from] io::Error),
-    #[error("inconsistent database state: {0}")]
-    Inconsistent(String),
-    #[error("error during database migration: {0}")]
-    Migration(#[source] anyhow::Error),
-}
-
-impl From<ActorError> for io::Error {
-    fn from(e: ActorError) -> Self {
-        match e {
-            ActorError::Io(e) => e,
-            e => io::Error::new(io::ErrorKind::Other, e),
-        }
-    }
+/// In addition to implementing the [`Store`](`crate::api::Store`) API via [`Deref`](`std::ops::Deref`),
+/// there are a few additional methods that are specific to file based stores, such as [`dump`](FsStore::dump).
+#[derive(Debug, Clone)]
+pub struct FsStore {
+    sender: ApiClient,
+    db: tokio::sync::mpsc::Sender<InternalCommand>,
 }
 
-/// Result type for handler functions of the redb actor.
-///
-/// See [`ActorError`] for what can go wrong.
-pub(crate) type ActorResult<T> = std::result::Result<T, ActorError>;
-
-/// Error type for calling the redb actor from the store.
-///
-/// What can go wrong is all the things in [`ActorError`] and in addition
-/// sending and receiving messages.
-#[derive(Debug, thiserror::Error)]
-pub(crate) enum OuterError {
-    #[error("inner error: {0}")]
-    Inner(#[from] ActorError),
-    #[error("send error")]
-    Send,
-    #[error("progress send error: {0}")]
-    ProgressSend(#[from] ProgressSendError),
-    #[error("recv error: {0}")]
-    Recv(#[from] oneshot::RecvError),
-    #[error("recv error: {0}")]
-    AsyncChannelRecv(#[from] async_channel::RecvError),
-    #[error("join error: {0}")]
-    JoinTask(#[from] tokio::task::JoinError),
-}
+impl Deref for FsStore {
+    type Target = Store;
 
-impl From<async_channel::SendError<ActorMessage>> for OuterError {
-    fn from(_e: async_channel::SendError<ActorMessage>) -> Self {
-        OuterError::Send
+    fn deref(&self) -> &Self::Target {
+        Store::ref_from_sender(&self.sender)
     }
 }
 
-/// Result type for calling the redb actor from the store.
-///
-/// See [`OuterError`] for what can go wrong.
-pub(crate) type OuterResult<T> = std::result::Result<T, OuterError>;
-
-impl From<io::Error> for OuterError {
-    fn from(e: io::Error) -> Self {
-        OuterError::Inner(ActorError::Io(e))
+impl AsRef<Store> for FsStore {
+    fn as_ref(&self) -> &Store {
+        self.deref()
     }
 }
 
-impl From<OuterError> for io::Error {
-    fn from(e: OuterError) -> Self {
-        match e {
-            OuterError::Inner(ActorError::Io(e)) => e,
-            e => io::Error::new(io::ErrorKind::Other, e),
+impl FsStore {
+    fn new(
+        sender: irpc::LocalSender<proto::Command, proto::StoreService>,
+        db: tokio::sync::mpsc::Sender<InternalCommand>,
+    ) -> Self {
+        Self {
+            sender: sender.into(),
+            db,
         }
     }
-}
-
-impl super::Map for Store {
-    type Entry = Entry;
-
-    async fn get(&self, hash: &Hash) -> io::Result<Option<Self::Entry>> {
-        Ok(self.0.get(*hash).await?)
-    }
-}
-
-impl super::MapMut for Store {
-    type EntryMut = Entry;
-
-    async fn get_or_create(&self, hash: Hash, _size: u64) -> io::Result<Self::EntryMut> {
-        Ok(self.0.get_or_create(hash).await?)
-    }
-
-    async fn entry_status(&self, hash: &Hash) -> io::Result<EntryStatus> {
-        Ok(self.0.entry_status(hash).await?)
-    }
-
-    async fn get_mut(&self, hash: &Hash) -> io::Result<Option<Self::EntryMut>> {
-        self.get(hash).await
-    }
-
-    async fn insert_complete(&self, entry: Self::EntryMut) -> io::Result<()> {
-        Ok(self.0.complete(entry).await?)
-    }
 
-    fn entry_status_sync(&self, hash: &Hash) -> io::Result<EntryStatus> {
-        Ok(self.0.entry_status_sync(hash)?)
+    pub async fn dump(&self) -> anyhow::Result<()> {
+        let (tx, rx) = oneshot::channel();
+        self.db
+            .send(
+                meta::Dump {
+                    tx,
+                    span: tracing::Span::current(),
+                }
+                .into(),
+            )
+            .await?;
+        rx.await??;
+        Ok(())
     }
 }
 
-impl super::ReadableStore for Store {
-    async fn blobs(&self) -> io::Result<super::DbIter<Hash>> {
-        Ok(Box::new(self.0.blobs().await?.into_iter()))
-    }
+#[cfg(test)]
+pub mod tests {
+    use core::panic;
+    use std::collections::{HashMap, HashSet};
 
-    async fn partial_blobs(&self) -> io::Result<super::DbIter<Hash>> {
-        Ok(Box::new(self.0.partial_blobs().await?.into_iter()))
-    }
+    use bao_tree::{
+        io::{outboard::PreOrderMemOutboard, round_up_to_chunks_groups},
+        ChunkRanges,
+    };
+    use n0_future::{stream, Stream, StreamExt};
+    use testresult::TestResult;
+    use walkdir::WalkDir;
+
+    use super::*;
+    use crate::{
+        api::blobs::Bitfield,
+        store::{
+            util::{read_checksummed, SliceInfoExt, Tag},
+            HashAndFormat, IROH_BLOCK_SIZE,
+        },
+    };
 
-    async fn tags(
-        &self,
-        from: Option<Tag>,
-        to: Option<Tag>,
-    ) -> io::Result<super::DbIter<(Tag, HashAndFormat)>> {
-        Ok(Box::new(self.0.tags(from, to).await?.into_iter()))
+    /// Interesting sizes for testing.
+    pub const INTERESTING_SIZES: [usize; 8] = [
+        0,               // annoying corner case - always present, handled by the api
+        1,               // less than 1 chunk, data inline, outboard not needed
+        1024,            // exactly 1 chunk, data inline, outboard not needed
+        1024 * 16 - 1,   // less than 1 chunk group, data inline, outboard not needed
+        1024 * 16,       // exactly 1 chunk group, data inline, outboard not needed
+        1024 * 16 + 1,   // data file, outboard inline (just 1 hash pair)
+        1024 * 1024,     // data file, outboard inline (many hash pairs)
+        1024 * 1024 * 8, // data file, outboard file
+    ];
+
+    /// Create n0 flavoured bao. Note that this can be used to request ranges below a chunk group size,
+    /// which can not be exported via bao because we don't store hashes below the chunk group level.
+    pub fn create_n0_bao(data: &[u8], ranges: &ChunkRanges) -> anyhow::Result<(Hash, Vec<u8>)> {
+        let outboard = PreOrderMemOutboard::create(data, IROH_BLOCK_SIZE);
+        let mut encoded = Vec::new();
+        let size = data.len() as u64;
+        encoded.extend_from_slice(&size.to_le_bytes());
+        bao_tree::io::sync::encode_ranges_validated(data, &outboard, ranges, &mut encoded)?;
+        Ok((outboard.root.into(), encoded))
+    }
+
+    pub fn round_up_request(size: u64, ranges: &ChunkRanges) -> ChunkRanges {
+        let last_chunk = ChunkNum::chunks(size);
+        let data_range = ChunkRanges::from(..last_chunk);
+        let ranges = if !data_range.intersects(ranges) && !ranges.is_empty() {
+            if last_chunk == 0 {
+                ChunkRanges::all()
+            } else {
+                ChunkRanges::from(last_chunk - 1..)
+            }
+        } else {
+            ranges.clone()
+        };
+        round_up_to_chunks_groups(ranges, IROH_BLOCK_SIZE)
+    }
+
+    fn create_n0_bao_full(
+        data: &[u8],
+        ranges: &ChunkRanges,
+    ) -> anyhow::Result<(Hash, ChunkRanges, Vec<u8>)> {
+        let ranges = round_up_request(data.len() as u64, ranges);
+        let (hash, encoded) = create_n0_bao(data, &ranges)?;
+        Ok((hash, ranges, encoded))
+    }
+
+    #[tokio::test]
+    // #[traced_test]
+    async fn test_observe() -> TestResult<()> {
+        tracing_subscriber::fmt::try_init().ok();
+        let testdir = tempfile::tempdir()?;
+        let db_dir = testdir.path().join("db");
+        let options = Options::new(&db_dir);
+        let store = FsStore::load_with_opts(db_dir.join("blobs.db"), options).await?;
+        let sizes = INTERESTING_SIZES;
+        for size in sizes {
+            let data = test_data(size);
+            let ranges = ChunkRanges::all();
+            let (hash, bao) = create_n0_bao(&data, &ranges)?;
+            let obs = store.observe(hash);
+            let task = tokio::spawn(async move {
+                obs.await_completion().await?;
+                api::Result::Ok(())
+            });
+            store.import_bao_bytes(hash, ranges, bao).await?;
+            task.await??;
+        }
+        Ok(())
     }
 
-    fn temp_tags(&self) -> Box<dyn Iterator<Item = HashAndFormat> + Send + Sync + 'static> {
-        Box::new(self.0.temp.read().unwrap().keys())
+    /// Generate test data for size n.
+    ///
+    /// We don't really care about the content, since we assume blake3 works.
+    /// The only thing it should not be is all zeros, since that is what you
+    /// will get for a gap.
+    pub fn test_data(n: usize) -> Bytes {
+        let mut res = Vec::with_capacity(n);
+        // Using uppercase A-Z (65-90), 26 possible characters
+        for i in 0..n {
+            // Change character every 1024 bytes
+            let block_num = i / 1024;
+            // Map to uppercase A-Z range (65-90)
+            let ascii_val = 65 + (block_num % 26) as u8;
+            res.push(ascii_val);
+        }
+        Bytes::from(res)
+    }
+
+    // import data via import_bytes, check that we can observe it and that it is complete
+    #[tokio::test]
+    async fn test_import_byte_stream() -> TestResult<()> {
+        tracing_subscriber::fmt::try_init().ok();
+        let testdir = tempfile::tempdir()?;
+        let db_dir = testdir.path().join("db");
+        let store = FsStore::load(db_dir).await?;
+        for size in INTERESTING_SIZES {
+            let expected = test_data(size);
+            let expected_hash = Hash::new(&expected);
+            let stream = bytes_to_stream(expected.clone(), 1023);
+            let obs = store.observe(expected_hash);
+            let tt = store.add_stream(stream).await.temp_tag().await?;
+            assert_eq!(expected_hash, *tt.hash());
+            // we must at some point see completion, otherwise the test will hang
+            obs.await_completion().await?;
+            let actual = store.get_bytes(expected_hash).await?;
+            // check that the data is there
+            assert_eq!(&expected, &actual);
+        }
+        Ok(())
     }
 
-    async fn consistency_check(
-        &self,
-        repair: bool,
-        tx: BoxedProgressSender<ConsistencyCheckProgress>,
-    ) -> io::Result<()> {
-        self.0.consistency_check(repair, tx.clone()).await?;
+    // import data via import_bytes, check that we can observe it and that it is complete
+    #[tokio::test]
+    async fn test_import_bytes() -> TestResult<()> {
+        tracing_subscriber::fmt::try_init().ok();
+        let testdir = tempfile::tempdir()?;
+        let db_dir = testdir.path().join("db");
+        let store = FsStore::load(&db_dir).await?;
+        let sizes = INTERESTING_SIZES;
+        trace!("{}", Options::new(&db_dir).is_inlined_data(16385));
+        for size in sizes {
+            let expected = test_data(size);
+            let expected_hash = Hash::new(&expected);
+            let obs = store.observe(expected_hash);
+            let tt = store.add_bytes(expected.clone()).await?;
+            assert_eq!(expected_hash, tt.hash);
+            // we must at some point see completion, otherwise the test will hang
+            obs.await_completion().await?;
+            let actual = store.get_bytes(expected_hash).await?;
+            // check that the data is there
+            assert_eq!(&expected, &actual);
+        }
+        store.shutdown().await?;
+        dump_dir_full(db_dir)?;
         Ok(())
     }
 
-    async fn export(
-        &self,
-        hash: Hash,
-        target: PathBuf,
-        mode: ExportMode,
-        progress: ExportProgressCb,
-    ) -> io::Result<()> {
-        Ok(self.0.export(hash, target, mode, progress).await?)
+    // import data via import_bytes, check that we can observe it and that it is complete
+    #[tokio::test]
+    #[ignore = "flaky. I need a reliable way to keep the handle alive"]
+    async fn test_roundtrip_bytes_small() -> TestResult<()> {
+        tracing_subscriber::fmt::try_init().ok();
+        let testdir = tempfile::tempdir()?;
+        let db_dir = testdir.path().join("db");
+        let store = FsStore::load(db_dir).await?;
+        for size in INTERESTING_SIZES
+            .into_iter()
+            .filter(|x| *x != 0 && *x <= IROH_BLOCK_SIZE.bytes())
+        {
+            let expected = test_data(size);
+            let expected_hash = Hash::new(&expected);
+            let obs = store.observe(expected_hash);
+            let tt = store.add_bytes(expected.clone()).await?;
+            assert_eq!(expected_hash, tt.hash);
+            let actual = store.get_bytes(expected_hash).await?;
+            // check that the data is there
+            assert_eq!(&expected, &actual);
+            assert_eq!(
+                &expected.addr(),
+                &actual.addr(),
+                "address mismatch for size {size}"
+            );
+            // we must at some point see completion, otherwise the test will hang
+            // keep the handle alive by observing until the end, otherwise the handle
+            // will change and the bytes won't be the same instance anymore
+            obs.await_completion().await?;
+        }
+        store.shutdown().await?;
+        Ok(())
     }
-}
 
-impl super::Store for Store {
-    async fn import_file(
-        &self,
-        path: PathBuf,
-        mode: ImportMode,
-        format: BlobFormat,
-        progress: impl ProgressSender<Msg = ImportProgress> + IdGenerator,
-    ) -> io::Result<(crate::TempTag, u64)> {
-        let this = self.0.clone();
-        Ok(
-            tokio::task::spawn_blocking(move || {
-                this.import_file_sync(path, mode, format, progress)
-            })
-            .await??,
-        )
-    }
-
-    async fn import_bytes(
-        &self,
-        data: bytes::Bytes,
-        format: crate::BlobFormat,
-    ) -> io::Result<crate::TempTag> {
-        let this = self.0.clone();
-        Ok(tokio::task::spawn_blocking(move || this.import_bytes_sync(data, format)).await??)
-    }
-
-    async fn import_stream(
-        &self,
-        mut data: impl Stream<Item = io::Result<Bytes>> + Unpin + Send + 'static,
-        format: BlobFormat,
-        progress: impl ProgressSender<Msg = ImportProgress> + IdGenerator,
-    ) -> io::Result<(TempTag, u64)> {
-        let this = self.clone();
-        let id = progress.new_id();
-        // write to a temp file
-        let temp_data_path = this.0.temp_file_name();
-        let name = temp_data_path
-            .file_name()
-            .expect("just created")
-            .to_string_lossy()
-            .to_string();
-        progress.send(ImportProgress::Found { id, name }).await?;
-        let mut writer = tokio::fs::File::create(&temp_data_path).await?;
-        let mut offset = 0;
-        while let Some(chunk) = data.next().await {
-            let chunk = chunk?;
-            writer.write_all(&chunk).await?;
-            offset += chunk.len() as u64;
-            progress.try_send(ImportProgress::CopyProgress { id, offset })?;
-        }
-        writer.flush().await?;
-        drop(writer);
-        let file = ImportSource::TempFile(temp_data_path);
-        Ok(tokio::task::spawn_blocking(move || {
-            this.0.finalize_import_sync(file, format, id, progress)
-        })
-        .await??)
+    // import data via import_bytes, check that we can observe it and that it is complete
+    #[tokio::test]
+    async fn test_import_path() -> TestResult<()> {
+        tracing_subscriber::fmt::try_init().ok();
+        let testdir = tempfile::tempdir()?;
+        let db_dir = testdir.path().join("db");
+        let store = FsStore::load(db_dir).await?;
+        for size in INTERESTING_SIZES {
+            let expected = test_data(size);
+            let expected_hash = Hash::new(&expected);
+            let path = testdir.path().join(format!("in-{size}"));
+            fs::write(&path, &expected)?;
+            let obs = store.observe(expected_hash);
+            let tt = store.add_path(&path).await?;
+            assert_eq!(expected_hash, tt.hash);
+            // we must at some point see completion, otherwise the test will hang
+            obs.await_completion().await?;
+            let actual = store.get_bytes(expected_hash).await?;
+            // check that the data is there
+            assert_eq!(&expected, &actual, "size={size}");
+        }
+        dump_dir_full(testdir.path())?;
+        Ok(())
     }
 
-    async fn set_tag(&self, name: Tag, hash: HashAndFormat) -> io::Result<()> {
-        Ok(self.0.set_tag(name, hash).await?)
+    // import data via import_bytes, check that we can observe it and that it is complete
+    #[tokio::test]
+    async fn test_export_path() -> TestResult<()> {
+        tracing_subscriber::fmt::try_init().ok();
+        let testdir = tempfile::tempdir()?;
+        let db_dir = testdir.path().join("db");
+        let store = FsStore::load(db_dir).await?;
+        for size in INTERESTING_SIZES {
+            let expected = test_data(size);
+            let expected_hash = Hash::new(&expected);
+            let tt = store.add_bytes(expected.clone()).await?;
+            assert_eq!(expected_hash, tt.hash);
+            let out_path = testdir.path().join(format!("out-{size}"));
+            store.export(expected_hash, &out_path).await?;
+            let actual = fs::read(&out_path)?;
+            assert_eq!(expected, actual);
+        }
+        Ok(())
     }
 
-    async fn delete_tags(&self, from: Option<Tag>, to: Option<Tag>) -> io::Result<()> {
-        Ok(self.0.delete_tags(from, to).await?)
+    #[tokio::test]
+    async fn test_import_bao_ranges() -> TestResult<()> {
+        tracing_subscriber::fmt::try_init().ok();
+        let testdir = tempfile::tempdir()?;
+        let db_dir = testdir.path().join("db");
+        {
+            let store = FsStore::load(&db_dir).await?;
+            let data = test_data(100000);
+            let ranges = ChunkRanges::chunks(16..32);
+            let (hash, bao) = create_n0_bao(&data, &ranges)?;
+            store
+                .import_bao_bytes(hash, ranges.clone(), bao.clone())
+                .await?;
+            let bitfield = store.observe(hash).await?;
+            assert_eq!(bitfield.ranges, ranges);
+            assert_eq!(bitfield.size(), data.len() as u64);
+            let export = store.export_bao(hash, ranges).bao_to_vec().await?;
+            assert_eq!(export, bao);
+        }
+        Ok(())
     }
 
-    async fn create_tag(&self, hash: HashAndFormat) -> io::Result<Tag> {
-        Ok(self.0.create_tag(hash).await?)
+    #[tokio::test]
+    async fn test_import_bao_minimal() -> TestResult<()> {
+        tracing_subscriber::fmt::try_init().ok();
+        let testdir = tempfile::tempdir()?;
+        let sizes = [1];
+        let db_dir = testdir.path().join("db");
+        {
+            let store = FsStore::load(&db_dir).await?;
+            for size in sizes {
+                let data = vec![0u8; size];
+                let (hash, encoded) = create_n0_bao(&data, &ChunkRanges::all())?;
+                let data = Bytes::from(encoded);
+                store
+                    .import_bao_bytes(hash, ChunkRanges::all(), data)
+                    .await?;
+            }
+            store.shutdown().await?;
+        }
+        Ok(())
     }
 
-    async fn rename_tag(&self, from: Tag, to: Tag) -> io::Result<()> {
-        Ok(self.0.rename_tag(from, to).await?)
+    #[tokio::test]
+    async fn test_import_bao_simple() -> TestResult<()> {
+        tracing_subscriber::fmt::try_init().ok();
+        let testdir = tempfile::tempdir()?;
+        let sizes = [1048576];
+        let db_dir = testdir.path().join("db");
+        {
+            let store = FsStore::load(&db_dir).await?;
+            for size in sizes {
+                let data = vec![0u8; size];
+                let (hash, encoded) = create_n0_bao(&data, &ChunkRanges::all())?;
+                let data = Bytes::from(encoded);
+                trace!("importing size={}", size);
+                store
+                    .import_bao_bytes(hash, ChunkRanges::all(), data)
+                    .await?;
+            }
+            store.shutdown().await?;
+        }
+        Ok(())
     }
 
-    async fn delete(&self, hashes: Vec<Hash>) -> io::Result<()> {
-        Ok(self.0.delete(hashes).await?)
+    #[tokio::test]
+    async fn test_import_bao_persistence_full() -> TestResult<()> {
+        tracing_subscriber::fmt::try_init().ok();
+        let testdir = tempfile::tempdir()?;
+        let sizes = INTERESTING_SIZES;
+        let db_dir = testdir.path().join("db");
+        {
+            let store = FsStore::load(&db_dir).await?;
+            for size in sizes {
+                let data = vec![0u8; size];
+                let (hash, encoded) = create_n0_bao(&data, &ChunkRanges::all())?;
+                let data = Bytes::from(encoded);
+                store
+                    .import_bao_bytes(hash, ChunkRanges::all(), data)
+                    .await?;
+            }
+            store.shutdown().await?;
+        }
+        {
+            let store = FsStore::load(&db_dir).await?;
+            for size in sizes {
+                let expected = vec![0u8; size];
+                let hash = Hash::new(&expected);
+                let actual = store
+                    .export_bao(hash, ChunkRanges::all())
+                    .data_to_vec()
+                    .await?;
+                assert_eq!(&expected, &actual);
+            }
+            store.shutdown().await?;
+        }
+        Ok(())
     }
 
-    async fn gc_run<G, Gut>(&self, config: super::GcConfig, protected_cb: G)
-    where
-        G: Fn() -> Gut,
-        Gut: Future<Output = BTreeSet<Hash>> + Send,
-    {
-        tracing::info!("Starting GC task with interval {:?}", config.period);
-        let mut live = BTreeSet::new();
-        'outer: loop {
-            if let Err(cause) = self.0.gc_start().await {
-                tracing::debug!(
-                    "unable to notify the db of GC start: {cause}. Shutting down GC loop."
-                );
-                break;
-            }
-            // do delay before the two phases of GC
-            tokio::time::sleep(config.period).await;
-            tracing::debug!("Starting GC");
-            live.clear();
-
-            let p = protected_cb().await;
-            live.extend(p);
-
-            tracing::debug!("Starting GC mark phase");
-            let live_ref = &mut live;
-            let mut stream = Gen::new(|co| async move {
-                if let Err(e) = super::gc_mark_task(self, live_ref, &co).await {
-                    co.yield_(GcMarkEvent::Error(e)).await;
-                }
-            });
-            while let Some(item) = stream.next().await {
-                match item {
-                    GcMarkEvent::CustomDebug(text) => {
-                        tracing::debug!("{}", text);
-                    }
-                    GcMarkEvent::CustomWarning(text, _) => {
-                        tracing::warn!("{}", text);
-                    }
-                    GcMarkEvent::Error(err) => {
-                        tracing::error!("Fatal error during GC mark {}", err);
-                        continue 'outer;
-                    }
-                }
-            }
-            drop(stream);
-
-            tracing::debug!("Starting GC sweep phase");
-            let live_ref = &live;
-            let mut stream = Gen::new(|co| async move {
-                if let Err(e) = gc_sweep_task(self, live_ref, &co).await {
-                    co.yield_(GcSweepEvent::Error(e)).await;
+    #[tokio::test]
+    async fn test_import_bao_persistence_just_size() -> TestResult<()> {
+        tracing_subscriber::fmt::try_init().ok();
+        let testdir = tempfile::tempdir()?;
+        let sizes = INTERESTING_SIZES;
+        let db_dir = testdir.path().join("db");
+        let just_size = ChunkRanges::last_chunk();
+        {
+            let store = FsStore::load(&db_dir).await?;
+            for size in sizes {
+                let data = test_data(size);
+                let (hash, ranges, encoded) = create_n0_bao_full(&data, &just_size)?;
+                let data = Bytes::from(encoded);
+                if let Err(cause) = store.import_bao_bytes(hash, ranges, data).await {
+                    panic!("failed to import size={size}: {cause}");
                 }
-            });
-            while let Some(item) = stream.next().await {
-                match item {
-                    GcSweepEvent::CustomDebug(text) => {
-                        tracing::debug!("{}", text);
-                    }
-                    GcSweepEvent::CustomWarning(text, _) => {
-                        tracing::warn!("{}", text);
-                    }
-                    GcSweepEvent::Error(err) => {
-                        tracing::error!("Fatal error during GC mark {}", err);
-                        continue 'outer;
-                    }
-                }
-            }
-            if let Some(ref cb) = config.done_callback {
-                cb();
             }
+            store.dump().await?;
+            store.shutdown().await?;
         }
-    }
-
-    fn temp_tag(&self, value: HashAndFormat) -> TempTag {
-        self.0.temp.temp_tag(value)
-    }
-
-    async fn sync(&self) -> io::Result<()> {
-        Ok(self.0.sync().await?)
-    }
-
-    async fn shutdown(&self) {
-        self.0.shutdown().await;
-    }
-}
-
-pub(super) async fn gc_sweep_task(
-    store: &Store,
-    live: &BTreeSet<Hash>,
-    co: &Co<GcSweepEvent>,
-) -> anyhow::Result<()> {
-    let blobs = store.blobs().await?.chain(store.partial_blobs().await?);
-    let mut count = 0;
-    let mut batch = Vec::new();
-    for hash in blobs {
-        let hash = hash?;
-        if !live.contains(&hash) {
-            batch.push(hash);
-            count += 1;
-        }
-        if batch.len() >= 100 {
-            store.0.gc_delete(batch.clone()).await?;
-            batch.clear();
+        {
+            let store = FsStore::load(&db_dir).await?;
+            store.dump().await?;
+            for size in sizes {
+                let data = test_data(size);
+                let (hash, ranges, expected) = create_n0_bao_full(&data, &just_size)?;
+                let actual = match store.export_bao(hash, ranges).bao_to_vec().await {
+                    Ok(actual) => actual,
+                    Err(cause) => panic!("failed to export size={size}: {cause}"),
+                };
+                assert_eq!(&expected, &actual);
+            }
+            store.shutdown().await?;
         }
+        dump_dir_full(testdir.path())?;
+        Ok(())
     }
-    if !batch.is_empty() {
-        store.0.gc_delete(batch).await?;
-    }
-    co.yield_(GcSweepEvent::CustomDebug(format!(
-        "deleted {} blobs",
-        count
-    )))
-    .await;
-    Ok(())
-}
-
-impl Actor {
-    fn new(
-        path: &Path,
-        options: Options,
-        temp: Arc<RwLock<TempCounterMap>>,
-        rt: tokio::runtime::Handle,
-    ) -> ActorResult<(Self, async_channel::Sender<ActorMessage>)> {
-        let db = match redb::Database::create(path) {
-            Ok(db) => db,
-            Err(DatabaseError::UpgradeRequired(1)) => {
-                return Err(ActorError::Migration(anyhow::anyhow!(
-                    "migration from v1 no longer supported"
-                )))
-            }
-            Err(err) => return Err(err.into()),
-        };
 
-        let txn = db.begin_write()?;
-        // create tables and drop them just to create them.
-        let mut t = Default::default();
-        let tables = Tables::new(&txn, &mut t)?;
-        drop(tables);
-        txn.commit()?;
-        // make the channel relatively large. there are some messages that don't
-        // require a response, it's fine if they pile up a bit.
-        let (tx, rx) = async_channel::bounded(1024);
-        let tx2 = tx.clone();
-        let on_file_create: CreateCb = Arc::new(move |hash| {
-            // todo: make the callback allow async
-            tx2.send_blocking(ActorMessage::OnMemSizeExceeded { hash: *hash })
-                .ok();
-            Ok(())
-        });
-        let create_options = BaoFileConfig::new(
-            Arc::new(options.path.data_path.clone()),
-            16 * 1024,
-            Some(on_file_create),
-        );
-        Ok((
-            Self {
-                db,
-                state: ActorState {
-                    temp,
-                    handles: BTreeMap::new(),
-                    protected: BTreeSet::new(),
-                    msgs_rx: rx,
-                    options,
-                    create_options: Arc::new(create_options),
-                    rt,
-                },
-            },
-            tx,
-        ))
-    }
-
-    async fn run_batched(mut self) -> ActorResult<()> {
-        let mut msgs = PeekableFlumeReceiver::new(self.state.msgs_rx.clone());
-        while let Some(msg) = msgs.recv().await {
-            if let ActorMessage::Shutdown { tx } = msg {
-                // Make sure the database is dropped before we send the reply.
-                drop(self);
-                if let Some(tx) = tx {
-                    tx.send(()).ok();
+    #[tokio::test]
+    async fn test_import_bao_persistence_two_stages() -> TestResult<()> {
+        tracing_subscriber::fmt::try_init().ok();
+        let testdir = tempfile::tempdir()?;
+        let sizes = INTERESTING_SIZES;
+        let db_dir = testdir.path().join("db");
+        let just_size = ChunkRanges::last_chunk();
+        // stage 1, import just the last full chunk group to get a validated size
+        {
+            let store = FsStore::load(&db_dir).await?;
+            for size in sizes {
+                let data = test_data(size);
+                let (hash, ranges, encoded) = create_n0_bao_full(&data, &just_size)?;
+                let data = Bytes::from(encoded);
+                if let Err(cause) = store.import_bao_bytes(hash, ranges, data).await {
+                    panic!("failed to import size={size}: {cause}");
                 }
-                break;
             }
-            match msg.category() {
-                MessageCategory::TopLevel => {
-                    self.state.handle_toplevel(&self.db, msg)?;
-                }
-                MessageCategory::ReadOnly => {
-                    msgs.push_back(msg).expect("just recv'd");
-                    tracing::debug!("starting read transaction");
-                    let txn = self.db.begin_read()?;
-                    let tables = ReadOnlyTables::new(&txn)?;
-                    let count = self.state.options.batch.max_read_batch;
-                    let timeout = tokio::time::sleep(self.state.options.batch.max_read_duration);
-                    tokio::pin!(timeout);
-                    for _ in 0..count {
-                        tokio::select! {
-                            msg = msgs.recv() => {
-                                if let Some(msg) = msg {
-                                    if let Err(msg) = self.state.handle_readonly(&tables, msg)? {
-                                        msgs.push_back(msg).expect("just recv'd");
-                                        break;
-                                    }
-                                } else {
-                                    break;
-                                }
-                            }
-                            _ = &mut timeout => {
-                                tracing::debug!("read transaction timed out");
-                                break;
-                            }
-                        }
-                    }
-                    tracing::debug!("done with read transaction");
+            store.dump().await?;
+            store.shutdown().await?;
+        }
+        dump_dir_full(testdir.path())?;
+        // stage 2, import the rest
+        {
+            let store = FsStore::load(&db_dir).await?;
+            for size in sizes {
+                let remaining = ChunkRanges::all() - round_up_request(size as u64, &just_size);
+                if remaining.is_empty() {
+                    continue;
                 }
-                MessageCategory::ReadWrite => {
-                    msgs.push_back(msg).expect("just recv'd");
-                    tracing::debug!("starting write transaction");
-                    let txn = self.db.begin_write()?;
-                    let mut delete_after_commit = Default::default();
-                    let mut tables = Tables::new(&txn, &mut delete_after_commit)?;
-                    let count = self.state.options.batch.max_write_batch;
-                    let timeout = tokio::time::sleep(self.state.options.batch.max_write_duration);
-                    tokio::pin!(timeout);
-                    for _ in 0..count {
-                        tokio::select! {
-                            msg = msgs.recv() => {
-                                if let Some(msg) = msg {
-                                    if let Err(msg) = self.state.handle_readwrite(&mut tables, msg)? {
-                                        msgs.push_back(msg).expect("just recv'd");
-                                        break;
-                                    }
-                                } else {
-                                    break;
-                                }
-                            }
-                            _ = &mut timeout => {
-                                tracing::debug!("write transaction timed out");
-                                break;
-                            }
-                        }
-                    }
-                    drop(tables);
-                    txn.commit()?;
-                    delete_after_commit.apply_and_clear(&self.state.options.path);
-                    tracing::debug!("write transaction committed");
+                let data = test_data(size);
+                let (hash, ranges, encoded) = create_n0_bao_full(&data, &remaining)?;
+                let data = Bytes::from(encoded);
+                if let Err(cause) = store.import_bao_bytes(hash, ranges, data).await {
+                    panic!("failed to import size={size}: {cause}");
                 }
             }
+            store.dump().await?;
+            store.shutdown().await?;
         }
-        tracing::debug!("redb actor done");
+        // check if the data is complete
+        {
+            let store = FsStore::load(&db_dir).await?;
+            store.dump().await?;
+            for size in sizes {
+                let data = test_data(size);
+                let (hash, ranges, expected) = create_n0_bao_full(&data, &ChunkRanges::all())?;
+                let actual = match store.export_bao(hash, ranges).bao_to_vec().await {
+                    Ok(actual) => actual,
+                    Err(cause) => panic!("failed to export size={size}: {cause}"),
+                };
+                assert_eq!(&expected, &actual);
+            }
+            store.dump().await?;
+            store.shutdown().await?;
+        }
+        dump_dir_full(testdir.path())?;
         Ok(())
     }
-}
-
-impl ActorState {
-    fn entry_status(
-        &mut self,
-        tables: &impl ReadableTables,
-        hash: Hash,
-    ) -> ActorResult<EntryStatus> {
-        let status = match tables.blobs().get(hash)? {
-            Some(guard) => match guard.value() {
-                EntryState::Complete { .. } => EntryStatus::Complete,
-                EntryState::Partial { .. } => EntryStatus::Partial,
-            },
-            None => EntryStatus::NotFound,
-        };
-        Ok(status)
-    }
 
-    fn get(
-        &mut self,
-        tables: &impl ReadableTables,
-        hash: Hash,
-    ) -> ActorResult<Option<BaoFileHandle>> {
-        if let Some(handle) = self.handles.get(&hash).and_then(|weak| weak.upgrade()) {
-            return Ok(Some(handle));
-        }
-        let Some(entry) = tables.blobs().get(hash)? else {
-            return Ok(None);
-        };
-        // todo: if complete, load inline data and/or outboard into memory if needed,
-        // and return a complete entry.
-        let entry = entry.value();
-        let config = self.create_options.clone();
-        let handle = match entry {
-            EntryState::Complete {
-                data_location,
-                outboard_location,
-            } => {
-                let data = load_data(tables, &self.options.path, data_location, &hash)?;
-                let outboard = load_outboard(
-                    tables,
-                    &self.options.path,
-                    outboard_location,
-                    data.size(),
-                    &hash,
-                )?;
-                BaoFileHandle::new_complete(config, hash, data, outboard)
-            }
-            EntryState::Partial { .. } => BaoFileHandle::incomplete_file(config, hash)?,
-        };
-        self.handles.insert(hash, handle.downgrade());
-        Ok(Some(handle))
-    }
-
-    fn export(
-        &mut self,
-        tables: &mut Tables,
-        cmd: Export,
-        tx: oneshot::Sender<ActorResult<()>>,
-    ) -> ActorResult<()> {
-        let Export {
-            temp_tag,
-            target,
-            mode,
-            progress,
-        } = cmd;
-        let guard = tables
-            .blobs
-            .get(temp_tag.hash())?
-            .ok_or_else(|| ActorError::Inconsistent("entry not found".to_owned()))?;
-        let entry = guard.value();
-        match entry {
-            EntryState::Complete {
-                data_location,
-                outboard_location,
-            } => match data_location {
-                DataLocation::Inline(()) => {
-                    // ignore export mode, just copy. For inline data we can not reference anyway.
-                    let data = tables.inline_data.get(temp_tag.hash())?.ok_or_else(|| {
-                        ActorError::Inconsistent("inline data not found".to_owned())
-                    })?;
-                    tracing::trace!("exporting inline data to {}", target.display());
-                    tx.send(std::fs::write(&target, data.value()).map_err(|e| e.into()))
-                        .ok();
+    fn just_size() -> ChunkRanges {
+        ChunkRanges::last_chunk()
+    }
+
+    #[tokio::test]
+    async fn test_import_bao_persistence_observe() -> TestResult<()> {
+        tracing_subscriber::fmt::try_init().ok();
+        let testdir = tempfile::tempdir()?;
+        let sizes = INTERESTING_SIZES;
+        let db_dir = testdir.path().join("db");
+        let just_size = just_size();
+        // stage 1, import just the last full chunk group to get a validated size
+        {
+            let store = FsStore::load(&db_dir).await?;
+            for size in sizes {
+                let data = test_data(size);
+                let (hash, ranges, encoded) = create_n0_bao_full(&data, &just_size)?;
+                let data = Bytes::from(encoded);
+                if let Err(cause) = store.import_bao_bytes(hash, ranges, data).await {
+                    panic!("failed to import size={size}: {cause}");
                 }
-                DataLocation::Owned(size) => {
-                    let path = self.options.path.owned_data_path(temp_tag.hash());
-                    match mode {
-                        ExportMode::Copy => {
-                            // copy in an external thread
-                            self.rt.spawn_blocking(move || {
-                                tx.send(export_file_copy(temp_tag, path, size, target, progress))
-                                    .ok();
-                            });
-                        }
-                        ExportMode::TryReference => match std::fs::rename(&path, &target) {
-                            Ok(()) => {
-                                let entry = EntryState::Complete {
-                                    data_location: DataLocation::External(vec![target], size),
-                                    outboard_location,
-                                };
-                                drop(guard);
-                                tables.blobs.insert(temp_tag.hash(), entry)?;
-                                drop(temp_tag);
-                                tx.send(Ok(())).ok();
-                            }
-                            Err(e) => {
-                                const ERR_CROSS: i32 = 18;
-                                if e.raw_os_error() == Some(ERR_CROSS) {
-                                    // Cross device renaming failed, copy instead
-                                    match std::fs::copy(&path, &target) {
-                                        Ok(_) => {
-                                            let entry = EntryState::Complete {
-                                                data_location: DataLocation::External(
-                                                    vec![target],
-                                                    size,
-                                                ),
-                                                outboard_location,
-                                            };
-
-                                            drop(guard);
-                                            tables.blobs.insert(temp_tag.hash(), entry)?;
-                                            tables
-                                                .delete_after_commit
-                                                .insert(*temp_tag.hash(), [BaoFilePart::Data]);
-                                            drop(temp_tag);
-
-                                            tx.send(Ok(())).ok();
-                                        }
-                                        Err(e) => {
-                                            drop(temp_tag);
-                                            tx.send(Err(e.into())).ok();
-                                        }
-                                    }
-                                } else {
-                                    drop(temp_tag);
-                                    tx.send(Err(e.into())).ok();
-                                }
-                            }
-                        },
-                    }
-                }
-                DataLocation::External(paths, size) => {
-                    let path = paths
-                        .first()
-                        .ok_or_else(|| {
-                            ActorError::Inconsistent("external path missing".to_owned())
-                        })?
-                        .to_owned();
-                    // we can not reference external files, so we just copy them. But this does not have to happen in the actor.
-                    if path == target {
-                        // export to the same path, nothing to do
-                        tx.send(Ok(())).ok();
-                    } else {
-                        // copy in an external thread
-                        self.rt.spawn_blocking(move || {
-                            tx.send(export_file_copy(temp_tag, path, size, target, progress))
-                                .ok();
-                        });
-                    }
-                }
-            },
-            EntryState::Partial { .. } => {
-                return Err(io::Error::new(io::ErrorKind::Unsupported, "partial entry").into());
             }
+            store.dump().await?;
+            store.shutdown().await?;
+        }
+        dump_dir_full(testdir.path())?;
+        // stage 2, import the rest
+        {
+            let store = FsStore::load(&db_dir).await?;
+            for size in sizes {
+                let expected_ranges = round_up_request(size as u64, &just_size);
+                let data = test_data(size);
+                let hash = Hash::new(&data);
+                let bitfield = store.observe(hash).await?;
+                assert_eq!(bitfield.ranges, expected_ranges);
+            }
+            store.dump().await?;
+            store.shutdown().await?;
         }
         Ok(())
     }
 
-    fn import(&mut self, tables: &mut Tables, cmd: Import) -> ActorResult<(TempTag, u64)> {
-        let Import {
-            content_id,
-            source: file,
-            outboard,
-            data_size,
-        } = cmd;
-        let outboard_size = outboard.as_ref().map(|x| x.len() as u64).unwrap_or(0);
-        let inline_data = data_size <= self.options.inline.max_data_inlined;
-        let inline_outboard =
-            outboard_size <= self.options.inline.max_outboard_inlined && outboard_size != 0;
-        // from here on, everything related to the hash is protected by the temp tag
-        let tag = self.temp.temp_tag(content_id);
-        let hash = *tag.hash();
-        self.protected.insert(hash);
-        // move the data file into place, or create a reference to it
-        let data_location = match file {
-            ImportSource::External(external_path) => {
-                tracing::debug!("stored external reference {}", external_path.display());
-                if inline_data {
-                    tracing::debug!(
-                        "reading external data to inline it: {}",
-                        external_path.display()
-                    );
-                    let data = Bytes::from(std::fs::read(&external_path)?);
-                    DataLocation::Inline(data)
-                } else {
-                    DataLocation::External(vec![external_path], data_size)
-                }
-            }
-            ImportSource::TempFile(temp_data_path) => {
-                if inline_data {
-                    tracing::debug!(
-                        "reading and deleting temp file to inline it: {}",
-                        temp_data_path.display()
-                    );
-                    let data = Bytes::from(read_and_remove(&temp_data_path)?);
-                    DataLocation::Inline(data)
-                } else {
-                    let data_path = self.options.path.owned_data_path(&hash);
-                    std::fs::rename(&temp_data_path, &data_path)?;
-                    tracing::debug!("created file {}", data_path.display());
-                    DataLocation::Owned(data_size)
+    #[tokio::test]
+    async fn test_import_bao_persistence_recover() -> TestResult<()> {
+        tracing_subscriber::fmt::try_init().ok();
+        let testdir = tempfile::tempdir()?;
+        let sizes = INTERESTING_SIZES;
+        let db_dir = testdir.path().join("db");
+        let options = Options::new(&db_dir);
+        let just_size = just_size();
+        // stage 1, import just the last full chunk group to get a validated size
+        {
+            let store = FsStore::load_with_opts(db_dir.join("blobs.db"), options.clone()).await?;
+            for size in sizes {
+                let data = test_data(size);
+                let (hash, ranges, encoded) = create_n0_bao_full(&data, &just_size)?;
+                let data = Bytes::from(encoded);
+                if let Err(cause) = store.import_bao_bytes(hash, ranges, data).await {
+                    panic!("failed to import size={size}: {cause}");
                 }
             }
-            ImportSource::Memory(data) => {
-                if inline_data {
-                    DataLocation::Inline(data)
-                } else {
-                    let data_path = self.options.path.owned_data_path(&hash);
-                    overwrite_and_sync(&data_path, &data)?;
-                    tracing::debug!("created file {}", data_path.display());
-                    DataLocation::Owned(data_size)
-                }
-            }
-        };
-        let outboard_location = if let Some(outboard) = outboard {
-            if inline_outboard {
-                OutboardLocation::Inline(Bytes::from(outboard))
-            } else {
-                let outboard_path = self.options.path.owned_outboard_path(&hash);
-                // todo: this blocks the actor when writing a large outboard
-                overwrite_and_sync(&outboard_path, &outboard)?;
-                OutboardLocation::Owned
-            }
-        } else {
-            OutboardLocation::NotNeeded
-        };
-        if let DataLocation::Inline(data) = &data_location {
-            tables.inline_data.insert(hash, data.as_ref())?;
-        }
-        if let OutboardLocation::Inline(outboard) = &outboard_location {
-            tables.inline_outboard.insert(hash, outboard.as_ref())?;
+            store.dump().await?;
+            store.shutdown().await?;
         }
-        if let DataLocation::Owned(_) = &data_location {
-            tables.delete_after_commit.remove(hash, [BaoFilePart::Data]);
-        }
-        if let OutboardLocation::Owned = &outboard_location {
-            tables
-                .delete_after_commit
-                .remove(hash, [BaoFilePart::Outboard]);
-        }
-        let entry = tables.blobs.get(hash)?;
-        let entry = entry.map(|x| x.value()).unwrap_or_default();
-        let data_location = data_location.discard_inline_data();
-        let outboard_location = outboard_location.discard_extra_data();
-        let entry = entry.union(EntryState::Complete {
-            data_location,
-            outboard_location,
-        })?;
-        tables.blobs.insert(hash, entry)?;
-        Ok((tag, data_size))
-    }
-
-    fn get_or_create(
-        &mut self,
-        tables: &impl ReadableTables,
-        hash: Hash,
-    ) -> ActorResult<BaoFileHandle> {
-        self.protected.insert(hash);
-        if let Some(handle) = self.handles.get(&hash).and_then(|x| x.upgrade()) {
-            return Ok(handle);
-        }
-        let entry = tables.blobs().get(hash)?;
-        let handle = if let Some(entry) = entry {
-            let entry = entry.value();
-            match entry {
-                EntryState::Complete {
-                    data_location,
-                    outboard_location,
-                    ..
-                } => {
-                    let data = load_data(tables, &self.options.path, data_location, &hash)?;
-                    let outboard = load_outboard(
-                        tables,
-                        &self.options.path,
-                        outboard_location,
-                        data.size(),
-                        &hash,
-                    )?;
-                    tracing::debug!("creating complete entry for {}", hash.to_hex());
-                    BaoFileHandle::new_complete(self.create_options.clone(), hash, data, outboard)
-                }
-                EntryState::Partial { .. } => {
-                    tracing::debug!("creating partial entry for {}", hash.to_hex());
-                    BaoFileHandle::incomplete_file(self.create_options.clone(), hash)?
-                }
-            }
-        } else {
-            BaoFileHandle::incomplete_mem(self.create_options.clone(), hash)
-        };
-        self.handles.insert(hash, handle.downgrade());
-        Ok(handle)
-    }
-
-    /// Read the entire blobs table. Callers can then sift through the results to find what they need
-    fn blobs(
-        &mut self,
-        tables: &impl ReadableTables,
-        filter: FilterPredicate<Hash, EntryState>,
-    ) -> ActorResult<Vec<std::result::Result<(Hash, EntryState), StorageError>>> {
-        let mut res = Vec::new();
-        let mut index = 0u64;
-        #[allow(clippy::explicit_counter_loop)]
-        for item in tables.blobs().iter()? {
-            match item {
-                Ok((k, v)) => {
-                    if let Some(item) = filter(index, k, v) {
-                        res.push(Ok(item));
-                    }
-                }
-                Err(e) => {
-                    res.push(Err(e));
-                }
-            }
-            index += 1;
+        delete_rec(testdir.path(), "bitfield")?;
+        dump_dir_full(testdir.path())?;
+        // stage 2, import the rest
+        {
+            let store = FsStore::load_with_opts(db_dir.join("blobs.db"), options.clone()).await?;
+            for size in sizes {
+                let expected_ranges = round_up_request(size as u64, &just_size);
+                let data = test_data(size);
+                let hash = Hash::new(&data);
+                let bitfield = store.observe(hash).await?;
+                assert_eq!(bitfield.ranges, expected_ranges, "size={size}");
+            }
+            store.dump().await?;
+            store.shutdown().await?;
         }
-        Ok(res)
+        Ok(())
     }
 
-    /// Read the entire tags table. Callers can then sift through the results to find what they need
-    fn tags(
-        &mut self,
-        tables: &impl ReadableTables,
-        from: Option<Tag>,
-        to: Option<Tag>,
-    ) -> ActorResult<Vec<std::result::Result<(Tag, HashAndFormat), StorageError>>> {
-        let mut res = Vec::new();
-        let from = from.map(Bound::Included).unwrap_or(Bound::Unbounded);
-        let to = to.map(Bound::Excluded).unwrap_or(Bound::Unbounded);
-        for item in tables.tags().range((from, to))? {
-            match item {
-                Ok((k, v)) => {
-                    res.push(Ok((k.value(), v.value())));
-                }
-                Err(e) => {
-                    res.push(Err(e));
-                }
+    #[tokio::test]
+    async fn test_import_bytes_persistence_full() -> TestResult<()> {
+        tracing_subscriber::fmt::try_init().ok();
+        let testdir = tempfile::tempdir()?;
+        let sizes = INTERESTING_SIZES;
+        let db_dir = testdir.path().join("db");
+        {
+            let store = FsStore::load(&db_dir).await?;
+            let mut tts = Vec::new();
+            for size in sizes {
+                let data = test_data(size);
+                let data = data;
+                tts.push(store.add_bytes(data.clone()).await?);
+            }
+            store.dump().await?;
+            store.shutdown().await?;
+        }
+        {
+            let store = FsStore::load(&db_dir).await?;
+            store.dump().await?;
+            for size in sizes {
+                let expected = test_data(size);
+                let hash = Hash::new(&expected);
+                let Ok(actual) = store
+                    .export_bao(hash, ChunkRanges::all())
+                    .data_to_vec()
+                    .await
+                else {
+                    panic!("failed to export size={size}");
+                };
+                assert_eq!(&expected, &actual, "size={size}");
             }
+            store.shutdown().await?;
         }
-        Ok(res)
-    }
-
-    fn create_tag(&mut self, tables: &mut Tables, content: HashAndFormat) -> ActorResult<Tag> {
-        let tag = {
-            let tag = Tag::auto(SystemTime::now(), |x| {
-                matches!(tables.tags.get(Tag(Bytes::copy_from_slice(x))), Ok(Some(_)))
-            });
-            tables.tags.insert(tag.clone(), content)?;
-            tag
-        };
-        Ok(tag)
-    }
-
-    fn rename_tag(&mut self, tables: &mut Tables, from: Tag, to: Tag) -> ActorResult<()> {
-        let value = tables
-            .tags
-            .remove(from)?
-            .ok_or_else(|| {
-                ActorError::Io(io::Error::new(io::ErrorKind::NotFound, "tag not found"))
-            })?
-            .value();
-        tables.tags.insert(to, value)?;
         Ok(())
     }
 
-    fn set_tag(&self, tables: &mut Tables, tag: Tag, value: HashAndFormat) -> ActorResult<()> {
-        tables.tags.insert(tag, value)?;
+    async fn test_batch(store: &Store) -> TestResult<()> {
+        let batch = store.blobs().batch().await?;
+        let tt1 = batch.temp_tag(Hash::new("foo")).await?;
+        let tt2 = batch.add_slice("boo").await?;
+        let tts = store
+            .tags()
+            .list_temp_tags()
+            .await?
+            .collect::<HashSet<_>>()
+            .await;
+        assert!(tts.contains(tt1.hash_and_format()));
+        assert!(tts.contains(tt2.hash_and_format()));
+        drop(batch);
+        store.sync_db().await?;
+        let tts = store
+            .tags()
+            .list_temp_tags()
+            .await?
+            .collect::<HashSet<_>>()
+            .await;
+        // temp tag went out of scope, so it does not work anymore
+        assert!(!tts.contains(tt1.hash_and_format()));
+        assert!(!tts.contains(tt2.hash_and_format()));
+        drop(tt1);
+        drop(tt2);
         Ok(())
     }
 
-    fn delete_tags(
-        &self,
-        tables: &mut Tables,
-        from: Option<Tag>,
-        to: Option<Tag>,
-    ) -> ActorResult<()> {
-        let from = from.map(Bound::Included).unwrap_or(Bound::Unbounded);
-        let to = to.map(Bound::Excluded).unwrap_or(Bound::Unbounded);
-        let removing = tables.tags.extract_from_if((from, to), |_, _| true)?;
-        // drain the iterator to actually remove the tags
-        for res in removing {
-            res?;
+    #[tokio::test]
+    async fn test_batch_fs() -> TestResult<()> {
+        tracing_subscriber::fmt::try_init().ok();
+        let testdir = tempfile::tempdir()?;
+        let db_dir = testdir.path().join("db");
+        let store = FsStore::load(db_dir).await?;
+        test_batch(&store).await
+    }
+
+    #[tokio::test]
+    async fn smoke() -> TestResult<()> {
+        tracing_subscriber::fmt::try_init().ok();
+        let testdir = tempfile::tempdir()?;
+        let db_dir = testdir.path().join("db");
+        let store = FsStore::load(db_dir).await?;
+        let haf = HashAndFormat::raw(Hash::from([0u8; 32]));
+        store.tags().set(Tag::from("test"), haf).await?;
+        store.tags().set(Tag::from("boo"), haf).await?;
+        store.tags().set(Tag::from("bar"), haf).await?;
+        let sizes = INTERESTING_SIZES;
+        let mut hashes = Vec::new();
+        let mut data_by_hash = HashMap::new();
+        let mut bao_by_hash = HashMap::new();
+        for size in sizes {
+            let data = vec![0u8; size];
+            let data = Bytes::from(data);
+            let tt = store.add_bytes(data.clone()).temp_tag().await?;
+            data_by_hash.insert(*tt.hash(), data);
+            hashes.push(tt);
         }
-        Ok(())
-    }
+        store.sync_db().await?;
+        for tt in &hashes {
+            let hash = *tt.hash();
+            let path = testdir.path().join(format!("{hash}.txt"));
+            store.export(hash, path).await?;
+        }
+        for tt in &hashes {
+            let hash = tt.hash();
+            let data = store
+                .export_bao(*hash, ChunkRanges::all())
+                .data_to_vec()
+                .await
+                .unwrap();
+            assert_eq!(data, data_by_hash[hash].to_vec());
+            let bao = store
+                .export_bao(*hash, ChunkRanges::all())
+                .bao_to_vec()
+                .await
+                .unwrap();
+            bao_by_hash.insert(*hash, bao);
+        }
+        store.dump().await?;
 
-    fn on_mem_size_exceeded(&mut self, tables: &mut Tables, hash: Hash) -> ActorResult<()> {
-        let entry = tables
-            .blobs
-            .get(hash)?
-            .map(|x| x.value())
-            .unwrap_or_default();
-        let entry = entry.union(EntryState::Partial { size: None })?;
-        tables.blobs.insert(hash, entry)?;
-        // protect all three parts of the entry
-        tables.delete_after_commit.remove(
-            hash,
-            [BaoFilePart::Data, BaoFilePart::Outboard, BaoFilePart::Sizes],
-        );
+        for size in sizes {
+            let data = test_data(size);
+            let ranges = ChunkRanges::all();
+            let (hash, bao) = create_n0_bao(&data, &ranges)?;
+            store.import_bao_bytes(hash, ranges, bao).await?;
+        }
+
+        for (_hash, _bao_tree) in bao_by_hash {
+            // let mut reader = Cursor::new(bao_tree);
+            // let size = reader.read_u64_le().await?;
+            // let tree = BaoTree::new(size, IROH_BLOCK_SIZE);
+            // let ranges = ChunkRanges::all();
+            // let mut decoder = DecodeResponseIter::new(hash, tree, reader, &ranges);
+            // while let Some(item) = decoder.next() {
+            //     let item = item?;
+            // }
+            // store.import_bao_bytes(hash, ChunkRanges::all(), bao_tree.into()).await?;
+        }
         Ok(())
     }
 
-    fn update_inline_options(
-        &mut self,
-        db: &redb::Database,
-        options: InlineOptions,
-        reapply: bool,
-    ) -> ActorResult<()> {
-        self.options.inline = options;
-        if reapply {
-            let mut delete_after_commit = Default::default();
-            let tx = db.begin_write()?;
-            {
-                let mut tables = Tables::new(&tx, &mut delete_after_commit)?;
-                let hashes = tables
-                    .blobs
-                    .iter()?
-                    .map(|x| x.map(|(k, _)| k.value()))
-                    .collect::<Result<Vec<_>, _>>()?;
-                for hash in hashes {
-                    let guard = tables
-                        .blobs
-                        .get(hash)?
-                        .ok_or_else(|| ActorError::Inconsistent("hash not found".to_owned()))?;
-                    let entry = guard.value();
-                    if let EntryState::Complete {
-                        data_location,
-                        outboard_location,
-                    } = entry
-                    {
-                        let (data_location, data_size, data_location_changed) = match data_location
-                        {
-                            DataLocation::Owned(size) => {
-                                // inline
-                                if size <= self.options.inline.max_data_inlined {
-                                    let path = self.options.path.owned_data_path(&hash);
-                                    let data = std::fs::read(&path)?;
-                                    tables.delete_after_commit.insert(hash, [BaoFilePart::Data]);
-                                    tables.inline_data.insert(hash, data.as_slice())?;
-                                    (DataLocation::Inline(()), size, true)
-                                } else {
-                                    (DataLocation::Owned(size), size, false)
-                                }
-                            }
-                            DataLocation::Inline(()) => {
-                                let guard = tables.inline_data.get(hash)?.ok_or_else(|| {
-                                    ActorError::Inconsistent("inline data missing".to_owned())
-                                })?;
-                                let data = guard.value();
-                                let size = data.len() as u64;
-                                if size > self.options.inline.max_data_inlined {
-                                    let path = self.options.path.owned_data_path(&hash);
-                                    std::fs::write(&path, data)?;
-                                    drop(guard);
-                                    tables.inline_data.remove(hash)?;
-                                    (DataLocation::Owned(size), size, true)
-                                } else {
-                                    (DataLocation::Inline(()), size, false)
-                                }
-                            }
-                            DataLocation::External(paths, size) => {
-                                (DataLocation::External(paths, size), size, false)
-                            }
-                        };
-                        let outboard_size = raw_outboard_size(data_size);
-                        let (outboard_location, outboard_location_changed) = match outboard_location
-                        {
-                            OutboardLocation::Owned
-                                if outboard_size <= self.options.inline.max_outboard_inlined =>
-                            {
-                                let path = self.options.path.owned_outboard_path(&hash);
-                                let outboard = std::fs::read(&path)?;
-                                tables
-                                    .delete_after_commit
-                                    .insert(hash, [BaoFilePart::Outboard]);
-                                tables.inline_outboard.insert(hash, outboard.as_slice())?;
-                                (OutboardLocation::Inline(()), true)
-                            }
-                            OutboardLocation::Inline(())
-                                if outboard_size > self.options.inline.max_outboard_inlined =>
-                            {
-                                let guard = tables.inline_outboard.get(hash)?.ok_or_else(|| {
-                                    ActorError::Inconsistent("inline outboard missing".to_owned())
-                                })?;
-                                let outboard = guard.value();
-                                let path = self.options.path.owned_outboard_path(&hash);
-                                std::fs::write(&path, outboard)?;
-                                drop(guard);
-                                tables.inline_outboard.remove(hash)?;
-                                (OutboardLocation::Owned, true)
-                            }
-                            x => (x, false),
-                        };
-                        drop(guard);
-                        if data_location_changed || outboard_location_changed {
-                            tables.blobs.insert(
-                                hash,
-                                EntryState::Complete {
-                                    data_location,
-                                    outboard_location,
-                                },
-                            )?;
-                        }
+    pub fn delete_rec(root_dir: impl AsRef<Path>, extension: &str) -> Result<(), std::io::Error> {
+        // Remove leading dot if present, so we have just the extension
+        let ext = extension.trim_start_matches('.').to_lowercase();
+
+        for entry in WalkDir::new(root_dir).into_iter().filter_map(|e| e.ok()) {
+            let path = entry.path();
+
+            if path.is_file() {
+                if let Some(file_ext) = path.extension() {
+                    if file_ext.to_string_lossy().to_lowercase() == ext {
+                        println!("Deleting: {}", path.display());
+                        fs::remove_file(path)?;
                     }
                 }
             }
-            tx.commit()?;
-            delete_after_commit.apply_and_clear(&self.options.path);
         }
+
         Ok(())
     }
 
-    fn delete(&mut self, tables: &mut Tables, hashes: Vec<Hash>, force: bool) -> ActorResult<()> {
-        for hash in hashes {
-            if self.temp.as_ref().read().unwrap().contains(&hash) {
-                continue;
-            }
-            if !force && self.protected.contains(&hash) {
-                tracing::debug!("protected hash, continuing {}", &hash.to_hex()[..8]);
-                continue;
-            }
+    pub fn dump_dir(path: impl AsRef<Path>) -> io::Result<()> {
+        let mut entries: Vec<_> = WalkDir::new(&path)
+            .into_iter()
+            .filter_map(Result::ok) // Skip errors
+            .collect();
 
-            tracing::debug!("deleting {}", &hash.to_hex()[..8]);
+        // Sort by path (name at each depth)
+        entries.sort_by(|a, b| a.path().cmp(b.path()));
 
-            self.handles.remove(&hash);
-            if let Some(entry) = tables.blobs.remove(hash)? {
-                match entry.value() {
-                    EntryState::Complete {
-                        data_location,
-                        outboard_location,
-                    } => {
-                        match data_location {
-                            DataLocation::Inline(_) => {
-                                tables.inline_data.remove(hash)?;
-                            }
-                            DataLocation::Owned(_) => {
-                                // mark the data for deletion
-                                tables.delete_after_commit.insert(hash, [BaoFilePart::Data]);
-                            }
-                            DataLocation::External(_, _) => {}
-                        }
-                        match outboard_location {
-                            OutboardLocation::Inline(_) => {
-                                tables.inline_outboard.remove(hash)?;
-                            }
-                            OutboardLocation::Owned => {
-                                // mark the outboard for deletion
-                                tables
-                                    .delete_after_commit
-                                    .insert(hash, [BaoFilePart::Outboard]);
-                            }
-                            OutboardLocation::NotNeeded => {}
-                        }
-                    }
-                    EntryState::Partial { .. } => {
-                        // mark all parts for deletion
-                        tables.delete_after_commit.insert(
-                            hash,
-                            [BaoFilePart::Outboard, BaoFilePart::Data, BaoFilePart::Sizes],
-                        );
-                    }
-                }
+        for entry in entries {
+            let depth = entry.depth();
+            let indent = "  ".repeat(depth); // Two spaces per level
+            let name = entry.file_name().to_string_lossy();
+            let size = entry.metadata()?.len(); // Size in bytes
+
+            if entry.file_type().is_file() {
+                println!("{indent}{name} ({size} bytes)");
+            } else if entry.file_type().is_dir() {
+                println!("{indent}{name}/");
             }
         }
         Ok(())
     }
 
-    fn on_complete(&mut self, tables: &mut Tables, entry: BaoFileHandle) -> ActorResult<()> {
-        let hash = entry.hash();
-        let mut info = None;
-        tracing::trace!("on_complete({})", hash.to_hex());
-        entry.transform(|state| {
-            tracing::trace!("on_complete transform {:?}", state);
-            let entry = match complete_storage(
-                state,
-                &hash,
-                &self.options.path,
-                &self.options.inline,
-                tables.delete_after_commit,
-            )? {
-                Ok(entry) => {
-                    // store the info so we can insert it into the db later
-                    info = Some((
-                        entry.data_size(),
-                        entry.data.mem().cloned(),
-                        entry.outboard_size(),
-                        entry.outboard.mem().cloned(),
-                    ));
-                    entry
-                }
-                Err(entry) => {
-                    // the entry was already complete, nothing to do
-                    entry
-                }
-            };
-            Ok(BaoFileStorage::Complete(entry))
-        })?;
-        if let Some((data_size, data, outboard_size, outboard)) = info {
-            let data_location = if data.is_some() {
-                DataLocation::Inline(())
-            } else {
-                DataLocation::Owned(data_size)
-            };
-            let outboard_location = if outboard_size == 0 {
-                OutboardLocation::NotNeeded
-            } else if outboard.is_some() {
-                OutboardLocation::Inline(())
-            } else {
-                OutboardLocation::Owned
-            };
-            {
-                tracing::debug!(
-                    "inserting complete entry for {}, {} bytes",
-                    hash.to_hex(),
-                    data_size,
-                );
-                let entry = tables
-                    .blobs()
-                    .get(hash)?
-                    .map(|x| x.value())
-                    .unwrap_or_default();
-                let entry = entry.union(EntryState::Complete {
-                    data_location,
-                    outboard_location,
-                })?;
-                tables.blobs.insert(hash, entry)?;
-                if let Some(data) = data {
-                    tables.inline_data.insert(hash, data.as_ref())?;
-                }
-                if let Some(outboard) = outboard {
-                    tables.inline_outboard.insert(hash, outboard.as_ref())?;
-                }
+    pub fn dump_dir_full(path: impl AsRef<Path>) -> io::Result<()> {
+        let mut entries: Vec<_> = WalkDir::new(&path)
+            .into_iter()
+            .filter_map(Result::ok) // Skip errors
+            .collect();
+
+        // Sort by path (name at each depth)
+        entries.sort_by(|a, b| a.path().cmp(b.path()));
+
+        for entry in entries {
+            let depth = entry.depth();
+            let indent = "  ".repeat(depth);
+            let name = entry.file_name().to_string_lossy();
+
+            if entry.file_type().is_dir() {
+                println!("{indent}{name}/");
+            } else if entry.file_type().is_file() {
+                let size = entry.metadata()?.len();
+                println!("{indent}{name} ({size} bytes)");
+
+                // Dump depending on file type
+                let path = entry.path();
+                if name.ends_with(".data") {
+                    print!("{indent}  ");
+                    dump_file(path, 1024 * 16)?;
+                } else if name.ends_with(".obao4") {
+                    print!("{indent}  ");
+                    dump_file(path, 64)?;
+                } else if name.ends_with(".sizes4") {
+                    print!("{indent}  ");
+                    dump_file(path, 8)?;
+                } else if name.ends_with(".bitfield") {
+                    match read_checksummed::<Bitfield>(path) {
+                        Ok(bitfield) => {
+                            println!("{indent}  bitfield: {bitfield:?}");
+                        }
+                        Err(cause) => {
+                            println!("{indent}  bitfield: error: {cause}");
+                        }
+                    }
+                } else {
+                    continue; // Skip content dump for other files
+                };
             }
         }
         Ok(())
     }
 
-    fn handle_toplevel(&mut self, db: &redb::Database, msg: ActorMessage) -> ActorResult<()> {
-        match msg {
-            ActorMessage::UpdateInlineOptions {
-                inline_options,
-                reapply,
-                tx,
-            } => {
-                let res = self.update_inline_options(db, inline_options, reapply);
-                tx.send(res?).ok();
-            }
-            ActorMessage::Fsck {
-                repair,
-                progress,
-                tx,
-            } => {
-                let res = self.consistency_check(db, repair, progress);
-                tx.send(res).ok();
-            }
-            ActorMessage::Sync { tx } => {
-                tx.send(()).ok();
-            }
-            x => {
-                return Err(ActorError::Inconsistent(format!(
-                    "unexpected message for handle_toplevel: {:?}",
-                    x
-                )))
-            }
-        }
+    pub fn dump_file<P: AsRef<Path>>(path: P, chunk_size: u64) -> io::Result<()> {
+        let bits = file_bits(path, chunk_size)?;
+        println!("{}", print_bitfield_ansi(bits));
         Ok(())
     }
 
-    fn handle_readonly(
-        &mut self,
-        tables: &impl ReadableTables,
-        msg: ActorMessage,
-    ) -> ActorResult<std::result::Result<(), ActorMessage>> {
-        match msg {
-            ActorMessage::Get { hash, tx } => {
-                let res = self.get(tables, hash);
-                tx.send(res).ok();
-            }
-            ActorMessage::GetOrCreate { hash, tx } => {
-                let res = self.get_or_create(tables, hash);
-                tx.send(res).ok();
-            }
-            ActorMessage::EntryStatus { hash, tx } => {
-                let res = self.entry_status(tables, hash);
-                tx.send(res).ok();
-            }
-            ActorMessage::Blobs { filter, tx } => {
-                let res = self.blobs(tables, filter);
-                tx.send(res).ok();
-            }
-            ActorMessage::Tags { from, to, tx } => {
-                let res = self.tags(tables, from, to);
-                tx.send(res).ok();
-            }
-            ActorMessage::GcStart { tx } => {
-                self.protected.clear();
-                self.handles.retain(|_, weak| weak.is_live());
-                tx.send(()).ok();
-            }
-            ActorMessage::Dump => {
-                dump(tables).ok();
-            }
-            #[cfg(test)]
-            ActorMessage::EntryState { hash, tx } => {
-                tx.send(self.entry_state(tables, hash)).ok();
-            }
-            ActorMessage::GetFullEntryState { hash, tx } => {
-                let res = self.get_full_entry_state(tables, hash);
-                tx.send(res).ok();
-            }
-            x => return Ok(Err(x)),
+    pub fn file_bits(path: impl AsRef<Path>, chunk_size: u64) -> io::Result<Vec<bool>> {
+        let file = fs::File::open(&path)?;
+        let file_size = file.metadata()?.len();
+        let mut buffer = vec![0u8; chunk_size as usize];
+        let mut bits = Vec::new();
+
+        let mut offset = 0u64;
+        while offset < file_size {
+            let remaining = file_size - offset;
+            let current_chunk_size = chunk_size.min(remaining);
+
+            let chunk = &mut buffer[..current_chunk_size as usize];
+            file.read_exact_at(offset, chunk)?;
+
+            let has_non_zero = chunk.iter().any(|&byte| byte != 0);
+            bits.push(has_non_zero);
+
+            offset += current_chunk_size;
         }
-        Ok(Ok(()))
+
+        Ok(bits)
     }
 
-    fn handle_readwrite(
-        &mut self,
-        tables: &mut Tables,
-        msg: ActorMessage,
-    ) -> ActorResult<std::result::Result<(), ActorMessage>> {
-        match msg {
-            ActorMessage::Import { cmd, tx } => {
-                let res = self.import(tables, cmd);
-                tx.send(res).ok();
-            }
-            ActorMessage::SetTag { tag, value, tx } => {
-                let res = self.set_tag(tables, tag, value);
-                tx.send(res).ok();
-            }
-            ActorMessage::DeleteTags { from, to, tx } => {
-                let res = self.delete_tags(tables, from, to);
-                tx.send(res).ok();
-            }
-            ActorMessage::CreateTag { hash, tx } => {
-                let res = self.create_tag(tables, hash);
-                tx.send(res).ok();
-            }
-            ActorMessage::RenameTag { from, to, tx } => {
-                let res = self.rename_tag(tables, from, to);
-                tx.send(res).ok();
-            }
-            ActorMessage::Delete { hashes, tx } => {
-                let res = self.delete(tables, hashes, true);
-                tx.send(res).ok();
-            }
-            ActorMessage::GcDelete { hashes, tx } => {
-                let res = self.delete(tables, hashes, false);
-                tx.send(res).ok();
-            }
-            ActorMessage::OnComplete { handle } => {
-                let res = self.on_complete(tables, handle);
-                res.ok();
-            }
-            ActorMessage::Export { cmd, tx } => {
-                self.export(tables, cmd, tx)?;
-            }
-            ActorMessage::OnMemSizeExceeded { hash } => {
-                let res = self.on_mem_size_exceeded(tables, hash);
-                res.ok();
-            }
-            ActorMessage::Dump => {
-                let res = dump(tables);
-                res.ok();
-            }
-            ActorMessage::SetFullEntryState { hash, entry, tx } => {
-                let res = self.set_full_entry_state(tables, hash, entry);
-                tx.send(res).ok();
-            }
-            msg => {
-                // try to handle it as readonly
-                if let Err(msg) = self.handle_readonly(tables, msg)? {
-                    return Ok(Err(msg));
-                }
-            }
-        }
-        Ok(Ok(()))
+    #[allow(dead_code)]
+    fn print_bitfield(bits: impl IntoIterator<Item = bool>) -> String {
+        bits.into_iter()
+            .map(|bit| if bit { '#' } else { '_' })
+            .collect()
     }
-}
 
-/// Export a file by copying out its content to a new location
-fn export_file_copy(
-    temp_tag: TempTag,
-    path: PathBuf,
-    size: u64,
-    target: PathBuf,
-    progress: ExportProgressCb,
-) -> ActorResult<()> {
-    progress(0)?;
-    // todo: fine grained copy progress
-    reflink_copy::reflink_or_copy(path, target)?;
-    progress(size)?;
-    drop(temp_tag);
-    Ok(())
-}
+    fn print_bitfield_ansi(bits: impl IntoIterator<Item = bool>) -> String {
+        let mut result = String::new();
+        let mut iter = bits.into_iter();
 
-fn dump(tables: &impl ReadableTables) -> ActorResult<()> {
-    for e in tables.blobs().iter()? {
-        let (k, v) = e?;
-        let k = k.value();
-        let v = v.value();
-        println!("blobs: {} -> {:?}", k.to_hex(), v);
-    }
-    for e in tables.tags().iter()? {
-        let (k, v) = e?;
-        let k = k.value();
-        let v = v.value();
-        println!("tags: {} -> {:?}", k, v);
-    }
-    for e in tables.inline_data().iter()? {
-        let (k, v) = e?;
-        let k = k.value();
-        let v = v.value();
-        println!("inline_data: {} -> {:?}", k.to_hex(), v.len());
-    }
-    for e in tables.inline_outboard().iter()? {
-        let (k, v) = e?;
-        let k = k.value();
-        let v = v.value();
-        println!("inline_outboard: {} -> {:?}", k.to_hex(), v.len());
-    }
-    Ok(())
-}
+        while let Some(b1) = iter.next() {
+            let b2 = iter.next();
 
-fn load_data(
-    tables: &impl ReadableTables,
-    options: &PathOptions,
-    location: DataLocation<(), u64>,
-    hash: &Hash,
-) -> ActorResult<MemOrFile<Bytes, (std::fs::File, u64)>> {
-    Ok(match location {
-        DataLocation::Inline(()) => {
-            let Some(data) = tables.inline_data().get(hash)? else {
-                return Err(ActorError::Inconsistent(format!(
-                    "inconsistent database state: {} should have inline data but does not",
-                    hash.to_hex()
-                )));
-            };
-            MemOrFile::Mem(Bytes::copy_from_slice(data.value()))
-        }
-        DataLocation::Owned(data_size) => {
-            let path = options.owned_data_path(hash);
-            let Ok(file) = std::fs::File::open(&path) else {
-                return Err(io::Error::new(
-                    io::ErrorKind::NotFound,
-                    format!("file not found: {}", path.display()),
-                )
-                .into());
-            };
-            MemOrFile::File((file, data_size))
-        }
-        DataLocation::External(paths, data_size) => {
-            if paths.is_empty() {
-                return Err(ActorError::Inconsistent(
-                    "external data location must not be empty".into(),
-                ));
-            }
-            let path = &paths[0];
-            let Ok(file) = std::fs::File::open(path) else {
-                return Err(io::Error::new(
-                    io::ErrorKind::NotFound,
-                    format!("external file not found: {}", path.display()),
-                )
-                .into());
-            };
-            MemOrFile::File((file, data_size))
-        }
-    })
-}
+            // ANSI color codes
+            let white_fg = "\x1b[97m"; // bright white foreground
+            let reset = "\x1b[0m"; // reset all attributes
+            let gray_bg = "\x1b[100m"; // bright black (gray) background
+            let black_bg = "\x1b[40m"; // black background
 
-fn load_outboard(
-    tables: &impl ReadableTables,
-    options: &PathOptions,
-    location: OutboardLocation,
-    size: u64,
-    hash: &Hash,
-) -> ActorResult<MemOrFile<Bytes, (std::fs::File, u64)>> {
-    Ok(match location {
-        OutboardLocation::NotNeeded => MemOrFile::Mem(Bytes::new()),
-        OutboardLocation::Inline(_) => {
-            let Some(outboard) = tables.inline_outboard().get(hash)? else {
-                return Err(ActorError::Inconsistent(format!(
-                    "inconsistent database state: {} should have inline outboard but does not",
-                    hash.to_hex()
-                )));
+            let colored_char = match (b1, b2) {
+                (true, Some(true)) => format!("{}{}{}", white_fg, '█', reset), // 11 - solid white on default background
+                (true, Some(false)) => format!("{}{}{}{}", gray_bg, white_fg, '▌', reset), // 10 - left half white on gray background
+                (false, Some(true)) => format!("{}{}{}{}", gray_bg, white_fg, '▐', reset), // 01 - right half white on gray background
+                (false, Some(false)) => format!("{}{}{}{}", gray_bg, white_fg, ' ', reset), // 00 - space with gray background
+                (true, None) => format!("{}{}{}{}", black_bg, white_fg, '▌', reset), // 1 (pad 0) - left half white on black background
+                (false, None) => format!("{}{}{}{}", black_bg, white_fg, ' ', reset), // 0 (pad 0) - space with black background
             };
-            MemOrFile::Mem(Bytes::copy_from_slice(outboard.value()))
-        }
-        OutboardLocation::Owned => {
-            let outboard_size = raw_outboard_size(size);
-            let path = options.owned_outboard_path(hash);
-            let Ok(file) = std::fs::File::open(&path) else {
-                return Err(io::Error::new(
-                    io::ErrorKind::NotFound,
-                    format!("file not found: {} size={}", path.display(), outboard_size),
-                )
-                .into());
-            };
-            MemOrFile::File((file, outboard_size))
-        }
-    })
-}
 
-/// Take a possibly incomplete storage and turn it into complete
-fn complete_storage(
-    storage: BaoFileStorage,
-    hash: &Hash,
-    path_options: &PathOptions,
-    inline_options: &InlineOptions,
-    delete_after_commit: &mut DeleteSet,
-) -> ActorResult<std::result::Result<CompleteStorage, CompleteStorage>> {
-    let (data, outboard, _sizes) = match storage {
-        BaoFileStorage::Complete(c) => return Ok(Err(c)),
-        BaoFileStorage::IncompleteMem(storage) => {
-            let (data, outboard, sizes) = storage.into_parts();
-            (
-                MemOrFile::Mem(Bytes::from(data.into_parts().0)),
-                MemOrFile::Mem(Bytes::from(outboard.into_parts().0)),
-                MemOrFile::Mem(Bytes::from(sizes.to_vec())),
-            )
-        }
-        BaoFileStorage::IncompleteFile(storage) => {
-            let (data, outboard, sizes) = storage.into_parts();
-            (
-                MemOrFile::File(data),
-                MemOrFile::File(outboard),
-                MemOrFile::File(sizes),
-            )
-        }
-    };
-    let data_size = data.size()?.unwrap();
-    let outboard_size = outboard.size()?.unwrap();
-    // todo: perform more sanity checks if in debug mode
-    debug_assert!(raw_outboard_size(data_size) == outboard_size);
-    // inline data if needed, or write to file if needed
-    let data = if data_size <= inline_options.max_data_inlined {
-        match data {
-            MemOrFile::File(data) => {
-                let mut buf = vec![0; data_size as usize];
-                data.read_at(0, &mut buf)?;
-                // mark data for deletion after commit
-                delete_after_commit.insert(*hash, [BaoFilePart::Data]);
-                MemOrFile::Mem(Bytes::from(buf))
-            }
-            MemOrFile::Mem(data) => MemOrFile::Mem(data),
-        }
-    } else {
-        // protect the data from previous deletions
-        delete_after_commit.remove(*hash, [BaoFilePart::Data]);
-        match data {
-            MemOrFile::Mem(data) => {
-                let path = path_options.owned_data_path(hash);
-                let file = overwrite_and_sync(&path, &data)?;
-                MemOrFile::File((file, data_size))
-            }
-            MemOrFile::File(data) => MemOrFile::File((data, data_size)),
+            result.push_str(&colored_char);
         }
-    };
-    // inline outboard if needed, or write to file if needed
-    let outboard = if outboard_size == 0 {
-        Default::default()
-    } else if outboard_size <= inline_options.max_outboard_inlined {
-        match outboard {
-            MemOrFile::File(outboard) => {
-                let mut buf = vec![0; outboard_size as usize];
-                outboard.read_at(0, &mut buf)?;
-                drop(outboard);
-                // mark outboard for deletion after commit
-                delete_after_commit.insert(*hash, [BaoFilePart::Outboard]);
-                MemOrFile::Mem(Bytes::from(buf))
-            }
-            MemOrFile::Mem(outboard) => MemOrFile::Mem(outboard),
-        }
-    } else {
-        // protect the outboard from previous deletions
-        delete_after_commit.remove(*hash, [BaoFilePart::Outboard]);
-        match outboard {
-            MemOrFile::Mem(outboard) => {
-                let path = path_options.owned_outboard_path(hash);
-                let file = overwrite_and_sync(&path, &outboard)?;
-                MemOrFile::File((file, outboard_size))
+
+        // Ensure we end with a reset code to prevent color bleeding
+        result.push_str("\x1b[0m");
+        result
+    }
+
+    fn bytes_to_stream(
+        bytes: Bytes,
+        chunk_size: usize,
+    ) -> impl Stream<Item = io::Result<Bytes>> + 'static {
+        assert!(chunk_size > 0, "Chunk size must be greater than 0");
+        stream::unfold((bytes, 0), move |(bytes, offset)| async move {
+            if offset >= bytes.len() {
+                None
+            } else {
+                let chunk_len = chunk_size.min(bytes.len() - offset);
+                let chunk = bytes.slice(offset..offset + chunk_len);
+                Some((Ok(chunk), (bytes, offset + chunk_len)))
             }
-            MemOrFile::File(outboard) => MemOrFile::File((outboard, outboard_size)),
-        }
-    };
-    // mark sizes for deletion after commit in any case - a complete entry
-    // does not need sizes.
-    delete_after_commit.insert(*hash, [BaoFilePart::Sizes]);
-    Ok(Ok(CompleteStorage { data, outboard }))
+        })
+    }
 }
diff --git a/src/store/fs/bao_file.rs b/src/store/fs/bao_file.rs
new file mode 100644
index 000000000..410317c25
--- /dev/null
+++ b/src/store/fs/bao_file.rs
@@ -0,0 +1,905 @@
+use core::fmt;
+use std::{
+    fs::{File, OpenOptions},
+    io,
+    ops::Deref,
+    path::Path,
+    sync::{Arc, Weak},
+};
+
+use bao_tree::{
+    blake3,
+    io::{
+        fsm::BaoContentItem,
+        mixed::ReadBytesAt,
+        outboard::PreOrderOutboard,
+        sync::{ReadAt, WriteAt},
+    },
+    BaoTree, ChunkRanges,
+};
+use bytes::{Bytes, BytesMut};
+use derive_more::Debug;
+use irpc::channel::mpsc;
+use tokio::sync::watch;
+use tracing::{debug, error, info, trace, Span};
+
+use super::{
+    entry_state::{DataLocation, EntryState, OutboardLocation},
+    meta::Update,
+    options::{Options, PathOptions},
+    BaoFilePart,
+};
+use crate::{
+    api::blobs::Bitfield,
+    store::{
+        fs::{
+            meta::{raw_outboard_size, Set},
+            TaskContext,
+        },
+        util::{
+            read_checksummed_and_truncate, write_checksummed, FixedSize, MemOrFile,
+            PartialMemStorage, SizeInfo, SparseMemFile, DD,
+        },
+        Hash, IROH_BLOCK_SIZE,
+    },
+};
+
+/// Storage for complete blobs. There is no longer any uncertainty about the
+/// size, so we don't need a sizes file.
+///
+/// Writing is not possible but also not needed, since the file is complete.
+/// This covers all combinations of data and outboard being in memory or on
+/// disk.
+///
+/// For the memory variant, it does reading in a zero copy way, since storage
+/// is already a `Bytes`.
+#[derive(Default)]
+pub struct CompleteStorage {
+    /// data part, which can be in memory or on disk.
+    pub data: MemOrFile<Bytes, FixedSize<File>>,
+    /// outboard part, which can be in memory or on disk.
+    pub outboard: MemOrFile<Bytes, File>,
+}
+
+impl fmt::Debug for CompleteStorage {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("CompleteStorage")
+            .field("data", &DD(self.data.fmt_short()))
+            .field("outboard", &DD(self.outboard.fmt_short()))
+            .finish()
+    }
+}
+
+impl CompleteStorage {
+    /// The size of the data file.
+    pub fn size(&self) -> u64 {
+        match &self.data {
+            MemOrFile::Mem(mem) => mem.len() as u64,
+            MemOrFile::File(file) => file.size,
+        }
+    }
+
+    pub fn bitfield(&self) -> Bitfield {
+        Bitfield::complete(self.size())
+    }
+}
+
+/// Create a file for reading and writing, but *without* truncating the existing
+/// file.
+fn create_read_write(path: impl AsRef<Path>) -> io::Result<File> {
+    OpenOptions::new()
+        .read(true)
+        .write(true)
+        .create(true)
+        .truncate(false)
+        .open(path)
+}
+
+/// Read from the given file at the given offset, until end of file or max bytes.
+fn read_to_end(file: impl ReadAt, offset: u64, max: usize) -> io::Result<Bytes> {
+    let mut res = BytesMut::new();
+    let mut buf = [0u8; 4096];
+    let mut remaining = max;
+    let mut offset = offset;
+    while remaining > 0 {
+        let end = buf.len().min(remaining);
+        let read = file.read_at(offset, &mut buf[..end])?;
+        if read == 0 {
+            // eof
+            break;
+        }
+        res.extend_from_slice(&buf[..read]);
+        offset += read as u64;
+        remaining -= read;
+    }
+    Ok(res.freeze())
+}
+
+fn max_offset(batch: &[BaoContentItem]) -> u64 {
+    batch
+        .iter()
+        .filter_map(|item| match item {
+            BaoContentItem::Leaf(leaf) => {
+                let len = leaf.data.len().try_into().unwrap();
+                let end = leaf
+                    .offset
+                    .checked_add(len)
+                    .expect("u64 overflow for leaf end");
+                Some(end)
+            }
+            _ => None,
+        })
+        .max()
+        .unwrap_or(0)
+}
+
+/// A file storage for an incomplete bao file.
+#[derive(Debug)]
+pub struct PartialFileStorage {
+    data: std::fs::File,
+    outboard: std::fs::File,
+    sizes: std::fs::File,
+    bitfield: Bitfield,
+}
+
+impl PartialFileStorage {
+    pub fn bitfield(&self) -> &Bitfield {
+        &self.bitfield
+    }
+
+    fn sync_all(&self, bitfield_path: &Path) -> io::Result<()> {
+        self.data.sync_all()?;
+        self.outboard.sync_all()?;
+        self.sizes.sync_all()?;
+        // only write the bitfield if the syncs were successful
+        trace!(
+            "writing bitfield {:?} to {}",
+            self.bitfield,
+            bitfield_path.display()
+        );
+        write_checksummed(bitfield_path, &self.bitfield)?;
+        Ok(())
+    }
+
+    fn load(hash: &Hash, options: &PathOptions) -> io::Result<Self> {
+        let bitfield_path = options.bitfield_path(hash);
+        let data = create_read_write(options.data_path(hash))?;
+        let outboard = create_read_write(options.outboard_path(hash))?;
+        let sizes = create_read_write(options.sizes_path(hash))?;
+        let bitfield = match read_checksummed_and_truncate(&bitfield_path) {
+            Ok(bitfield) => bitfield,
+            Err(cause) => {
+                trace!(
+                    "failed to read bitfield for {} at {}: {:?}",
+                    hash.to_hex(),
+                    bitfield_path.display(),
+                    cause
+                );
+                trace!("reconstructing bitfield from outboard");
+                let size = read_size(&sizes).ok().unwrap_or_default();
+                let outboard = PreOrderOutboard {
+                    data: &outboard,
+                    tree: BaoTree::new(size, IROH_BLOCK_SIZE),
+                    root: blake3::Hash::from(*hash),
+                };
+                let mut ranges = ChunkRanges::empty();
+                for range in bao_tree::io::sync::valid_ranges(outboard, &data, &ChunkRanges::all())
+                    .into_iter()
+                    .flatten()
+                {
+                    ranges |= ChunkRanges::from(range);
+                }
+                info!("reconstructed range is {:?}", ranges);
+                Bitfield::new(ranges, size)
+            }
+        };
+        Ok(Self {
+            data,
+            outboard,
+            sizes,
+            bitfield,
+        })
+    }
+
+    fn into_complete(
+        self,
+        size: u64,
+        options: &Options,
+    ) -> io::Result<(CompleteStorage, EntryState<Bytes>)> {
+        let outboard_size = raw_outboard_size(size);
+        let (data, data_location) = if options.is_inlined_data(size) {
+            let data = read_to_end(&self.data, 0, size as usize)?;
+            (MemOrFile::Mem(data.clone()), DataLocation::Inline(data))
+        } else {
+            (
+                MemOrFile::File(FixedSize::new(self.data, size)),
+                DataLocation::Owned(size),
+            )
+        };
+        let (outboard, outboard_location) = if options.is_inlined_outboard(outboard_size) {
+            if outboard_size == 0 {
+                (MemOrFile::empty(), OutboardLocation::NotNeeded)
+            } else {
+                let outboard = read_to_end(&self.outboard, 0, outboard_size as usize)?;
+                trace!("read outboard from file: {:?}", outboard.len());
+                (
+                    MemOrFile::Mem(outboard.clone()),
+                    OutboardLocation::Inline(outboard),
+                )
+            }
+        } else {
+            (MemOrFile::File(self.outboard), OutboardLocation::Owned)
+        };
+        // todo: notify the store that the state has changed to complete
+        Ok((
+            CompleteStorage { data, outboard },
+            EntryState::Complete {
+                data_location,
+                outboard_location,
+            },
+        ))
+    }
+
+    fn current_size(&self) -> io::Result<u64> {
+        read_size(&self.sizes)
+    }
+
+    fn write_batch(&mut self, size: u64, batch: &[BaoContentItem]) -> io::Result<()> {
+        let tree = BaoTree::new(size, IROH_BLOCK_SIZE);
+        for item in batch {
+            match item {
+                BaoContentItem::Parent(parent) => {
+                    if let Some(offset) = tree.pre_order_offset(parent.node) {
+                        let o0 = offset * 64;
+                        self.outboard
+                            .write_all_at(o0, parent.pair.0.as_bytes().as_slice())?;
+                        self.outboard
+                            .write_all_at(o0 + 32, parent.pair.1.as_bytes().as_slice())?;
+                    }
+                }
+                BaoContentItem::Leaf(leaf) => {
+                    let o0 = leaf.offset;
+                    // divide by chunk size, multiply by 8
+                    let index = (leaf.offset >> (tree.block_size().chunk_log() + 10)) << 3;
+                    trace!(
+                        "write_batch f={:?} o={} l={}",
+                        self.data,
+                        o0,
+                        leaf.data.len()
+                    );
+                    self.data.write_all_at(o0, leaf.data.as_ref())?;
+                    let size = tree.size();
+                    self.sizes.write_all_at(index, &size.to_le_bytes())?;
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+fn read_size(size_file: &File) -> io::Result<u64> {
+    let len = size_file.metadata()?.len();
+    if len < 8 {
+        Ok(0)
+    } else {
+        let len = len & !7;
+        let mut buf = [0u8; 8];
+        size_file.read_exact_at(len - 8, &mut buf)?;
+        Ok(u64::from_le_bytes(buf))
+    }
+}
+
+/// The storage for a bao file. This can be either in memory or on disk.
+#[derive(derive_more::From)]
+pub(crate) enum BaoFileStorage {
+    /// The entry is incomplete and in memory.
+    ///
+    /// Since it is incomplete, it must be writeable.
+    ///
+    /// This is used mostly for tiny entries, <= 16 KiB. But in principle it
+    /// can be used for larger sizes.
+    ///
+    /// Incomplete mem entries are *not* persisted at all. So if the store
+    /// crashes they will be gone.
+    PartialMem(PartialMemStorage),
+    /// The entry is incomplete and on disk.
+    Partial(PartialFileStorage),
+    /// The entry is complete. Outboard and data can come from different sources
+    /// (memory or file).
+    ///
+    /// Writing to this is a no-op, since it is already complete.
+    Complete(CompleteStorage),
+    /// We will get into that state if there is an io error in the middle of an operation
+    ///
+    /// Also, when the handle is dropped we will poison the storage, so poisoned
+    /// can be seen when the handle is revived during the drop.
+    ///
+    /// BaoFileHandleWeak::upgrade() will return None if the storage is poisoned,
+    /// treat it as dead.
+    Poisoned,
+}
+
+impl fmt::Debug for BaoFileStorage {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            BaoFileStorage::PartialMem(x) => x.fmt(f),
+            BaoFileStorage::Partial(x) => x.fmt(f),
+            BaoFileStorage::Complete(x) => x.fmt(f),
+            BaoFileStorage::Poisoned => f.debug_struct("Poisoned").finish(),
+        }
+    }
+}
+
+impl Default for BaoFileStorage {
+    fn default() -> Self {
+        BaoFileStorage::Complete(Default::default())
+    }
+}
+
+impl PartialMemStorage {
+    /// Converts this storage into a complete storage, using the given hash for
+    /// path names and the given options for decisions about inlining.
+    fn into_complete(
+        self,
+        hash: &Hash,
+        ctx: &TaskContext,
+    ) -> io::Result<(CompleteStorage, EntryState<Bytes>)> {
+        let size = self.current_size();
+        let outboard_size = raw_outboard_size(size);
+        let (data, data_location) = if ctx.options.is_inlined_data(size) {
+            let data: Bytes = self.data.to_vec().into();
+            (MemOrFile::Mem(data.clone()), DataLocation::Inline(data))
+        } else {
+            let data_path = ctx.options.path.data_path(hash);
+            let mut data_file = create_read_write(&data_path)?;
+            self.data.persist(&mut data_file)?;
+            (
+                MemOrFile::File(FixedSize::new(data_file, size)),
+                DataLocation::Owned(size),
+            )
+        };
+        let (outboard, outboard_location) = if ctx.options.is_inlined_outboard(outboard_size) {
+            if outboard_size > 0 {
+                let outboard: Bytes = self.outboard.to_vec().into();
+                (
+                    MemOrFile::Mem(outboard.clone()),
+                    OutboardLocation::Inline(outboard),
+                )
+            } else {
+                (MemOrFile::empty(), OutboardLocation::NotNeeded)
+            }
+        } else {
+            let outboard_path = ctx.options.path.outboard_path(hash);
+            let mut outboard_file = create_read_write(&outboard_path)?;
+            self.outboard.persist(&mut outboard_file)?;
+            let outboard_location = if outboard_size == 0 {
+                OutboardLocation::NotNeeded
+            } else {
+                OutboardLocation::Owned
+            };
+            (MemOrFile::File(outboard_file), outboard_location)
+        };
+        Ok((
+            CompleteStorage { data, outboard },
+            EntryState::Complete {
+                data_location,
+                outboard_location,
+            },
+        ))
+    }
+}
+
+impl BaoFileStorage {
+    pub fn bitfield(&self) -> Bitfield {
+        match self {
+            BaoFileStorage::Complete(x) => Bitfield::complete(x.data.size()),
+            BaoFileStorage::PartialMem(x) => x.bitfield.clone(),
+            BaoFileStorage::Partial(x) => x.bitfield.clone(),
+            BaoFileStorage::Poisoned => {
+                panic!("poisoned storage should not be used")
+            }
+        }
+    }
+
+    fn write_batch(
+        self,
+        batch: &[BaoContentItem],
+        bitfield: &Bitfield,
+        ctx: &TaskContext,
+        hash: &Hash,
+    ) -> io::Result<(Self, Option<EntryState<bytes::Bytes>>)> {
+        Ok(match self {
+            BaoFileStorage::PartialMem(mut ms) => {
+                // check if we need to switch to file mode, otherwise write to memory
+                if max_offset(batch) <= ctx.options.inline.max_data_inlined {
+                    ms.write_batch(bitfield.size(), batch)?;
+                    let changes = ms.bitfield.update(bitfield);
+                    let new = changes.new_state();
+                    if new.complete {
+                        let (cs, update) = ms.into_complete(hash, ctx)?;
+                        (cs.into(), Some(update))
+                    } else {
+                        let fs = ms.persist(ctx, hash)?;
+                        let update = EntryState::Partial {
+                            size: new.validated_size,
+                        };
+                        (fs.into(), Some(update))
+                    }
+                } else {
+                    // *first* switch to file mode, *then* write the batch.
+                    //
+                    // otherwise we might allocate a lot of memory if we get
+                    // a write at the end of a very large file.
+                    //
+                    // opt: we should check if we become complete to avoid going from mem to partial to complete
+                    let mut fs = ms.persist(ctx, hash)?;
+                    fs.write_batch(bitfield.size(), batch)?;
+                    let changes = fs.bitfield.update(bitfield);
+                    let new = changes.new_state();
+                    if new.complete {
+                        let size = new.validated_size.unwrap();
+                        let (cs, update) = fs.into_complete(size, &ctx.options)?;
+                        (cs.into(), Some(update))
+                    } else {
+                        let update = EntryState::Partial {
+                            size: new.validated_size,
+                        };
+                        (fs.into(), Some(update))
+                    }
+                }
+            }
+            BaoFileStorage::Partial(mut fs) => {
+                fs.write_batch(bitfield.size(), batch)?;
+                let changes = fs.bitfield.update(bitfield);
+                let new = changes.new_state();
+                if new.complete {
+                    let size = new.validated_size.unwrap();
+                    let (cs, update) = fs.into_complete(size, &ctx.options)?;
+                    (cs.into(), Some(update))
+                } else if changes.was_validated() {
+                    // we are still partial, but now we know the size
+                    let update = EntryState::Partial {
+                        size: new.validated_size,
+                    };
+                    (fs.into(), Some(update))
+                } else {
+                    (fs.into(), None)
+                }
+            }
+            BaoFileStorage::Complete(_) => {
+                // we are complete, so just ignore the write
+                // unless there is a bug, this would just write the exact same data
+                (self, None)
+            }
+            BaoFileStorage::Poisoned => {
+                // we are poisoned, so just ignore the write
+                (self, None)
+            }
+        })
+    }
+
+    /// Create a new mutable mem storage.
+    pub fn partial_mem() -> Self {
+        Self::PartialMem(Default::default())
+    }
+
+    /// Call sync_all on all the files.
+    #[allow(dead_code)]
+    fn sync_all(&self) -> io::Result<()> {
+        match self {
+            Self::Complete(_) => Ok(()),
+            Self::PartialMem(_) => Ok(()),
+            Self::Partial(file) => {
+                file.data.sync_all()?;
+                file.outboard.sync_all()?;
+                file.sizes.sync_all()?;
+                Ok(())
+            }
+            Self::Poisoned => {
+                // we are poisoned, so just ignore the sync
+                Ok(())
+            }
+        }
+    }
+
+    pub fn take(&mut self) -> Self {
+        std::mem::replace(self, BaoFileStorage::Poisoned)
+    }
+}
+
+/// A weak reference to a bao file handle.
+#[derive(Debug, Clone)]
+pub struct BaoFileHandleWeak(Weak<BaoFileHandleInner>);
+
+impl BaoFileHandleWeak {
+    /// Upgrade to a strong reference if possible.
+    pub fn upgrade(&self) -> Option<BaoFileHandle> {
+        let inner = self.0.upgrade()?;
+        if let &BaoFileStorage::Poisoned = inner.storage.borrow().deref() {
+            trace!("poisoned storage, cannot upgrade");
+            return None;
+        };
+        Some(BaoFileHandle(inner))
+    }
+
+    /// True if the handle is definitely dead.
+    pub fn is_dead(&self) -> bool {
+        self.0.strong_count() == 0
+    }
+}
+
+/// The inner part of a bao file handle.
+pub struct BaoFileHandleInner {
+    pub(crate) storage: watch::Sender<BaoFileStorage>,
+    hash: Hash,
+    options: Arc<Options>,
+}
+
+impl fmt::Debug for BaoFileHandleInner {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let guard = self.storage.borrow();
+        let storage = guard.deref();
+        f.debug_struct("BaoFileHandleInner")
+            .field("hash", &DD(self.hash))
+            .field("storage", &storage)
+            .finish_non_exhaustive()
+    }
+}
+
+/// A cheaply cloneable handle to a bao file, including the hash and the configuration.
+#[derive(Debug, Clone, derive_more::Deref)]
+pub struct BaoFileHandle(Arc<BaoFileHandleInner>);
+
+impl Drop for BaoFileHandle {
+    fn drop(&mut self) {
+        self.0.storage.send_if_modified(|guard| {
+            if Arc::strong_count(&self.0) > 1 {
+                return false;
+            }
+            // there is the possibility that somebody else will increase the strong count
+            // here. there is nothing we can do about it, but they won't be able to
+            // access the internals of the handle because we have the lock.
+            //
+            // We poison the storage. A poisoned storage is considered dead and will
+            // have to be recreated, but only *after* we are done with persisting
+            // the bitfield.
+            let BaoFileStorage::Partial(fs) = guard.take() else {
+                return false;
+            };
+            let options = &self.options;
+            let path = options.path.bitfield_path(&self.hash);
+            trace!(
+                "writing bitfield for hash {} to {}",
+                self.hash,
+                path.display()
+            );
+            if let Err(cause) = fs.sync_all(&path) {
+                error!(
+                    "failed to write bitfield for {} at {}: {:?}",
+                    self.hash,
+                    path.display(),
+                    cause
+                );
+            }
+            false
+        });
+    }
+}
+
+/// A reader for a bao file, reading just the data.
+#[derive(Debug)]
+pub struct DataReader(BaoFileHandle);
+
+impl ReadBytesAt for DataReader {
+    fn read_bytes_at(&self, offset: u64, size: usize) -> std::io::Result<Bytes> {
+        let guard = self.0.storage.borrow();
+        match guard.deref() {
+            BaoFileStorage::PartialMem(x) => x.data.read_bytes_at(offset, size),
+            BaoFileStorage::Partial(x) => x.data.read_bytes_at(offset, size),
+            BaoFileStorage::Complete(x) => x.data.read_bytes_at(offset, size),
+            BaoFileStorage::Poisoned => io::Result::Err(io::Error::other("poisoned storage")),
+        }
+    }
+}
+
+/// A reader for the outboard part of a bao file.
+#[derive(Debug)]
+pub struct OutboardReader(BaoFileHandle);
+
+impl ReadAt for OutboardReader {
+    fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<usize> {
+        let guard = self.0.storage.borrow();
+        match guard.deref() {
+            BaoFileStorage::Complete(x) => x.outboard.read_at(offset, buf),
+            BaoFileStorage::PartialMem(x) => x.outboard.read_at(offset, buf),
+            BaoFileStorage::Partial(x) => x.outboard.read_at(offset, buf),
+            BaoFileStorage::Poisoned => io::Result::Err(io::Error::other("poisoned storage")),
+        }
+    }
+}
+
+impl BaoFileHandle {
+    #[allow(dead_code)]
+    pub fn id(&self) -> usize {
+        Arc::as_ptr(&self.0) as usize
+    }
+
+    /// Create a new bao file handle.
+    ///
+    /// This will create a new file handle with an empty memory storage.
+    pub fn new_partial_mem(hash: Hash, options: Arc<Options>) -> Self {
+        let storage = BaoFileStorage::partial_mem();
+        Self(Arc::new(BaoFileHandleInner {
+            storage: watch::Sender::new(storage),
+            hash,
+            options: options.clone(),
+        }))
+    }
+
+    /// Create a new bao file handle with a partial file.
+    pub(super) async fn new_partial_file(hash: Hash, ctx: &TaskContext) -> io::Result<Self> {
+        let options = ctx.options.clone();
+        let storage = PartialFileStorage::load(&hash, &options.path)?;
+        let storage = if storage.bitfield.is_complete() {
+            let size = storage.bitfield.size;
+            let (storage, entry_state) = storage.into_complete(size, &options)?;
+            debug!("File was reconstructed as complete");
+            let (tx, rx) = crate::util::channel::oneshot::channel();
+            ctx.db
+                .sender
+                .send(
+                    Set {
+                        hash,
+                        state: entry_state,
+                        tx,
+                        span: Span::current(),
+                    }
+                    .into(),
+                )
+                .await
+                .map_err(|_| io::Error::other("send update"))?;
+            rx.await.map_err(|_| io::Error::other("receive update"))??;
+            storage.into()
+        } else {
+            storage.into()
+        };
+        Ok(Self(Arc::new(BaoFileHandleInner {
+            storage: watch::Sender::new(storage),
+            hash,
+            options,
+        })))
+    }
+
+    /// Create a new complete bao file handle.
+    pub fn new_complete(
+        hash: Hash,
+        data: MemOrFile<Bytes, FixedSize<File>>,
+        outboard: MemOrFile<Bytes, File>,
+        options: Arc<Options>,
+    ) -> Self {
+        let storage = CompleteStorage { data, outboard }.into();
+        Self(Arc::new(BaoFileHandleInner {
+            storage: watch::Sender::new(storage),
+            hash,
+            options,
+        }))
+    }
+
+    /// Complete the handle
+    pub fn complete(
+        &self,
+        data: MemOrFile<Bytes, FixedSize<File>>,
+        outboard: MemOrFile<Bytes, File>,
+    ) {
+        self.storage.send_if_modified(|guard| {
+            let res = match guard {
+                BaoFileStorage::Complete(_) => None,
+                BaoFileStorage::PartialMem(entry) => Some(&mut entry.bitfield),
+                BaoFileStorage::Partial(entry) => Some(&mut entry.bitfield),
+                BaoFileStorage::Poisoned => None,
+            };
+            if let Some(bitfield) = res {
+                bitfield.update(&Bitfield::complete(data.size()));
+                *guard = BaoFileStorage::Complete(CompleteStorage { data, outboard });
+                true
+            } else {
+                true
+            }
+        });
+    }
+
+    pub fn subscribe(&self) -> BaoFileStorageSubscriber {
+        BaoFileStorageSubscriber::new(self.0.storage.subscribe())
+    }
+
+    /// True if the file is complete.
+    #[allow(dead_code)]
+    pub fn is_complete(&self) -> bool {
+        matches!(self.storage.borrow().deref(), BaoFileStorage::Complete(_))
+    }
+
+    /// An AsyncSliceReader for the data file.
+    ///
+    /// Caution: this is a reader for the unvalidated data file. Reading this
+    /// can produce data that does not match the hash.
+    pub fn data_reader(&self) -> DataReader {
+        DataReader(self.clone())
+    }
+
+    /// An AsyncSliceReader for the outboard file.
+    ///
+    /// The outboard file is used to validate the data file. It is not guaranteed
+    /// to be complete.
+    pub fn outboard_reader(&self) -> OutboardReader {
+        OutboardReader(self.clone())
+    }
+
+    /// The most precise known total size of the data file.
+    pub fn current_size(&self) -> io::Result<u64> {
+        match self.storage.borrow().deref() {
+            BaoFileStorage::Complete(mem) => Ok(mem.size()),
+            BaoFileStorage::PartialMem(mem) => Ok(mem.current_size()),
+            BaoFileStorage::Partial(file) => file.current_size(),
+            BaoFileStorage::Poisoned => io::Result::Err(io::Error::other("poisoned storage")),
+        }
+    }
+
+    /// The most precise known total size of the data file.
+    pub fn bitfield(&self) -> io::Result<Bitfield> {
+        match self.storage.borrow().deref() {
+            BaoFileStorage::Complete(mem) => Ok(mem.bitfield()),
+            BaoFileStorage::PartialMem(mem) => Ok(mem.bitfield().clone()),
+            BaoFileStorage::Partial(file) => Ok(file.bitfield().clone()),
+            BaoFileStorage::Poisoned => io::Result::Err(io::Error::other("poisoned storage")),
+        }
+    }
+
+    /// The outboard for the file.
+    pub fn outboard(&self) -> io::Result<PreOrderOutboard<OutboardReader>> {
+        let root = self.hash.into();
+        let tree = BaoTree::new(self.current_size()?, IROH_BLOCK_SIZE);
+        let outboard = self.outboard_reader();
+        Ok(PreOrderOutboard {
+            root,
+            tree,
+            data: outboard,
+        })
+    }
+
+    /// The hash of the file.
+    pub fn hash(&self) -> Hash {
+        self.hash
+    }
+
+    /// Downgrade to a weak reference.
+    pub fn downgrade(&self) -> BaoFileHandleWeak {
+        BaoFileHandleWeak(Arc::downgrade(&self.0))
+    }
+
+    /// Write a batch and notify the db
+    pub(super) async fn write_batch(
+        &self,
+        batch: &[BaoContentItem],
+        bitfield: &Bitfield,
+        ctx: &TaskContext,
+    ) -> io::Result<()> {
+        trace!("write_batch bitfield={:?} batch={}", bitfield, batch.len());
+        let mut res = Ok(None);
+        self.storage.send_if_modified(|state| {
+            let Ok((state1, update)) = state.take().write_batch(batch, bitfield, ctx, &self.hash)
+            else {
+                res = Err(io::Error::other("write batch failed"));
+                return false;
+            };
+            res = Ok(update);
+            *state = state1;
+            true
+        });
+        if let Some(update) = res? {
+            ctx.db
+                .sender
+                .send(
+                    Update {
+                        hash: self.hash,
+                        state: update,
+                        tx: None,
+                        span: Span::current(),
+                    }
+                    .into(),
+                )
+                .await
+                .map_err(|_| io::Error::other("send update"))?;
+        }
+        Ok(())
+    }
+}
+
+impl PartialMemStorage {
+    /// Persist the batch to disk, creating a FileBatch.
+    fn persist(self, ctx: &TaskContext, hash: &Hash) -> io::Result<PartialFileStorage> {
+        let options = &ctx.options.path;
+        ctx.protect.protect(
+            *hash,
+            [
+                BaoFilePart::Data,
+                BaoFilePart::Outboard,
+                BaoFilePart::Sizes,
+                BaoFilePart::Bitfield,
+            ],
+        );
+        let mut data = create_read_write(options.data_path(hash))?;
+        let mut outboard = create_read_write(options.outboard_path(hash))?;
+        let mut sizes = create_read_write(options.sizes_path(hash))?;
+        self.data.persist(&mut data)?;
+        self.outboard.persist(&mut outboard)?;
+        self.size.persist(&mut sizes)?;
+        data.sync_all()?;
+        outboard.sync_all()?;
+        sizes.sync_all()?;
+        Ok(PartialFileStorage {
+            data,
+            outboard,
+            sizes,
+            bitfield: self.bitfield,
+        })
+    }
+
+    /// Get the parts data, outboard and sizes
+    #[allow(dead_code)]
+    pub fn into_parts(self) -> (SparseMemFile, SparseMemFile, SizeInfo) {
+        (self.data, self.outboard, self.size)
+    }
+}
+
+pub struct BaoFileStorageSubscriber {
+    receiver: watch::Receiver<BaoFileStorage>,
+}
+
+impl BaoFileStorageSubscriber {
+    pub fn new(receiver: watch::Receiver<BaoFileStorage>) -> Self {
+        Self { receiver }
+    }
+
+    /// Forward observed *values* to the given sender
+    ///
+    /// Returns an error if sending fails, or if the last sender is dropped
+    pub async fn forward(mut self, mut tx: mpsc::Sender<Bitfield>) -> anyhow::Result<()> {
+        let value = self.receiver.borrow().bitfield();
+        tx.send(value).await?;
+        loop {
+            self.update_or_closed(&mut tx).await?;
+            let value = self.receiver.borrow().bitfield();
+            tx.send(value.clone()).await?;
+        }
+    }
+
+    /// Forward observed *deltas* to the given sender
+    ///
+    /// Returns an error if sending fails, or if the last sender is dropped
+    #[allow(dead_code)]
+    pub async fn forward_delta(mut self, mut tx: mpsc::Sender<Bitfield>) -> anyhow::Result<()> {
+        let value = self.receiver.borrow().bitfield();
+        let mut old = value.clone();
+        tx.send(value).await?;
+        loop {
+            self.update_or_closed(&mut tx).await?;
+            let new = self.receiver.borrow().bitfield();
+            let diff = old.diff(&new);
+            if diff.is_empty() {
+                continue;
+            }
+            tx.send(diff).await?;
+            old = new;
+        }
+    }
+
+    async fn update_or_closed(&mut self, tx: &mut mpsc::Sender<Bitfield>) -> anyhow::Result<()> {
+        tokio::select! {
+            _ = tx.closed() => {
+                // the sender is closed, we are done
+                Err(irpc::channel::SendError::ReceiverClosed.into())
+            }
+            e = self.receiver.changed() => Ok(e?),
+        }
+    }
+}
diff --git a/src/store/fs/delete_set.rs b/src/store/fs/delete_set.rs
new file mode 100644
index 000000000..ec7b7103a
--- /dev/null
+++ b/src/store/fs/delete_set.rs
@@ -0,0 +1,149 @@
+use std::{
+    collections::BTreeSet,
+    sync::{Arc, Mutex},
+};
+
+use tracing::warn;
+
+use super::options::PathOptions;
+use crate::Hash;
+
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub(super) enum BaoFilePart {
+    Outboard,
+    Data,
+    Sizes,
+    Bitfield,
+}
+
+/// Creates a pair of a protect handle and a delete handle.
+///
+/// The protect handle can be used to protect files from deletion.
+/// The delete handle can be used to create transactions in which files can be marked for deletion.
+pub(super) fn pair(options: Arc<PathOptions>) -> (ProtectHandle, DeleteHandle) {
+    let ds = Arc::new(Mutex::new(DeleteSet::default()));
+    (ProtectHandle(ds.clone()), DeleteHandle::new(ds, options))
+}
+
+/// Helper to keep track of files to delete after a transaction is committed.
+#[derive(Debug, Default)]
+struct DeleteSet(BTreeSet<(Hash, BaoFilePart)>);
+
+impl DeleteSet {
+    /// Mark a file as to be deleted after the transaction is committed.
+    fn delete(&mut self, hash: Hash, parts: impl IntoIterator<Item = BaoFilePart>) {
+        for part in parts {
+            self.0.insert((hash, part));
+        }
+    }
+
+    /// Mark a file as to be kept after the transaction is committed.
+    ///
+    /// This will cancel any previous delete for the same file in the same transaction.
+    fn protect(&mut self, hash: Hash, parts: impl IntoIterator<Item = BaoFilePart>) {
+        for part in parts {
+            self.0.remove(&(hash, part));
+        }
+    }
+
+    /// Apply the delete set and clear it.
+    ///
+    /// This will delete all files marked for deletion and then clear the set.
+    /// Errors will just be logged.
+    fn commit(&mut self, options: &PathOptions) {
+        for (hash, to_delete) in &self.0 {
+            tracing::debug!("deleting {:?} for {hash}", to_delete);
+            let path = match to_delete {
+                BaoFilePart::Data => options.data_path(hash),
+                BaoFilePart::Outboard => options.outboard_path(hash),
+                BaoFilePart::Sizes => options.sizes_path(hash),
+                BaoFilePart::Bitfield => options.bitfield_path(hash),
+            };
+            if let Err(cause) = std::fs::remove_file(&path) {
+                // Ignore NotFound errors, if the file is already gone that's fine.
+                if cause.kind() != std::io::ErrorKind::NotFound {
+                    warn!(
+                        "failed to delete {:?} {}: {}",
+                        to_delete,
+                        path.display(),
+                        cause
+                    );
+                }
+            }
+        }
+        self.0.clear();
+    }
+
+    fn clear(&mut self) {
+        self.0.clear();
+    }
+
+    fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct ProtectHandle(Arc<Mutex<DeleteSet>>);
+
+/// Protect handle, to be used concurrently with transactions to mark files for keeping.
+impl ProtectHandle {
+    /// Inside or outside a transaction, mark files as to be kept
+    ///
+    /// If we are not inside a transaction, this will do nothing.
+    pub(super) fn protect(&self, hash: Hash, parts: impl IntoIterator<Item = BaoFilePart>) {
+        let mut guard = self.0.lock().unwrap();
+        guard.protect(hash, parts);
+    }
+}
+
+/// A delete handle. The only thing you can do with this is to open transactions that keep track of files to delete.
+#[derive(Debug)]
+pub(super) struct DeleteHandle {
+    ds: Arc<Mutex<DeleteSet>>,
+    options: Arc<PathOptions>,
+}
+
+impl DeleteHandle {
+    fn new(ds: Arc<Mutex<DeleteSet>>, options: Arc<PathOptions>) -> Self {
+        Self { ds, options }
+    }
+
+    /// Open a file transaction. You can open only one transaction at a time.
+    pub(super) fn begin_write(&mut self) -> FileTransaction<'_> {
+        FileTransaction::new(self)
+    }
+}
+
+/// A file transaction. Inside a transaction, you can mark files for deletion.
+///
+/// Dropping a transaction will clear the delete set. Committing a transaction will apply the delete set by actually deleting the files.
+#[derive(Debug)]
+pub(super) struct FileTransaction<'a>(&'a DeleteHandle);
+
+impl<'a> FileTransaction<'a> {
+    fn new(inner: &'a DeleteHandle) -> Self {
+        let guard = inner.ds.lock().unwrap();
+        debug_assert!(guard.is_empty());
+        drop(guard);
+        Self(inner)
+    }
+
+    /// Mark files as to be deleted
+    pub fn delete(&self, hash: Hash, parts: impl IntoIterator<Item = BaoFilePart>) {
+        let mut guard = self.0.ds.lock().unwrap();
+        guard.delete(hash, parts);
+    }
+
+    /// Apply the delete set and clear it.
+    pub fn commit(self) {
+        let mut guard = self.0.ds.lock().unwrap();
+        guard.commit(&self.0.options);
+    }
+}
+
+impl Drop for FileTransaction<'_> {
+    fn drop(&mut self) {
+        self.0.ds.lock().unwrap().clear();
+    }
+}
diff --git a/src/store/fs/entry_state.rs b/src/store/fs/entry_state.rs
new file mode 100644
index 000000000..4cdf9ec0c
--- /dev/null
+++ b/src/store/fs/entry_state.rs
@@ -0,0 +1,297 @@
+use std::{fmt::Debug, path::PathBuf};
+
+use serde::{Deserialize, Serialize};
+use smallvec::SmallVec;
+
+use super::meta::{ActorError, ActorResult};
+use crate::store::util::SliceInfoExt;
+
+/// Location of the data.
+///
+/// Data can be inlined in the database, a file conceptually owned by the store,
+/// or a number of external files conceptually owned by the user.
+///
+/// Only complete data can be inlined.
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+pub enum DataLocation<I = (), E = ()> {
+    /// Data is in the inline_data table.
+    Inline(I),
+    /// Data is in the canonical location in the data directory.
+    Owned(E),
+    /// Data is in several external locations. This should be a non-empty list.
+    External(Vec<PathBuf>, E),
+}
+
+impl<I: AsRef<[u8]>, E: Debug> DataLocation<I, E> {
+    fn fmt_short(&self) -> String {
+        match self {
+            DataLocation::Inline(d) => {
+                format!("Inline({}, addr={})", d.as_ref().len(), d.addr_short())
+            }
+            DataLocation::Owned(e) => format!("Owned({e:?})"),
+            DataLocation::External(paths, e) => {
+                let paths = paths.iter().map(|p| p.display()).collect::<Vec<_>>();
+                format!("External({paths:?}, {e:?})")
+            }
+        }
+    }
+}
+
+impl<X> DataLocation<X, u64> {
+    #[allow(clippy::result_large_err)]
+    fn union(self, that: DataLocation<X, u64>) -> ActorResult<Self> {
+        Ok(match (self, that) {
+            (
+                DataLocation::External(mut paths, a_size),
+                DataLocation::External(b_paths, b_size),
+            ) => {
+                if a_size != b_size {
+                    return Err(ActorError::inconsistent(format!(
+                        "complete size mismatch {a_size} {b_size}"
+                    )));
+                }
+                paths.extend(b_paths);
+                paths.sort();
+                paths.dedup();
+                DataLocation::External(paths, a_size)
+            }
+            (_, b @ DataLocation::Owned(_)) => {
+                // owned needs to win, since it has an associated file. Choosing
+                // external would orphan the file.
+                b
+            }
+            (a @ DataLocation::Owned(_), _) => {
+                // owned needs to win, since it has an associated file. Choosing
+                // external would orphan the file.
+                a
+            }
+            (_, b @ DataLocation::Inline(_)) => {
+                // inline needs to win, since it has associated data. Choosing
+                // external would orphan the file.
+                b
+            }
+            (a @ DataLocation::Inline(_), _) => {
+                // inline needs to win, since it has associated data. Choosing
+                // external would orphan the file.
+                a
+            }
+        })
+    }
+}
+
+impl<I, E> DataLocation<I, E> {
+    #[allow(dead_code)]
+    pub fn discard_inline_data(self) -> DataLocation<(), E> {
+        match self {
+            DataLocation::Inline(_) => DataLocation::Inline(()),
+            DataLocation::Owned(x) => DataLocation::Owned(x),
+            DataLocation::External(paths, x) => DataLocation::External(paths, x),
+        }
+    }
+
+    pub fn split_inline_data(self) -> (DataLocation<(), E>, Option<I>) {
+        match self {
+            DataLocation::Inline(x) => (DataLocation::Inline(()), Some(x)),
+            DataLocation::Owned(x) => (DataLocation::Owned(x), None),
+            DataLocation::External(paths, x) => (DataLocation::External(paths, x), None),
+        }
+    }
+}
+
+/// Location of the outboard.
+///
+/// Outboard can be inlined in the database or a file conceptually owned by the store.
+/// Outboards are implementation specific to the store and as such are always owned.
+///
+/// Only complete outboards can be inlined.
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+pub enum OutboardLocation<I = ()> {
+    /// Outboard is in the inline_outboard table.
+    Inline(I),
+    /// Outboard is in the canonical location in the data directory.
+    Owned,
+    /// Outboard is not needed
+    NotNeeded,
+}
+
+impl<I: AsRef<[u8]>> OutboardLocation<I> {
+    fn fmt_short(&self) -> String {
+        match self {
+            OutboardLocation::Inline(d) => format!("Inline({})", d.as_ref().len()),
+            OutboardLocation::Owned => "Owned".to_string(),
+            OutboardLocation::NotNeeded => "NotNeeded".to_string(),
+        }
+    }
+}
+
+impl<I> OutboardLocation<I> {
+    pub fn inline(data: I) -> Self
+    where
+        I: AsRef<[u8]>,
+    {
+        if data.as_ref().is_empty() {
+            OutboardLocation::NotNeeded
+        } else {
+            OutboardLocation::Inline(data)
+        }
+    }
+
+    #[allow(dead_code)]
+    pub fn discard_extra_data(self) -> OutboardLocation<()> {
+        match self {
+            Self::Inline(_) => OutboardLocation::Inline(()),
+            Self::Owned => OutboardLocation::Owned,
+            Self::NotNeeded => OutboardLocation::NotNeeded,
+        }
+    }
+
+    pub fn split_inline_data(self) -> (OutboardLocation<()>, Option<I>) {
+        match self {
+            Self::Inline(x) => (OutboardLocation::Inline(()), Some(x)),
+            Self::Owned => (OutboardLocation::Owned, None),
+            Self::NotNeeded => (OutboardLocation::NotNeeded, None),
+        }
+    }
+}
+
+/// The information about an entry that we keep in the entry table for quick access.
+///
+/// The exact info to store here is TBD, so usually you should use the accessor methods.
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+pub enum EntryState<I = ()> {
+    /// For a complete entry we always know the size. It does not make much sense
+    /// to write to a complete entry, so they are much easier to share.
+    Complete {
+        /// Location of the data.
+        data_location: DataLocation<I, u64>,
+        /// Location of the outboard.
+        outboard_location: OutboardLocation<I>,
+    },
+    /// Partial entries are entries for which we know the hash, but don't have
+    /// all the data. They are created when syncing from somewhere else by hash.
+    ///
+    /// As such they are always owned. There is also no inline storage for them.
+    /// Non short lived partial entries always live in the file system, and for
+    /// short lived ones we never create a database entry in the first place.
+    Partial {
+        /// Once we get the last chunk of a partial entry, we have validated
+        /// the size of the entry despite it still being incomplete.
+        ///
+        /// E.g. a giant file where we just requested the last chunk.
+        size: Option<u64>,
+    },
+}
+
+impl<I> EntryState<I> {
+    pub fn is_complete(&self) -> bool {
+        matches!(self, Self::Complete { .. })
+    }
+
+    pub fn is_partial(&self) -> bool {
+        matches!(self, Self::Partial { .. })
+    }
+}
+
+impl Default for EntryState {
+    fn default() -> Self {
+        Self::Partial { size: None }
+    }
+}
+
+impl<I: AsRef<[u8]>> EntryState<I> {
+    pub fn fmt_short(&self) -> String {
+        match self {
+            Self::Complete {
+                data_location,
+                outboard_location,
+            } => format!(
+                "Complete {{ data: {}, outboard: {} }}",
+                data_location.fmt_short(),
+                outboard_location.fmt_short()
+            ),
+            Self::Partial { size } => format!("Partial {{ size: {size:?} }}"),
+        }
+    }
+}
+
+impl EntryState {
+    #[allow(clippy::result_large_err)]
+    pub fn union(old: Self, new: Self) -> ActorResult<Self> {
+        match (old, new) {
+            (
+                Self::Complete {
+                    data_location,
+                    outboard_location,
+                },
+                Self::Complete {
+                    data_location: b_data_location,
+                    ..
+                },
+            ) => Ok(Self::Complete {
+                // combine external paths if needed
+                data_location: data_location.union(b_data_location)?,
+                outboard_location,
+            }),
+            (a @ Self::Complete { .. }, Self::Partial { .. }) =>
+            // complete wins over partial
+            {
+                Ok(a)
+            }
+            (Self::Partial { .. }, b @ Self::Complete { .. }) =>
+            // complete wins over partial
+            {
+                Ok(b)
+            }
+            (Self::Partial { size: a_size }, Self::Partial { size: b_size }) =>
+            // keep known size from either entry
+            {
+                let size = match (a_size, b_size) {
+                    (Some(a_size), Some(b_size)) => {
+                        // validated sizes are different. this means that at
+                        // least one validation was wrong, which would be a bug
+                        // in bao-tree.
+                        if a_size != b_size {
+                            return Err(ActorError::inconsistent(format!(
+                                "validated size mismatch {a_size} {b_size}"
+                            )));
+                        }
+                        Some(a_size)
+                    }
+                    (Some(a_size), None) => Some(a_size),
+                    (None, Some(b_size)) => Some(b_size),
+                    (None, None) => None,
+                };
+                Ok(Self::Partial { size })
+            }
+        }
+    }
+}
+
+impl redb::Value for EntryState {
+    type SelfType<'a> = EntryState;
+
+    type AsBytes<'a> = SmallVec<[u8; 128]>;
+
+    fn fixed_width() -> Option<usize> {
+        None
+    }
+
+    fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a>
+    where
+        Self: 'a,
+    {
+        postcard::from_bytes(data).unwrap()
+    }
+
+    fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a>
+    where
+        Self: 'a,
+        Self: 'b,
+    {
+        postcard::to_extend(value, SmallVec::new()).unwrap()
+    }
+
+    fn type_name() -> redb::TypeName {
+        redb::TypeName::new("EntryState")
+    }
+}
diff --git a/src/store/fs/gc.rs b/src/store/fs/gc.rs
new file mode 100644
index 000000000..4aa16c141
--- /dev/null
+++ b/src/store/fs/gc.rs
@@ -0,0 +1,329 @@
+use std::collections::HashSet;
+
+use bao_tree::ChunkRanges;
+use genawaiter::sync::{Co, Gen};
+use n0_future::{Stream, StreamExt};
+use tracing::{debug, error, warn};
+
+use crate::{api::Store, Hash, HashAndFormat};
+
+/// An event related to GC
+#[derive(Debug)]
+pub enum GcMarkEvent {
+    /// A custom event (info)
+    CustomDebug(String),
+    /// A custom non critical error
+    CustomWarning(String, Option<crate::api::Error>),
+    /// An unrecoverable error during GC
+    Error(crate::api::Error),
+}
+
+/// An event related to GC
+#[derive(Debug)]
+pub enum GcSweepEvent {
+    /// A custom event (debug)
+    CustomDebug(String),
+    /// A custom non critical error
+    #[allow(dead_code)]
+    CustomWarning(String, Option<crate::api::Error>),
+    /// An unrecoverable error during GC
+    Error(crate::api::Error),
+}
+
+/// Compute the set of live hashes
+pub(super) async fn gc_mark_task(
+    store: &Store,
+    live: &mut HashSet<Hash>,
+    co: &Co<GcMarkEvent>,
+) -> crate::api::Result<()> {
+    macro_rules! trace {
+        ($($arg:tt)*) => {
+            co.yield_(GcMarkEvent::CustomDebug(format!($($arg)*))).await;
+        };
+    }
+    macro_rules! warn {
+        ($($arg:tt)*) => {
+            co.yield_(GcMarkEvent::CustomWarning(format!($($arg)*), None)).await;
+        };
+    }
+    let mut roots = HashSet::new();
+    trace!("traversing tags");
+    let mut tags = store.tags().list().await?;
+    while let Some(tag) = tags.next().await {
+        let info = tag?;
+        trace!("adding root {:?} {:?}", info.name, info.hash_and_format());
+        roots.insert(info.hash_and_format());
+    }
+    trace!("traversing temp roots");
+    let mut tts = store.tags().list_temp_tags().await?;
+    while let Some(tt) = tts.next().await {
+        trace!("adding temp root {:?}", tt);
+        roots.insert(tt);
+    }
+    for HashAndFormat { hash, format } in roots {
+        // we need to do this for all formats except raw
+        if live.insert(hash) && !format.is_raw() {
+            let mut stream = store.export_bao(hash, ChunkRanges::all()).hashes();
+            while let Some(hash) = stream.next().await {
+                match hash {
+                    Ok(hash) => {
+                        live.insert(hash);
+                    }
+                    Err(e) => {
+                        warn!("error while traversing hashseq: {e:?}");
+                    }
+                }
+            }
+        }
+    }
+    trace!("gc mark done. found {} live blobs", live.len());
+    Ok(())
+}
+
+async fn gc_sweep_task(
+    store: &Store,
+    live: &HashSet<Hash>,
+    co: &Co<GcSweepEvent>,
+) -> crate::api::Result<()> {
+    let mut blobs = store.blobs().list().stream().await?;
+    let mut count = 0;
+    let mut batch = Vec::new();
+    while let Some(hash) = blobs.next().await {
+        let hash = hash?;
+        if !live.contains(&hash) {
+            batch.push(hash);
+            count += 1;
+        }
+        if batch.len() >= 100 {
+            store.blobs().delete(batch.clone()).await?;
+            batch.clear();
+        }
+    }
+    if !batch.is_empty() {
+        store.blobs().delete(batch).await?;
+    }
+    store.sync_db().await?;
+    co.yield_(GcSweepEvent::CustomDebug(format!("deleted {count} blobs")))
+        .await;
+    Ok(())
+}
+
+fn gc_mark<'a>(
+    store: &'a Store,
+    live: &'a mut HashSet<Hash>,
+) -> impl Stream<Item = GcMarkEvent> + 'a {
+    Gen::new(|co| async move {
+        if let Err(e) = gc_mark_task(store, live, &co).await {
+            co.yield_(GcMarkEvent::Error(e)).await;
+        }
+    })
+}
+
+fn gc_sweep<'a>(
+    store: &'a Store,
+    live: &'a HashSet<Hash>,
+) -> impl Stream<Item = GcSweepEvent> + 'a {
+    Gen::new(|co| async move {
+        if let Err(e) = gc_sweep_task(store, live, &co).await {
+            co.yield_(GcSweepEvent::Error(e)).await;
+        }
+    })
+}
+
+#[derive(Debug, Clone)]
+pub struct GcConfig {
+    pub interval: std::time::Duration,
+}
+
+pub async fn gc_run_once(store: &Store, live: &mut HashSet<Hash>) -> crate::api::Result<()> {
+    {
+        live.clear();
+        store.clear_protected().await?;
+        let mut stream = gc_mark(store, live);
+        while let Some(ev) = stream.next().await {
+            match ev {
+                GcMarkEvent::CustomDebug(msg) => {
+                    debug!("{}", msg);
+                }
+                GcMarkEvent::CustomWarning(msg, err) => {
+                    warn!("{}: {:?}", msg, err);
+                }
+                GcMarkEvent::Error(err) => {
+                    error!("error during gc mark: {:?}", err);
+                    return Err(err);
+                }
+            }
+        }
+    }
+    {
+        let mut stream = gc_sweep(store, live);
+        while let Some(ev) = stream.next().await {
+            match ev {
+                GcSweepEvent::CustomDebug(msg) => {
+                    debug!("{}", msg);
+                }
+                GcSweepEvent::CustomWarning(msg, err) => {
+                    warn!("{}: {:?}", msg, err);
+                }
+                GcSweepEvent::Error(err) => {
+                    error!("error during gc sweep: {:?}", err);
+                    return Err(err);
+                }
+            }
+        }
+    }
+
+    Ok(())
+}
+
+pub async fn run_gc(store: Store, config: GcConfig) {
+    let mut live = HashSet::new();
+    loop {
+        tokio::time::sleep(config.interval).await;
+        if let Err(e) = gc_run_once(&store, &mut live).await {
+            error!("error during gc run: {e}");
+            break;
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::path::Path;
+
+    use bao_tree::ChunkNum;
+    use testresult::TestResult;
+
+    use super::*;
+    use crate::{
+        api::{blobs::AddBytesOptions, Store},
+        hashseq::HashSeq,
+        store::fs::{options::PathOptions, tests::create_n0_bao},
+        BlobFormat,
+    };
+
+    async fn gc_smoke(store: &Store) -> TestResult<()> {
+        let blobs = store.blobs();
+        let at = blobs.add_slice("a").temp_tag().await?;
+        let bt = blobs.add_slice("b").temp_tag().await?;
+        let ct = blobs.add_slice("c").temp_tag().await?;
+        let dt = blobs.add_slice("d").temp_tag().await?;
+        let et = blobs.add_slice("e").temp_tag().await?;
+        let ft = blobs.add_slice("f").temp_tag().await?;
+        let gt = blobs.add_slice("g").temp_tag().await?;
+        let a = *at.hash();
+        let b = *bt.hash();
+        let c = *ct.hash();
+        let d = *dt.hash();
+        let e = *et.hash();
+        let f = *ft.hash();
+        let g = *gt.hash();
+        store.tags().set("c", *ct.hash_and_format()).await?;
+        let dehs = [d, e].into_iter().collect::<HashSeq>();
+        let hehs = blobs
+            .add_bytes_with_opts(AddBytesOptions {
+                data: dehs.into(),
+                format: BlobFormat::HashSeq,
+            })
+            .await?;
+        let fghs = [f, g].into_iter().collect::<HashSeq>();
+        let fghs = blobs
+            .add_bytes_with_opts(AddBytesOptions {
+                data: fghs.into(),
+                format: BlobFormat::HashSeq,
+            })
+            .temp_tag()
+            .await?;
+        store.tags().set("fg", *fghs.hash_and_format()).await?;
+        drop(fghs);
+        drop(bt);
+        let mut live = HashSet::new();
+        gc_run_once(store, &mut live).await?;
+        // a is protected because we keep the temp tag
+        assert!(live.contains(&a));
+        assert!(store.has(a).await?);
+        // b is not protected because we drop the temp tag
+        assert!(!live.contains(&b));
+        assert!(!store.has(b).await?);
+        // c is protected because we set an explicit tag
+        assert!(live.contains(&c));
+        assert!(store.has(c).await?);
+        // d and e are protected because they are part of a hashseq protected by a temp tag
+        assert!(live.contains(&d));
+        assert!(store.has(d).await?);
+        assert!(live.contains(&e));
+        assert!(store.has(e).await?);
+        // f and g are protected because they are part of a hashseq protected by a tag
+        assert!(live.contains(&f));
+        assert!(store.has(f).await?);
+        assert!(live.contains(&g));
+        assert!(store.has(g).await?);
+        drop(at);
+        drop(hehs);
+        Ok(())
+    }
+
+    async fn gc_file_delete(path: &Path, store: &Store) -> TestResult<()> {
+        let mut live = HashSet::new();
+        let options = PathOptions::new(&path.join("db"));
+        // create a large complete file and check that the data and outboard files are deleted by gc
+        {
+            let a = store
+                .blobs()
+                .add_slice(vec![0u8; 8000000])
+                .temp_tag()
+                .await?;
+            let ah = a.hash();
+            let data_path = options.data_path(ah);
+            let outboard_path = options.outboard_path(ah);
+            assert!(data_path.exists());
+            assert!(outboard_path.exists());
+            assert!(store.has(*ah).await?);
+            drop(a);
+            gc_run_once(store, &mut live).await?;
+            assert!(!data_path.exists());
+            assert!(!outboard_path.exists());
+        }
+        // create a large partial file and check that the data and outboard file as well as
+        // the sizes and bitfield files are deleted by gc
+        {
+            let data = vec![1u8; 8000000];
+            let ranges = ChunkRanges::from(..ChunkNum(19));
+            let (bh, b_bao) = create_n0_bao(&data, &ranges)?;
+            store.import_bao_bytes(bh, ranges, b_bao).await?;
+            let data_path = options.data_path(&bh);
+            let outboard_path = options.outboard_path(&bh);
+            let sizes_path = options.sizes_path(&bh);
+            let bitfield_path = options.bitfield_path(&bh);
+            assert!(data_path.exists());
+            assert!(outboard_path.exists());
+            assert!(sizes_path.exists());
+            assert!(bitfield_path.exists());
+            gc_run_once(store, &mut live).await?;
+            assert!(!data_path.exists());
+            assert!(!outboard_path.exists());
+            assert!(!sizes_path.exists());
+            assert!(!bitfield_path.exists());
+        }
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn gc_smoke_fs() -> TestResult {
+        tracing_subscriber::fmt::try_init().ok();
+        let testdir = tempfile::tempdir()?;
+        let db_path = testdir.path().join("db");
+        let store = crate::store::fs::FsStore::load(&db_path).await?;
+        gc_smoke(&store).await?;
+        gc_file_delete(testdir.path(), &store).await?;
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn gc_smoke_mem() -> TestResult {
+        tracing_subscriber::fmt::try_init().ok();
+        let store = crate::store::mem::MemStore::new();
+        gc_smoke(&store).await?;
+        Ok(())
+    }
+}
diff --git a/src/store/fs/import.rs b/src/store/fs/import.rs
new file mode 100644
index 000000000..1502ffec5
--- /dev/null
+++ b/src/store/fs/import.rs
@@ -0,0 +1,636 @@
+//! The entire logic for importing data from three different sources: bytes, byte stream, and file path.
+//!
+//! For bytes, the size is known in advance. But they might still have to be persisted to disk if they are too large.
+//! E.g. you add a 1GB bytes to the store, you still want this to end up on disk.
+//!
+//! For byte streams, the size is not known in advance.
+//!
+//! For file paths, the size is known in advance. This is also the only case where we might reference the data instead
+//! of copying it.
+//!
+//! The various ..._task fns return an `Option<ImportEntry>`. If import fails for whatever reason, the error goes
+//! to the requester, and the task returns None.
+use std::{
+    fmt,
+    fs::{self, File, OpenOptions},
+    io::{self, Seek, Write},
+    path::PathBuf,
+    sync::Arc,
+};
+
+use bao_tree::{
+    io::outboard::{PreOrderMemOutboard, PreOrderOutboard},
+    BaoTree, ChunkNum,
+};
+use bytes::Bytes;
+use genawaiter::sync::Gen;
+use irpc::{
+    channel::{mpsc, none::NoReceiver},
+    Channels, WithChannels,
+};
+use n0_future::{stream, Stream, StreamExt};
+use ref_cast::RefCast;
+use smallvec::SmallVec;
+use tracing::{instrument, trace};
+
+use super::{meta::raw_outboard_size, options::Options, TaskContext};
+use crate::{
+    api::{
+        blobs::{AddProgressItem, ImportMode},
+        proto::{
+            HashSpecific, ImportByteStreamMsg, ImportByteStreamRequest, ImportByteStreamUpdate,
+            ImportBytesMsg, ImportBytesRequest, ImportPathMsg, ImportPathRequest, Scope,
+            StoreService,
+        },
+    },
+    store::{
+        util::{MemOrFile, DD},
+        IROH_BLOCK_SIZE,
+    },
+    util::{outboard_with_progress::init_outboard, sink::Sink},
+    BlobFormat, Hash,
+};
+
+/// An import source.
+///
+/// It must provide a way to read the data synchronously, as well as the size
+/// and the file location.
+///
+/// This serves as an intermediate result between copying and outboard computation.
+pub enum ImportSource {
+    TempFile(PathBuf, File, u64),
+    External(PathBuf, File, u64),
+    Memory(Bytes),
+}
+
+impl std::fmt::Debug for ImportSource {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::TempFile(path, _, size) => {
+                f.debug_tuple("TempFile").field(path).field(size).finish()
+            }
+            Self::External(path, _, size) => {
+                f.debug_tuple("External").field(path).field(size).finish()
+            }
+            Self::Memory(data) => f.debug_tuple("Memory").field(&data.len()).finish(),
+        }
+    }
+}
+
+impl ImportSource {
+    pub fn fmt_short(&self) -> String {
+        match self {
+            Self::TempFile(path, _, _) => format!("TempFile({})", path.display()),
+            Self::External(path, _, _) => format!("External({})", path.display()),
+            Self::Memory(data) => format!("Memory({})", data.len()),
+        }
+    }
+
+    fn is_mem(&self) -> bool {
+        matches!(self, Self::Memory(_))
+    }
+
+    /// A reader for the import source.
+    fn read(&self) -> MemOrFile<std::io::Cursor<&[u8]>, &File> {
+        match self {
+            Self::TempFile(_, file, _) => MemOrFile::File(file),
+            Self::External(_, file, _) => MemOrFile::File(file),
+            Self::Memory(data) => MemOrFile::Mem(std::io::Cursor::new(data.as_ref())),
+        }
+    }
+
+    /// The size of the import source.
+    fn size(&self) -> u64 {
+        match self {
+            Self::TempFile(_, _, size) => *size,
+            Self::External(_, _, size) => *size,
+            Self::Memory(data) => data.len() as u64,
+        }
+    }
+}
+
+/// An import entry.
+///
+/// This is the final result of an import operation. It gets passed to the store
+/// for integration.
+///
+/// The store can assume that the outboard, if on disk, is in a location where
+/// it can be moved to the final location (basically it needs to be on the same device).
+pub struct ImportEntry {
+    pub hash: Hash,
+    pub format: BlobFormat,
+    pub scope: Scope,
+    pub source: ImportSource,
+    pub outboard: MemOrFile<Bytes, PathBuf>,
+}
+
+impl std::fmt::Debug for ImportEntry {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("ImportEntry")
+            .field("hash", &self.hash)
+            .field("format", &self.format)
+            .field("scope", &self.scope)
+            .field("source", &DD(self.source.fmt_short()))
+            .field("outboard", &DD(self.outboard.fmt_short()))
+            .finish()
+    }
+}
+
+impl Channels<StoreService> for ImportEntry {
+    type Tx = mpsc::Sender<AddProgressItem>;
+    type Rx = NoReceiver;
+}
+
+pub type ImportEntryMsg = WithChannels<ImportEntry, StoreService>;
+
+impl HashSpecific for ImportEntryMsg {
+    fn hash(&self) -> Hash {
+        self.hash
+    }
+}
+
+impl ImportEntry {
+    /// True if both data and outboard are in memory.
+    pub fn is_mem(&self) -> bool {
+        self.source.is_mem() && self.outboard.is_mem()
+    }
+}
+
+/// Start a task to import from a [`Bytes`] in memory.
+#[instrument(skip_all, fields(data = cmd.data.len()))]
+pub async fn import_bytes(cmd: ImportBytesMsg, ctx: Arc<TaskContext>) {
+    let size = cmd.data.len() as u64;
+    if ctx.options.is_inlined_all(size) {
+        import_bytes_tiny_outer(cmd, ctx).await;
+    } else {
+        let request = ImportByteStreamRequest {
+            format: cmd.format,
+            scope: cmd.scope,
+        };
+        let stream = stream::iter(Some(Ok(cmd.data.clone())));
+        import_byte_stream_mid(request, cmd.tx, cmd.span, stream, ctx).await;
+    }
+}
+
+async fn import_bytes_tiny_outer(mut cmd: ImportBytesMsg, ctx: Arc<TaskContext>) {
+    match import_bytes_tiny_impl(cmd.inner, &mut cmd.tx).await {
+        Ok(entry) => {
+            let entry = ImportEntryMsg {
+                inner: entry,
+                tx: cmd.tx,
+                rx: cmd.rx,
+                span: cmd.span,
+            };
+            ctx.internal_cmd_tx.send(entry.into()).await.ok();
+        }
+        Err(cause) => {
+            cmd.tx.send(cause.into()).await.ok();
+        }
+    }
+}
+
+async fn import_bytes_tiny_impl(
+    cmd: ImportBytesRequest,
+    tx: &mut mpsc::Sender<AddProgressItem>,
+) -> io::Result<ImportEntry> {
+    let size = cmd.data.len() as u64;
+    // send the required progress events
+    // AddProgressItem::Done will be sent when finishing the import!
+    tx.send(AddProgressItem::Size(size))
+        .await
+        .map_err(|_e| io::Error::other("error"))?;
+    tx.send(AddProgressItem::CopyDone)
+        .await
+        .map_err(|_e| io::Error::other("error"))?;
+    Ok(if raw_outboard_size(size) == 0 {
+        // the thing is so small that it does not even need an outboard
+        ImportEntry {
+            hash: Hash::new(&cmd.data),
+            format: cmd.format,
+            scope: cmd.scope,
+            source: ImportSource::Memory(cmd.data),
+            outboard: MemOrFile::empty(),
+        }
+    } else {
+        // we still know that computing the outboard will be super fast
+        let outboard = PreOrderMemOutboard::create(&cmd.data, IROH_BLOCK_SIZE);
+        ImportEntry {
+            hash: outboard.root.into(),
+            format: cmd.format,
+            scope: cmd.scope,
+            source: ImportSource::Memory(cmd.data),
+            outboard: MemOrFile::Mem(Bytes::from(outboard.data)),
+        }
+    })
+}
+
+#[instrument(skip_all)]
+pub async fn import_byte_stream(cmd: ImportByteStreamMsg, ctx: Arc<TaskContext>) {
+    let stream = into_stream(cmd.rx);
+    import_byte_stream_mid(cmd.inner, cmd.tx, cmd.span, stream, ctx).await
+}
+
+fn into_stream(
+    mut rx: mpsc::Receiver<ImportByteStreamUpdate>,
+) -> impl Stream<Item = io::Result<Bytes>> {
+    Gen::new(|co| async move {
+        loop {
+            match rx.recv().await {
+                Ok(Some(ImportByteStreamUpdate::Bytes(data))) => {
+                    co.yield_(Ok(data)).await;
+                }
+                Ok(Some(ImportByteStreamUpdate::Done)) => {
+                    break;
+                }
+                Ok(None) => {
+                    co.yield_(Err(io::ErrorKind::UnexpectedEof.into())).await;
+                    break;
+                }
+                Err(e) => {
+                    co.yield_(Err(e.into())).await;
+                    break;
+                }
+            }
+        }
+    })
+}
+
+async fn import_byte_stream_mid(
+    request: ImportByteStreamRequest,
+    mut tx: mpsc::Sender<AddProgressItem>,
+    span: tracing::Span,
+    stream: impl Stream<Item = io::Result<Bytes>> + Unpin,
+    ctx: Arc<TaskContext>,
+) {
+    match import_byte_stream_impl(request, &mut tx, stream, ctx.options.clone()).await {
+        Ok(entry) => {
+            let entry = ImportEntryMsg {
+                inner: entry,
+                tx,
+                rx: NoReceiver,
+                span,
+            };
+            ctx.internal_cmd_tx.send(entry.into()).await.ok();
+        }
+        Err(cause) => {
+            tx.send(cause.into()).await.ok();
+        }
+    }
+}
+
+async fn import_byte_stream_impl(
+    cmd: ImportByteStreamRequest,
+    tx: &mut mpsc::Sender<AddProgressItem>,
+    stream: impl Stream<Item = io::Result<Bytes>> + Unpin,
+    options: Arc<Options>,
+) -> io::Result<ImportEntry> {
+    let ImportByteStreamRequest { format, scope } = cmd;
+    let import_source = get_import_source(stream, tx, &options).await?;
+    tx.send(AddProgressItem::Size(import_source.size()))
+        .await
+        .map_err(|_e| io::Error::other("error"))?;
+    tx.send(AddProgressItem::CopyDone)
+        .await
+        .map_err(|_e| io::Error::other("error"))?;
+    compute_outboard(import_source, format, scope, options, tx).await
+}
+
+async fn get_import_source(
+    stream: impl Stream<Item = io::Result<Bytes>> + Unpin,
+    tx: &mut mpsc::Sender<AddProgressItem>,
+    options: &Options,
+) -> io::Result<ImportSource> {
+    let mut stream = stream.fuse();
+    let mut peek = SmallVec::<[_; 2]>::new();
+    let Some(first) = stream.next().await.transpose()? else {
+        return Ok(ImportSource::Memory(Bytes::new()));
+    };
+    match stream.next().await.transpose()? {
+        Some(second) => {
+            peek.push(Ok(first));
+            peek.push(Ok(second));
+        }
+        None => {
+            let size = first.len() as u64;
+            if options.is_inlined_data(size) {
+                return Ok(ImportSource::Memory(first));
+            }
+            peek.push(Ok(first));
+        }
+    };
+    // todo: if both first and second are giant, we might want to write them to disk immediately
+    let mut stream = stream::iter(peek).chain(stream);
+    let mut size = 0;
+    let mut data = Vec::new();
+    let mut disk = None;
+    while let Some(chunk) = stream.next().await {
+        let chunk = chunk?;
+        size += chunk.len() as u64;
+        if size > options.inline.max_data_inlined {
+            let temp_path = options.path.temp_file_name();
+            trace!("writing to temp file: {:?}", temp_path);
+            let mut file = fs::OpenOptions::new()
+                .read(true)
+                .write(true)
+                .create(true)
+                .truncate(true)
+                .open(&temp_path)?;
+            file.write_all(&data)?;
+            file.write_all(&chunk)?;
+            data.clear();
+            disk = Some((file, temp_path));
+            break;
+        } else {
+            data.extend_from_slice(&chunk);
+        }
+        // todo: don't send progress for every chunk if the chunks are small?
+        tx.try_send(AddProgressItem::CopyProgress(size))
+            .await
+            .map_err(|_e| io::Error::other("error"))?;
+    }
+    Ok(if let Some((mut file, temp_path)) = disk {
+        while let Some(chunk) = stream.next().await {
+            let chunk = chunk?;
+            file.write_all(&chunk)?;
+            size += chunk.len() as u64;
+            tx.send(AddProgressItem::CopyProgress(size))
+                .await
+                .map_err(|_e| io::Error::other("error"))?;
+        }
+        ImportSource::TempFile(temp_path, file, size)
+    } else {
+        ImportSource::Memory(data.into())
+    })
+}
+
+#[derive(ref_cast::RefCast)]
+#[repr(transparent)]
+struct OutboardProgress(mpsc::Sender<AddProgressItem>);
+
+impl Sink<ChunkNum> for OutboardProgress {
+    type Error = irpc::channel::SendError;
+
+    async fn send(&mut self, offset: ChunkNum) -> std::result::Result<(), Self::Error> {
+        // if offset.0 % 1024 != 0 {
+        //     return Ok(());
+        // }
+        self.0
+            .try_send(AddProgressItem::OutboardProgress(offset.to_bytes()))
+            .await?;
+        Ok(())
+    }
+}
+
+async fn compute_outboard(
+    source: ImportSource,
+    format: BlobFormat,
+    scope: Scope,
+    options: Arc<Options>,
+    tx: &mut mpsc::Sender<AddProgressItem>,
+) -> io::Result<ImportEntry> {
+    let size = source.size();
+    let tree = BaoTree::new(size, IROH_BLOCK_SIZE);
+    let root = bao_tree::blake3::Hash::from_bytes([0; 32]);
+    let outboard_size = raw_outboard_size(size);
+    let send_progress = OutboardProgress::ref_cast_mut(tx);
+    let mut data = source.read();
+    data.rewind()?;
+    let (hash, outboard) = if outboard_size > options.inline.max_outboard_inlined {
+        // outboard will eventually be stored as a file, so compute it directly to a file
+        // we don't know the hash yet, so we need to create a temp file
+        let outboard_path = options.path.temp_file_name();
+        trace!("Creating outboard file in {}", outboard_path.display());
+        // we don't need to read from this file!
+        let mut outboard_file = File::create(&outboard_path)?;
+        let mut outboard = PreOrderOutboard {
+            tree,
+            root,
+            data: &mut outboard_file,
+        };
+        init_outboard(data, &mut outboard, send_progress).await??;
+        (outboard.root, MemOrFile::File(outboard_path))
+    } else {
+        // outboard will be stored in memory, so compute it to a memory buffer
+        trace!("Creating outboard in memory");
+        let mut outboard_file: Vec<u8> = Vec::new();
+        let mut outboard = PreOrderOutboard {
+            tree,
+            root,
+            data: &mut outboard_file,
+        };
+        init_outboard(data, &mut outboard, send_progress).await??;
+        (outboard.root, MemOrFile::Mem(Bytes::from(outboard_file)))
+    };
+    Ok(ImportEntry {
+        hash: hash.into(),
+        format,
+        scope,
+        source,
+        outboard,
+    })
+}
+
+#[instrument(skip_all, fields(path = %cmd.path.display()))]
+pub async fn import_path(mut cmd: ImportPathMsg, context: Arc<TaskContext>) {
+    match import_path_impl(cmd.inner, &mut cmd.tx, context.options.clone()).await {
+        Ok(inner) => {
+            let res = ImportEntryMsg {
+                inner,
+                tx: cmd.tx,
+                rx: cmd.rx,
+                span: cmd.span,
+            };
+            context.internal_cmd_tx.send(res.into()).await.ok();
+        }
+        Err(cause) => {
+            cmd.tx.send(cause.into()).await.ok();
+        }
+    }
+}
+
+async fn import_path_impl(
+    cmd: ImportPathRequest,
+    tx: &mut mpsc::Sender<AddProgressItem>,
+    options: Arc<Options>,
+) -> io::Result<ImportEntry> {
+    let ImportPathRequest {
+        path,
+        mode,
+        format,
+        scope: batch,
+    } = cmd;
+    if !path.is_absolute() {
+        return Err(io::Error::new(
+            io::ErrorKind::InvalidInput,
+            "path must be absolute",
+        ));
+    }
+    if !path.is_file() && !path.is_symlink() {
+        return Err(io::Error::new(
+            io::ErrorKind::InvalidInput,
+            "path is not a file or symlink",
+        ));
+    }
+
+    let size = path.metadata()?.len();
+    tx.send(AddProgressItem::Size(size))
+        .await
+        .map_err(|_e| io::Error::other("error"))?;
+    let import_source = if size <= options.inline.max_data_inlined {
+        let data = std::fs::read(path)?;
+        tx.send(AddProgressItem::CopyDone)
+            .await
+            .map_err(|_e| io::Error::other("error"))?;
+        ImportSource::Memory(data.into())
+    } else if mode == ImportMode::TryReference {
+        // reference where it is. We are going to need the file handle to
+        // compute the outboard, so open it here. If this fails, the import
+        // can't proceed.
+        let file = OpenOptions::new().read(true).open(&path)?;
+        ImportSource::External(path, file, size)
+    } else {
+        let temp_path = options.path.temp_file_name();
+        // todo: if reflink works, we don't need progress.
+        // But if it does not, it might take a while and we won't get progress.
+        if reflink_copy::reflink_or_copy(&path, &temp_path)?.is_none() {
+            trace!("reflinked {} to {}", path.display(), temp_path.display());
+        } else {
+            trace!("copied {} to {}", path.display(), temp_path.display());
+        }
+        // copy from path to temp_path
+        let file = OpenOptions::new().read(true).open(&temp_path)?;
+        tx.send(AddProgressItem::CopyDone)
+            .await
+            .map_err(|_| io::Error::other("error"))?;
+        ImportSource::TempFile(temp_path, file, size)
+    };
+    compute_outboard(import_source, format, batch, options, tx).await
+}
+
+#[cfg(test)]
+mod tests {
+
+    use bao_tree::io::outboard::PreOrderMemOutboard;
+    use irpc::RpcMessage;
+    use n0_future::stream;
+    use testresult::TestResult;
+
+    use super::*;
+    use crate::{
+        api::proto::BoxedByteStream,
+        store::fs::options::{InlineOptions, PathOptions},
+    };
+
+    async fn drain<T: RpcMessage>(mut recv: mpsc::Receiver<T>) -> TestResult<Vec<T>> {
+        let mut res = Vec::new();
+        while let Some(item) = recv.recv().await? {
+            res.push(item);
+        }
+        Ok(res)
+    }
+
+    fn assert_expected_progress(progress: &[AddProgressItem]) {
+        assert!(progress
+            .iter()
+            .any(|x| matches!(&x, AddProgressItem::Size { .. })));
+        assert!(progress
+            .iter()
+            .any(|x| matches!(&x, AddProgressItem::CopyDone)));
+    }
+
+    fn chunk_bytes(data: Bytes, chunk_size: usize) -> impl Iterator<Item = Bytes> {
+        assert!(chunk_size > 0, "Chunk size must be positive");
+        (0..data.len())
+            .step_by(chunk_size)
+            .map(move |i| data.slice(i..std::cmp::min(i + chunk_size, data.len())))
+    }
+
+    async fn test_import_byte_stream_task(data: Bytes, options: Arc<Options>) -> TestResult<()> {
+        let stream: BoxedByteStream =
+            Box::pin(stream::iter(chunk_bytes(data.clone(), 999).map(Ok)));
+        let expected_outboard = PreOrderMemOutboard::create(data.as_ref(), IROH_BLOCK_SIZE);
+        // make the channel absurdly large, so we don't have to drain it
+        let (mut tx, rx) = mpsc::channel(1024 * 1024);
+        let data = stream.collect::<Vec<_>>().await;
+        let data = data.into_iter().collect::<io::Result<Vec<_>>>()?;
+        let cmd = ImportByteStreamRequest {
+            format: BlobFormat::Raw,
+            scope: Default::default(),
+        };
+        let stream = stream::iter(data.into_iter().map(Ok));
+        let res = import_byte_stream_impl(cmd, &mut tx, stream, options).await;
+        let Ok(res) = res else {
+            panic!("import failed");
+        };
+        let ImportEntry { outboard, .. } = res;
+        drop(tx);
+        let actual_outboard = match &outboard {
+            MemOrFile::Mem(data) => data.clone(),
+            MemOrFile::File(path) => std::fs::read(path)?.into(),
+        };
+        assert_eq!(expected_outboard.data.as_slice(), actual_outboard.as_ref());
+        let progress = drain(rx).await?;
+        assert_expected_progress(&progress);
+        Ok(())
+    }
+
+    async fn test_import_file_task(data: Bytes, options: Arc<Options>) -> TestResult<()> {
+        let path = options.path.temp_file_name();
+        std::fs::write(&path, &data)?;
+        let expected_outboard = PreOrderMemOutboard::create(data.as_ref(), IROH_BLOCK_SIZE);
+        // make the channel absurdly large, so we don't have to drain it
+        let (mut tx, rx) = mpsc::channel(1024 * 1024);
+        let cmd = ImportPathRequest {
+            path,
+            mode: ImportMode::Copy,
+            format: BlobFormat::Raw,
+            scope: Scope::default(),
+        };
+        let res = import_path_impl(cmd, &mut tx, options).await;
+        let Ok(res) = res else {
+            panic!("import failed");
+        };
+        let ImportEntry { outboard, .. } = res;
+        drop(tx);
+        let actual_outboard = match &outboard {
+            MemOrFile::Mem(data) => data.clone(),
+            MemOrFile::File(path) => std::fs::read(path)?.into(),
+        };
+        assert_eq!(expected_outboard.data.as_slice(), actual_outboard.as_ref());
+        let progress = drain(rx).await?;
+        assert_expected_progress(&progress);
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn smoke() -> TestResult<()> {
+        let dir = tempfile::tempdir()?;
+        std::fs::create_dir_all(dir.path().join("data"))?;
+        std::fs::create_dir_all(dir.path().join("temp"))?;
+        let options = Arc::new(Options {
+            inline: InlineOptions {
+                max_data_inlined: 1024 * 16,
+                max_outboard_inlined: 1024 * 16,
+            },
+            batch: Default::default(),
+            path: PathOptions::new(dir.path()),
+            gc: None,
+        });
+        // test different sizes, below, at, and above the inline threshold
+        let sizes = [
+            0,               // empty, no outboard
+            1024,            // data in mem, no outboard
+            1024 * 16 - 1,   // data in mem, no outboard
+            1024 * 16,       // data in mem, no outboard
+            1024 * 16 + 1,   // data in file, outboard in mem
+            1024 * 1024,     // data in file, outboard in mem
+            1024 * 1024 * 8, // data in file, outboard in file
+        ];
+        for size in sizes {
+            let data = Bytes::from(vec![0; size]);
+            test_import_byte_stream_task(data.clone(), options.clone()).await?;
+            test_import_file_task(data, options.clone()).await?;
+        }
+        Ok(())
+    }
+}
diff --git a/src/store/fs/meta.rs b/src/store/fs/meta.rs
new file mode 100644
index 000000000..a08314104
--- /dev/null
+++ b/src/store/fs/meta.rs
@@ -0,0 +1,839 @@
+//! The metadata database
+#![allow(clippy::result_large_err)]
+use std::{
+    collections::HashSet,
+    io,
+    ops::{Bound, Deref, DerefMut},
+    path::PathBuf,
+    time::SystemTime,
+};
+
+use bao_tree::BaoTree;
+use bytes::Bytes;
+use irpc::channel::mpsc;
+use n0_snafu::SpanTrace;
+use nested_enum_utils::common_fields;
+use redb::{Database, DatabaseError, ReadableTable};
+use snafu::{Backtrace, ResultExt, Snafu};
+use tokio::pin;
+
+use crate::{
+    api::{
+        self,
+        blobs::BlobStatus,
+        proto::{
+            BlobDeleteRequest, BlobStatusMsg, BlobStatusRequest, ClearProtectedMsg,
+            CreateTagRequest, DeleteBlobsMsg, DeleteTagsRequest, ListBlobsMsg, ListRequest,
+            ListTagsRequest, RenameTagRequest, SetTagRequest, ShutdownMsg, SyncDbMsg,
+        },
+        tags::TagInfo,
+    },
+    util::channel::oneshot,
+};
+mod proto;
+pub use proto::*;
+pub(crate) mod tables;
+use tables::{ReadOnlyTables, ReadableTables, Tables};
+use tracing::{debug, error, info_span, trace};
+
+use super::{
+    delete_set::DeleteHandle,
+    entry_state::{DataLocation, EntryState, OutboardLocation},
+    options::BatchOptions,
+    util::PeekableReceiver,
+    BaoFilePart,
+};
+use crate::store::{util::Tag, Hash, IROH_BLOCK_SIZE};
+
+/// Error type for message handler functions of the redb actor.
+///
+/// What can go wrong are various things with redb, as well as io errors related
+/// to files other than redb.
+#[common_fields({
+    backtrace: Option<Backtrace>,
+    #[snafu(implicit)]
+    span_trace: SpanTrace,
+})]
+#[allow(missing_docs)]
+#[non_exhaustive]
+#[derive(Debug, Snafu)]
+pub enum ActorError {
+    #[snafu(display("table error: {source}"))]
+    Table { source: redb::TableError },
+    #[snafu(display("database error: {source}"))]
+    Database { source: redb::DatabaseError },
+    #[snafu(display("transaction error: {source}"))]
+    Transaction { source: redb::TransactionError },
+    #[snafu(display("commit error: {source}"))]
+    Commit { source: redb::CommitError },
+    #[snafu(display("storage error: {source}"))]
+    Storage { source: redb::StorageError },
+    #[snafu(display("inconsistent database state: {msg}"))]
+    Inconsistent { msg: String },
+}
+
+impl From<ActorError> for io::Error {
+    fn from(e: ActorError) -> Self {
+        io::Error::other(e)
+    }
+}
+
+impl ActorError {
+    pub(super) fn inconsistent(msg: String) -> Self {
+        InconsistentSnafu { msg }.build()
+    }
+}
+
+pub type ActorResult<T> = Result<T, ActorError>;
+
+#[derive(Debug, Clone)]
+pub struct Db {
+    pub sender: tokio::sync::mpsc::Sender<Command>,
+}
+
+impl Db {
+    pub fn new(sender: tokio::sync::mpsc::Sender<Command>) -> Self {
+        Self { sender }
+    }
+
+    /// Get the entry state for a hash, if any.
+    pub async fn get(&self, hash: Hash) -> anyhow::Result<Option<EntryState<Bytes>>> {
+        let (tx, rx) = oneshot::channel();
+        self.sender
+            .send(
+                Get {
+                    hash,
+                    tx,
+                    span: tracing::Span::current(),
+                }
+                .into(),
+            )
+            .await?;
+        let res = rx.await?;
+        Ok(res.state?)
+    }
+
+    /// Send a command. This exists so the main actor can directly forward commands.
+    ///
+    /// This will fail only if the database actor is dead. In that case the main
+    /// actor should probably also shut down.
+    pub async fn send(&self, cmd: Command) -> io::Result<()> {
+        self.sender
+            .send(cmd)
+            .await
+            .map_err(|_e| io::Error::other("actor down"))?;
+        Ok(())
+    }
+}
+
+fn handle_get(cmd: Get, tables: &impl ReadableTables) -> ActorResult<()> {
+    trace!("{cmd:?}");
+    let Get { hash, tx, .. } = cmd;
+    let Some(entry) = tables.blobs().get(hash).context(StorageSnafu)? else {
+        tx.send(GetResult { state: Ok(None) });
+        return Ok(());
+    };
+    let entry = entry.value();
+    let entry = match entry {
+        EntryState::Complete {
+            data_location,
+            outboard_location,
+        } => {
+            let data_location = load_data(tables, data_location, &hash)?;
+            let outboard_location = load_outboard(tables, outboard_location, &hash)?;
+            EntryState::Complete {
+                data_location,
+                outboard_location,
+            }
+        }
+        EntryState::Partial { size } => EntryState::Partial { size },
+    };
+    tx.send(GetResult {
+        state: Ok(Some(entry)),
+    });
+    Ok(())
+}
+
+fn handle_dump(cmd: Dump, tables: &impl ReadableTables) -> ActorResult<()> {
+    trace!("{cmd:?}");
+    trace!("dumping database");
+    for e in tables.blobs().iter().context(StorageSnafu)? {
+        let (k, v) = e.context(StorageSnafu)?;
+        let k = k.value();
+        let v = v.value();
+        println!("blobs: {} -> {:?}", k.to_hex(), v);
+    }
+    for e in tables.tags().iter().context(StorageSnafu)? {
+        let (k, v) = e.context(StorageSnafu)?;
+        let k = k.value();
+        let v = v.value();
+        println!("tags: {k} -> {v:?}");
+    }
+    for e in tables.inline_data().iter().context(StorageSnafu)? {
+        let (k, v) = e.context(StorageSnafu)?;
+        let k = k.value();
+        let v = v.value();
+        println!("inline_data: {} -> {:?}", k.to_hex(), v.len());
+    }
+    for e in tables.inline_outboard().iter().context(StorageSnafu)? {
+        let (k, v) = e.context(StorageSnafu)?;
+        let k = k.value();
+        let v = v.value();
+        println!("inline_outboard: {} -> {:?}", k.to_hex(), v.len());
+    }
+    cmd.tx.send(Ok(()));
+    Ok(())
+}
+
+async fn handle_clear_protected(
+    cmd: ClearProtectedMsg,
+    protected: &mut HashSet<Hash>,
+) -> ActorResult<()> {
+    trace!("{cmd:?}");
+    protected.clear();
+    cmd.tx.send(Ok(())).await.ok();
+    Ok(())
+}
+
+async fn handle_get_blob_status(
+    msg: BlobStatusMsg,
+    tables: &impl ReadableTables,
+) -> ActorResult<()> {
+    trace!("{msg:?}");
+    let BlobStatusMsg {
+        inner: BlobStatusRequest { hash },
+        tx,
+        ..
+    } = msg;
+    let res = match tables.blobs().get(hash).context(StorageSnafu)? {
+        Some(entry) => match entry.value() {
+            EntryState::Complete { data_location, .. } => match data_location {
+                DataLocation::Inline(_) => {
+                    let Some(data) = tables.inline_data().get(hash).context(StorageSnafu)? else {
+                        return Err(ActorError::inconsistent(format!(
+                            "inconsistent database state: {} not found",
+                            hash.to_hex()
+                        )));
+                    };
+                    BlobStatus::Complete {
+                        size: data.value().len() as u64,
+                    }
+                }
+                DataLocation::Owned(size) => BlobStatus::Complete { size },
+                DataLocation::External(_, size) => BlobStatus::Complete { size },
+            },
+            EntryState::Partial { size } => BlobStatus::Partial { size },
+        },
+        None => BlobStatus::NotFound,
+    };
+    tx.send(res).await.ok();
+    Ok(())
+}
+
+async fn handle_list_tags(msg: ListTagsMsg, tables: &impl ReadableTables) -> ActorResult<()> {
+    trace!("{msg:?}");
+    let ListTagsMsg {
+        inner:
+            ListTagsRequest {
+                from,
+                to,
+                raw,
+                hash_seq,
+            },
+        tx,
+        ..
+    } = msg;
+    let from = from.map(Bound::Included).unwrap_or(Bound::Unbounded);
+    let to = to.map(Bound::Excluded).unwrap_or(Bound::Unbounded);
+    let mut res = Vec::new();
+    for item in tables.tags().range((from, to)).context(StorageSnafu)? {
+        match item {
+            Ok((k, v)) => {
+                let v = v.value();
+                if raw && v.format.is_raw() || hash_seq && v.format.is_hash_seq() {
+                    let info = TagInfo {
+                        name: k.value(),
+                        hash: v.hash,
+                        format: v.format,
+                    };
+                    res.push(crate::api::Result::Ok(info));
+                }
+            }
+            Err(e) => {
+                res.push(Err(crate::api::Error::other(e)));
+            }
+        }
+    }
+    tx.send(res).await.ok();
+    Ok(())
+}
+
+fn handle_update(
+    cmd: Update,
+    protected: &mut HashSet<Hash>,
+    tables: &mut Tables,
+) -> ActorResult<()> {
+    trace!("{cmd:?}");
+    let Update {
+        hash, state, tx, ..
+    } = cmd;
+    protected.insert(hash);
+    trace!("updating hash {} to {}", hash.to_hex(), state.fmt_short());
+    let old_entry_opt = tables
+        .blobs
+        .get(hash)
+        .context(StorageSnafu)?
+        .map(|e| e.value());
+    let (state, data, outboard): (_, Option<Bytes>, Option<Bytes>) = match state {
+        EntryState::Complete {
+            data_location,
+            outboard_location,
+        } => {
+            let (data_location, data) = data_location.split_inline_data();
+            let (outboard_location, outboard) = outboard_location.split_inline_data();
+            (
+                EntryState::Complete {
+                    data_location,
+                    outboard_location,
+                },
+                data,
+                outboard,
+            )
+        }
+        EntryState::Partial { size } => (EntryState::Partial { size }, None, None),
+    };
+    let state = match old_entry_opt {
+        Some(old) => {
+            let partial_to_complete = old.is_partial() && state.is_complete();
+            let res = EntryState::union(old, state)?;
+            if partial_to_complete {
+                tables
+                    .ftx
+                    .delete(hash, [BaoFilePart::Sizes, BaoFilePart::Bitfield]);
+            }
+            res
+        }
+        None => state,
+    };
+    tables.blobs.insert(hash, state).context(StorageSnafu)?;
+    if let Some(data) = data {
+        tables
+            .inline_data
+            .insert(hash, data.as_ref())
+            .context(StorageSnafu)?;
+    }
+    if let Some(outboard) = outboard {
+        tables
+            .inline_outboard
+            .insert(hash, outboard.as_ref())
+            .context(StorageSnafu)?;
+    }
+    if let Some(tx) = tx {
+        tx.send(Ok(()));
+    }
+    Ok(())
+}
+
+fn handle_set(cmd: Set, protected: &mut HashSet<Hash>, tables: &mut Tables) -> ActorResult<()> {
+    trace!("{cmd:?}");
+    let Set {
+        state, hash, tx, ..
+    } = cmd;
+    protected.insert(hash);
+    let (state, data, outboard): (_, Option<Bytes>, Option<Bytes>) = match state {
+        EntryState::Complete {
+            data_location,
+            outboard_location,
+        } => {
+            let (data_location, data) = data_location.split_inline_data();
+            let (outboard_location, outboard) = outboard_location.split_inline_data();
+            (
+                EntryState::Complete {
+                    data_location,
+                    outboard_location,
+                },
+                data,
+                outboard,
+            )
+        }
+        EntryState::Partial { size } => (EntryState::Partial { size }, None, None),
+    };
+    tables.blobs.insert(hash, state).context(StorageSnafu)?;
+    if let Some(data) = data {
+        tables
+            .inline_data
+            .insert(hash, data.as_ref())
+            .context(StorageSnafu)?;
+    }
+    if let Some(outboard) = outboard {
+        tables
+            .inline_outboard
+            .insert(hash, outboard.as_ref())
+            .context(StorageSnafu)?;
+    }
+    tx.send(Ok(()));
+    Ok(())
+}
+
+#[derive(Clone, Copy)]
+enum TxnNum {
+    Read(u64),
+    Write(u64),
+    TopLevel(u64),
+}
+
+impl std::fmt::Debug for TxnNum {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            TxnNum::Read(n) => write!(f, "r{n}"),
+            TxnNum::Write(n) => write!(f, "w{n}"),
+            TxnNum::TopLevel(n) => write!(f, "t{n}"),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct Actor {
+    db: redb::Database,
+    cmds: PeekableReceiver<Command>,
+    ds: DeleteHandle,
+    options: BatchOptions,
+    protected: HashSet<Hash>,
+}
+
+impl Actor {
+    pub fn new(
+        db_path: PathBuf,
+        cmds: tokio::sync::mpsc::Receiver<Command>,
+        mut ds: DeleteHandle,
+        options: BatchOptions,
+    ) -> anyhow::Result<Self> {
+        debug!("creating or opening meta database at {}", db_path.display());
+        let db = match redb::Database::create(db_path) {
+            Ok(db) => db,
+            Err(DatabaseError::UpgradeRequired(1)) => {
+                return Err(anyhow::anyhow!("migration from v1 no longer supported"));
+            }
+            Err(err) => return Err(err.into()),
+        };
+        let tx = db.begin_write()?;
+        let ftx = ds.begin_write();
+        Tables::new(&tx, &ftx)?;
+        tx.commit()?;
+        drop(ftx);
+        let cmds = PeekableReceiver::new(cmds);
+        Ok(Self {
+            db,
+            cmds,
+            ds,
+            options,
+            protected: Default::default(),
+        })
+    }
+
+    async fn handle_readonly(
+        protected: &mut HashSet<Hash>,
+        tables: &impl ReadableTables,
+        cmd: ReadOnlyCommand,
+        op: TxnNum,
+    ) -> ActorResult<()> {
+        let span = info_span!(
+            parent: &cmd.parent_span(),
+            "tx",
+            op = tracing::field::debug(op),
+        );
+        let _guard = span.enter();
+        match cmd {
+            ReadOnlyCommand::Get(cmd) => handle_get(cmd, tables),
+            ReadOnlyCommand::Dump(cmd) => handle_dump(cmd, tables),
+            ReadOnlyCommand::ListTags(cmd) => handle_list_tags(cmd, tables).await,
+            ReadOnlyCommand::ClearProtected(cmd) => handle_clear_protected(cmd, protected).await,
+            ReadOnlyCommand::GetBlobStatus(cmd) => handle_get_blob_status(cmd, tables).await,
+        }
+    }
+
+    async fn delete(
+        protected: &mut HashSet<Hash>,
+        tables: &mut Tables<'_>,
+        cmd: DeleteBlobsMsg,
+    ) -> ActorResult<()> {
+        let DeleteBlobsMsg {
+            inner: BlobDeleteRequest { hashes, force },
+            ..
+        } = cmd;
+        for hash in hashes {
+            if !force && protected.contains(&hash) {
+                continue;
+            }
+            if let Some(entry) = tables.blobs.remove(hash).context(StorageSnafu)? {
+                match entry.value() {
+                    EntryState::Complete {
+                        data_location,
+                        outboard_location,
+                    } => {
+                        match data_location {
+                            DataLocation::Inline(_) => {
+                                tables.inline_data.remove(hash).context(StorageSnafu)?;
+                            }
+                            DataLocation::Owned(_) => {
+                                // mark the data for deletion
+                                tables.ftx.delete(hash, [BaoFilePart::Data]);
+                            }
+                            DataLocation::External(_, _) => {}
+                        }
+                        match outboard_location {
+                            OutboardLocation::Inline(_) => {
+                                tables.inline_outboard.remove(hash).context(StorageSnafu)?;
+                            }
+                            OutboardLocation::Owned => {
+                                // mark the outboard for deletion
+                                tables.ftx.delete(hash, [BaoFilePart::Outboard]);
+                            }
+                            OutboardLocation::NotNeeded => {}
+                        }
+                    }
+                    EntryState::Partial { .. } => {
+                        tables.ftx.delete(
+                            hash,
+                            [
+                                BaoFilePart::Outboard,
+                                BaoFilePart::Data,
+                                BaoFilePart::Sizes,
+                                BaoFilePart::Bitfield,
+                            ],
+                        );
+                    }
+                }
+            }
+        }
+        cmd.tx.send(Ok(())).await.ok();
+        Ok(())
+    }
+
+    async fn set_tag(tables: &mut Tables<'_>, cmd: SetTagMsg) -> ActorResult<()> {
+        trace!("{cmd:?}");
+        let SetTagMsg {
+            inner: SetTagRequest { name: tag, value },
+            tx,
+            ..
+        } = cmd;
+        let res = tables.tags.insert(tag, value).map(|_| ());
+        tx.send(res.map_err(crate::api::Error::other)).await.ok();
+        Ok(())
+    }
+
+    async fn create_tag(tables: &mut Tables<'_>, cmd: CreateTagMsg) -> ActorResult<()> {
+        trace!("{cmd:?}");
+        let CreateTagMsg {
+            inner: CreateTagRequest { value },
+            tx,
+            ..
+        } = cmd;
+        let tag = {
+            let tag = Tag::auto(SystemTime::now(), |x| {
+                matches!(tables.tags.get(Tag(Bytes::copy_from_slice(x))), Ok(Some(_)))
+            });
+            tables
+                .tags
+                .insert(tag.clone(), value)
+                .context(StorageSnafu)?;
+            tag
+        };
+        tx.send(Ok(tag.clone())).await.ok();
+        Ok(())
+    }
+
+    async fn delete_tags(tables: &mut Tables<'_>, cmd: DeleteTagsMsg) -> ActorResult<()> {
+        trace!("{cmd:?}");
+        let DeleteTagsMsg {
+            inner: DeleteTagsRequest { from, to },
+            tx,
+            ..
+        } = cmd;
+        let from = from.map(Bound::Included).unwrap_or(Bound::Unbounded);
+        let to = to.map(Bound::Excluded).unwrap_or(Bound::Unbounded);
+        let removing = tables
+            .tags
+            .extract_from_if((from, to), |_, _| true)
+            .context(StorageSnafu)?;
+        // drain the iterator to actually remove the tags
+        for res in removing {
+            res.context(StorageSnafu)?;
+        }
+        tx.send(Ok(())).await.ok();
+        Ok(())
+    }
+
+    async fn rename_tag(tables: &mut Tables<'_>, cmd: RenameTagMsg) -> ActorResult<()> {
+        trace!("{cmd:?}");
+        let RenameTagMsg {
+            inner: RenameTagRequest { from, to },
+            tx,
+            ..
+        } = cmd;
+        let value = match tables.tags.remove(from).context(StorageSnafu)? {
+            Some(value) => value.value(),
+            None => {
+                tx.send(Err(api::Error::io(
+                    io::ErrorKind::NotFound,
+                    "tag not found",
+                )))
+                .await
+                .ok();
+                return Ok(());
+            }
+        };
+        tables.tags.insert(to, value).context(StorageSnafu)?;
+        tx.send(Ok(())).await.ok();
+        Ok(())
+    }
+
+    async fn handle_readwrite(
+        protected: &mut HashSet<Hash>,
+        tables: &mut Tables<'_>,
+        cmd: ReadWriteCommand,
+        op: TxnNum,
+    ) -> ActorResult<()> {
+        let span = info_span!(
+            parent: &cmd.parent_span(),
+            "tx",
+            op = tracing::field::debug(op),
+        );
+        let _guard = span.enter();
+        match cmd {
+            ReadWriteCommand::Update(cmd) => handle_update(cmd, protected, tables),
+            ReadWriteCommand::Set(cmd) => handle_set(cmd, protected, tables),
+            ReadWriteCommand::DeleteBlobw(cmd) => Self::delete(protected, tables, cmd).await,
+            ReadWriteCommand::SetTag(cmd) => Self::set_tag(tables, cmd).await,
+            ReadWriteCommand::CreateTag(cmd) => Self::create_tag(tables, cmd).await,
+            ReadWriteCommand::DeleteTags(cmd) => Self::delete_tags(tables, cmd).await,
+            ReadWriteCommand::RenameTag(cmd) => Self::rename_tag(tables, cmd).await,
+            ReadWriteCommand::ProcessExit(cmd) => {
+                std::process::exit(cmd.code);
+            }
+        }
+    }
+
+    async fn handle_non_toplevel(
+        protected: &mut HashSet<Hash>,
+        tables: &mut Tables<'_>,
+        cmd: NonTopLevelCommand,
+        op: TxnNum,
+    ) -> ActorResult<()> {
+        match cmd {
+            NonTopLevelCommand::ReadOnly(cmd) => {
+                Self::handle_readonly(protected, tables, cmd, op).await
+            }
+            NonTopLevelCommand::ReadWrite(cmd) => {
+                Self::handle_readwrite(protected, tables, cmd, op).await
+            }
+        }
+    }
+
+    async fn sync_db(_db: &mut Database, cmd: SyncDbMsg) -> ActorResult<()> {
+        trace!("{cmd:?}");
+        let SyncDbMsg { tx, .. } = cmd;
+        // nothing to do here, since for a toplevel cmd we are outside a write transaction
+        tx.send(Ok(())).await.ok();
+        Ok(())
+    }
+
+    async fn handle_toplevel(
+        db: &mut Database,
+        cmd: TopLevelCommand,
+        op: TxnNum,
+    ) -> ActorResult<Option<ShutdownMsg>> {
+        let span = info_span!(
+            parent: &cmd.parent_span(),
+            "tx",
+            op = tracing::field::debug(op),
+        );
+        let _guard = span.enter();
+        Ok(match cmd {
+            TopLevelCommand::SyncDb(cmd) => {
+                Self::sync_db(db, cmd).await?;
+                None
+            }
+            TopLevelCommand::Shutdown(cmd) => {
+                trace!("{cmd:?}");
+                // nothing to do here, since the database will be dropped
+                Some(cmd)
+            }
+            TopLevelCommand::Snapshot(cmd) => {
+                trace!("{cmd:?}");
+                let txn = db.begin_read().context(TransactionSnafu)?;
+                let snapshot = ReadOnlyTables::new(&txn).context(TableSnafu)?;
+                cmd.tx.send(snapshot).ok();
+                None
+            }
+        })
+    }
+
+    pub async fn run(mut self) -> ActorResult<()> {
+        let mut db = DbWrapper::from(self.db);
+        let options = &self.options;
+        let mut op = 0u64;
+        let shutdown = loop {
+            op += 1;
+            let Some(cmd) = self.cmds.recv().await else {
+                break None;
+            };
+            match cmd {
+                Command::TopLevel(cmd) => {
+                    let op = TxnNum::TopLevel(op);
+                    if let Some(shutdown) = Self::handle_toplevel(&mut db, cmd, op).await? {
+                        break Some(shutdown);
+                    }
+                }
+                Command::ReadOnly(cmd) => {
+                    let op = TxnNum::Read(op);
+                    self.cmds.push_back(cmd.into()).ok();
+                    let tx = db.begin_read().context(TransactionSnafu)?;
+                    let tables = ReadOnlyTables::new(&tx).context(TableSnafu)?;
+                    let timeout = tokio::time::sleep(self.options.max_read_duration);
+                    pin!(timeout);
+                    let mut n = 0;
+                    while let Some(cmd) = self.cmds.extract(Command::read_only, &mut timeout).await
+                    {
+                        Self::handle_readonly(&mut self.protected, &tables, cmd, op).await?;
+                        n += 1;
+                        if n >= options.max_read_batch {
+                            break;
+                        }
+                    }
+                }
+                Command::ReadWrite(cmd) => {
+                    let op = TxnNum::Write(op);
+                    self.cmds.push_back(cmd.into()).ok();
+                    let ftx = self.ds.begin_write();
+                    let tx = db.begin_write().context(TransactionSnafu)?;
+                    let mut tables = Tables::new(&tx, &ftx).context(TableSnafu)?;
+                    let timeout = tokio::time::sleep(self.options.max_read_duration);
+                    pin!(timeout);
+                    let mut n = 0;
+                    while let Some(cmd) = self
+                        .cmds
+                        .extract(Command::non_top_level, &mut timeout)
+                        .await
+                    {
+                        Self::handle_non_toplevel(&mut self.protected, &mut tables, cmd, op)
+                            .await?;
+                        n += 1;
+                        if n >= options.max_write_batch {
+                            break;
+                        }
+                    }
+                    drop(tables);
+                    tx.commit().context(CommitSnafu)?;
+                    ftx.commit();
+                }
+            }
+        };
+        if let Some(shutdown) = shutdown {
+            drop(db);
+            shutdown.tx.send(()).await.ok();
+        }
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+struct DbWrapper(Option<Database>);
+
+impl Deref for DbWrapper {
+    type Target = Database;
+
+    fn deref(&self) -> &Self::Target {
+        self.0.as_ref().expect("database not open")
+    }
+}
+
+impl DerefMut for DbWrapper {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self.0.as_mut().expect("database not open")
+    }
+}
+
+impl From<Database> for DbWrapper {
+    fn from(db: Database) -> Self {
+        Self(Some(db))
+    }
+}
+
+impl Drop for DbWrapper {
+    fn drop(&mut self) {
+        if let Some(db) = self.0.take() {
+            debug!("closing database");
+            drop(db);
+            debug!("database closed");
+        }
+    }
+}
+
+fn load_data(
+    tables: &impl ReadableTables,
+    location: DataLocation<(), u64>,
+    hash: &Hash,
+) -> ActorResult<DataLocation<Bytes, u64>> {
+    Ok(match location {
+        DataLocation::Inline(()) => {
+            let Some(data) = tables.inline_data().get(hash).context(StorageSnafu)? else {
+                return Err(ActorError::inconsistent(format!(
+                    "inconsistent database state: {} should have inline data but does not",
+                    hash.to_hex()
+                )));
+            };
+            DataLocation::Inline(Bytes::copy_from_slice(data.value()))
+        }
+        DataLocation::Owned(data_size) => DataLocation::Owned(data_size),
+        DataLocation::External(paths, data_size) => DataLocation::External(paths, data_size),
+    })
+}
+
+fn load_outboard(
+    tables: &impl ReadableTables,
+    location: OutboardLocation,
+    hash: &Hash,
+) -> ActorResult<OutboardLocation<Bytes>> {
+    Ok(match location {
+        OutboardLocation::NotNeeded => OutboardLocation::NotNeeded,
+        OutboardLocation::Inline(_) => {
+            let Some(outboard) = tables.inline_outboard().get(hash).context(StorageSnafu)? else {
+                return Err(ActorError::inconsistent(format!(
+                    "inconsistent database state: {} should have inline outboard but does not",
+                    hash.to_hex()
+                )));
+            };
+            OutboardLocation::Inline(Bytes::copy_from_slice(outboard.value()))
+        }
+        OutboardLocation::Owned => OutboardLocation::Owned,
+    })
+}
+
+pub(crate) fn raw_outboard_size(size: u64) -> u64 {
+    BaoTree::new(size, IROH_BLOCK_SIZE).outboard_size()
+}
+
+pub async fn list_blobs(snapshot: ReadOnlyTables, cmd: ListBlobsMsg) {
+    let ListBlobsMsg { mut tx, inner, .. } = cmd;
+    match list_blobs_impl(snapshot, inner, &mut tx).await {
+        Ok(()) => {}
+        Err(e) => {
+            error!("error listing blobs: {}", e);
+            tx.send(Err(e)).await.ok();
+        }
+    }
+}
+
+async fn list_blobs_impl(
+    snapshot: ReadOnlyTables,
+    _cmd: ListRequest,
+    tx: &mut mpsc::Sender<api::Result<Hash>>,
+) -> api::Result<()> {
+    for item in snapshot.blobs.iter().map_err(api::Error::other)? {
+        let (k, _) = item.map_err(api::Error::other)?;
+        let k = k.value();
+        tx.send(Ok(k)).await.ok();
+    }
+    Ok(())
+}
diff --git a/src/store/fs/meta/proto.rs b/src/store/fs/meta/proto.rs
new file mode 100644
index 000000000..6f4aaa6ce
--- /dev/null
+++ b/src/store/fs/meta/proto.rs
@@ -0,0 +1,228 @@
+//! Protocol for the metadata database.
+use std::fmt;
+
+use bytes::Bytes;
+use nested_enum_utils::enum_conversions;
+use tracing::Span;
+
+use super::{ActorResult, ReadOnlyTables};
+use crate::{
+    api::proto::{
+        BlobStatusMsg, ClearProtectedMsg, DeleteBlobsMsg, ProcessExitRequest, ShutdownMsg,
+        SyncDbMsg,
+    },
+    store::{fs::entry_state::EntryState, util::DD},
+    util::channel::oneshot,
+    Hash,
+};
+
+/// Get the entry state for a hash.
+///
+/// This will read from the blobs table and enrich the result with the content
+/// of the inline data and inline outboard tables if necessary.
+pub struct Get {
+    pub hash: Hash,
+    pub tx: oneshot::Sender<GetResult>,
+    pub span: Span,
+}
+
+impl fmt::Debug for Get {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Get")
+            .field("hash", &DD(self.hash.to_hex()))
+            .finish_non_exhaustive()
+    }
+}
+
+#[derive(Debug)]
+pub struct GetResult {
+    pub state: ActorResult<Option<EntryState<Bytes>>>,
+}
+
+/// Get the entry state for a hash.
+///
+/// This will read from the blobs table and enrich the result with the content
+/// of the inline data and inline outboard tables if necessary.
+#[derive(Debug)]
+pub struct Dump {
+    pub tx: oneshot::Sender<anyhow::Result<()>>,
+    pub span: Span,
+}
+
+#[derive(Debug)]
+pub struct Snapshot {
+    pub(crate) tx: tokio::sync::oneshot::Sender<ReadOnlyTables>,
+    pub span: Span,
+}
+
+pub struct Update {
+    pub hash: Hash,
+    pub state: EntryState<Bytes>,
+    /// do I need this? Optional?
+    pub tx: Option<oneshot::Sender<ActorResult<()>>>,
+    pub span: Span,
+}
+
+impl fmt::Debug for Update {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Update")
+            .field("hash", &self.hash)
+            .field("state", &DD(self.state.fmt_short()))
+            .field("tx", &self.tx.is_some())
+            .finish()
+    }
+}
+
+pub struct Set {
+    pub hash: Hash,
+    pub state: EntryState<Bytes>,
+    pub tx: oneshot::Sender<ActorResult<()>>,
+    pub span: Span,
+}
+
+impl fmt::Debug for Set {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Set")
+            .field("hash", &self.hash)
+            .field("state", &DD(self.state.fmt_short()))
+            .finish_non_exhaustive()
+    }
+}
+
+/// Modification method: create a new unique tag and set it to a value.
+pub use crate::api::proto::CreateTagMsg;
+/// Modification method: remove a range of tags.
+pub use crate::api::proto::DeleteTagsMsg;
+/// Read method: list a range of tags.
+pub use crate::api::proto::ListTagsMsg;
+/// Modification method: rename a tag.
+pub use crate::api::proto::RenameTagMsg;
+/// Modification method: set a tag to a value, or remove it.
+pub use crate::api::proto::SetTagMsg;
+
+#[derive(Debug)]
+#[enum_conversions(Command)]
+pub enum ReadOnlyCommand {
+    Get(Get),
+    Dump(Dump),
+    ListTags(ListTagsMsg),
+    ClearProtected(ClearProtectedMsg),
+    GetBlobStatus(BlobStatusMsg),
+}
+
+impl ReadOnlyCommand {
+    pub fn parent_span(&self) -> tracing::Span {
+        self.parent_span_opt()
+            .cloned()
+            .unwrap_or_else(tracing::Span::current)
+    }
+
+    pub fn parent_span_opt(&self) -> Option<&tracing::Span> {
+        match self {
+            Self::Get(x) => Some(&x.span),
+            Self::Dump(x) => Some(&x.span),
+            Self::ListTags(x) => x.parent_span_opt(),
+            Self::ClearProtected(x) => x.parent_span_opt(),
+            Self::GetBlobStatus(x) => x.parent_span_opt(),
+        }
+    }
+}
+
+#[derive(Debug)]
+#[enum_conversions(Command)]
+pub enum ReadWriteCommand {
+    Update(Update),
+    Set(Set),
+    DeleteBlobw(DeleteBlobsMsg),
+    SetTag(SetTagMsg),
+    DeleteTags(DeleteTagsMsg),
+    RenameTag(RenameTagMsg),
+    CreateTag(CreateTagMsg),
+    ProcessExit(ProcessExitRequest),
+}
+
+impl ReadWriteCommand {
+    pub fn parent_span(&self) -> tracing::Span {
+        self.parent_span_opt()
+            .cloned()
+            .unwrap_or_else(tracing::Span::current)
+    }
+
+    pub fn parent_span_opt(&self) -> Option<&tracing::Span> {
+        match self {
+            Self::Update(x) => Some(&x.span),
+            Self::Set(x) => Some(&x.span),
+            Self::DeleteBlobw(x) => Some(&x.span),
+            Self::SetTag(x) => x.parent_span_opt(),
+            Self::DeleteTags(x) => x.parent_span_opt(),
+            Self::RenameTag(x) => x.parent_span_opt(),
+            Self::CreateTag(x) => x.parent_span_opt(),
+            Self::ProcessExit(_) => None,
+        }
+    }
+}
+
+#[derive(Debug)]
+#[enum_conversions(Command)]
+pub enum TopLevelCommand {
+    SyncDb(SyncDbMsg),
+    Shutdown(ShutdownMsg),
+    Snapshot(Snapshot),
+}
+
+impl TopLevelCommand {
+    pub fn parent_span(&self) -> tracing::Span {
+        self.parent_span_opt()
+            .cloned()
+            .unwrap_or_else(tracing::Span::current)
+    }
+
+    pub fn parent_span_opt(&self) -> Option<&tracing::Span> {
+        match self {
+            Self::SyncDb(x) => x.parent_span_opt(),
+            Self::Shutdown(x) => x.parent_span_opt(),
+            Self::Snapshot(x) => Some(&x.span),
+        }
+    }
+}
+
+#[enum_conversions()]
+pub enum Command {
+    ReadOnly(ReadOnlyCommand),
+    ReadWrite(ReadWriteCommand),
+    TopLevel(TopLevelCommand),
+}
+
+impl Command {
+    pub fn non_top_level(self) -> std::result::Result<NonTopLevelCommand, Self> {
+        match self {
+            Self::ReadOnly(cmd) => Ok(NonTopLevelCommand::ReadOnly(cmd)),
+            Self::ReadWrite(cmd) => Ok(NonTopLevelCommand::ReadWrite(cmd)),
+            _ => Err(self),
+        }
+    }
+
+    pub fn read_only(self) -> std::result::Result<ReadOnlyCommand, Self> {
+        match self {
+            Self::ReadOnly(cmd) => Ok(cmd),
+            _ => Err(self),
+        }
+    }
+}
+
+#[derive(Debug)]
+#[enum_conversions()]
+pub enum NonTopLevelCommand {
+    ReadOnly(ReadOnlyCommand),
+    ReadWrite(ReadWriteCommand),
+}
+
+impl fmt::Debug for Command {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::ReadOnly(cmd) => cmd.fmt(f),
+            Self::ReadWrite(cmd) => cmd.fmt(f),
+            Self::TopLevel(cmd) => cmd.fmt(f),
+        }
+    }
+}
diff --git a/src/store/fs/tables.rs b/src/store/fs/meta/tables.rs
similarity index 58%
rename from src/store/fs/tables.rs
rename to src/store/fs/meta/tables.rs
index 591326dc5..a983a275a 100644
--- a/src/store/fs/tables.rs
+++ b/src/store/fs/meta/tables.rs
@@ -1,10 +1,8 @@
 //! Table definitions and accessors for the redb database.
-use std::collections::BTreeSet;
-
 use redb::{ReadableTable, TableDefinition, TableError};
 
-use super::{EntryState, PathOptions};
-use crate::{util::Tag, Hash, HashAndFormat};
+use super::EntryState;
+use crate::store::{fs::delete_set::FileTransaction, util::Tag, Hash, HashAndFormat};
 
 pub(super) const BLOBS_TABLE: TableDefinition<Hash, EntryState> = TableDefinition::new("blobs-0");
 
@@ -33,27 +31,20 @@ pub(super) struct Tables<'a> {
     pub tags: redb::Table<'a, Tag, HashAndFormat>,
     pub inline_data: redb::Table<'a, Hash, &'static [u8]>,
     pub inline_outboard: redb::Table<'a, Hash, &'static [u8]>,
-    pub delete_after_commit: &'a mut DeleteSet,
-}
-
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
-pub(super) enum BaoFilePart {
-    Outboard,
-    Data,
-    Sizes,
+    pub ftx: &'a FileTransaction<'a>,
 }
 
 impl<'txn> Tables<'txn> {
     pub fn new(
         tx: &'txn redb::WriteTransaction,
-        delete_after_commit: &'txn mut DeleteSet,
+        ds: &'txn FileTransaction<'txn>,
     ) -> std::result::Result<Self, TableError> {
         Ok(Self {
             blobs: tx.open_table(BLOBS_TABLE)?,
             tags: tx.open_table(TAGS_TABLE)?,
             inline_data: tx.open_table(INLINE_DATA_TABLE)?,
             inline_outboard: tx.open_table(INLINE_OUTBOARD_TABLE)?,
-            delete_after_commit,
+            ftx: ds,
         })
     }
 }
@@ -75,7 +66,8 @@ impl ReadableTables for Tables<'_> {
 
 /// A struct similar to [`redb::ReadOnlyTable`] but for all tables that make up
 /// the blob store.
-pub(super) struct ReadOnlyTables {
+#[derive(Debug)]
+pub(crate) struct ReadOnlyTables {
     pub blobs: redb::ReadOnlyTable<Hash, EntryState>,
     pub tags: redb::ReadOnlyTable<Tag, HashAndFormat>,
     pub inline_data: redb::ReadOnlyTable<Hash, &'static [u8]>,
@@ -107,57 +99,3 @@ impl ReadableTables for ReadOnlyTables {
         &self.inline_outboard
     }
 }
-
-/// Helper to keep track of files to delete after a transaction is committed.
-#[derive(Debug, Default)]
-pub(super) struct DeleteSet(BTreeSet<(Hash, BaoFilePart)>);
-
-impl DeleteSet {
-    /// Mark a file as to be deleted after the transaction is committed.
-    pub fn insert(&mut self, hash: Hash, parts: impl IntoIterator<Item = BaoFilePart>) {
-        for part in parts {
-            self.0.insert((hash, part));
-        }
-    }
-
-    /// Mark a file as to be kept after the transaction is committed.
-    ///
-    /// This will cancel any previous delete for the same file in the same transaction.
-    pub fn remove(&mut self, hash: Hash, parts: impl IntoIterator<Item = BaoFilePart>) {
-        for part in parts {
-            self.0.remove(&(hash, part));
-        }
-    }
-
-    /// Get the inner set of files to delete.
-    pub fn into_inner(self) -> BTreeSet<(Hash, BaoFilePart)> {
-        self.0
-    }
-
-    /// Apply the delete set and clear it.
-    ///
-    /// This will delete all files marked for deletion and then clear the set.
-    /// Errors will just be logged.
-    pub fn apply_and_clear(&mut self, options: &PathOptions) {
-        for (hash, to_delete) in &self.0 {
-            tracing::debug!("deleting {:?} for {hash}", to_delete);
-            let path = match to_delete {
-                BaoFilePart::Data => options.owned_data_path(hash),
-                BaoFilePart::Outboard => options.owned_outboard_path(hash),
-                BaoFilePart::Sizes => options.owned_sizes_path(hash),
-            };
-            if let Err(cause) = std::fs::remove_file(&path) {
-                // Ignore NotFound errors, if the file is already gone that's fine.
-                if cause.kind() != std::io::ErrorKind::NotFound {
-                    tracing::warn!(
-                        "failed to delete {:?} {}: {}",
-                        to_delete,
-                        path.display(),
-                        cause
-                    );
-                }
-            }
-        }
-        self.0.clear();
-    }
-}
diff --git a/src/store/fs/options.rs b/src/store/fs/options.rs
new file mode 100644
index 000000000..6e123b75d
--- /dev/null
+++ b/src/store/fs/options.rs
@@ -0,0 +1,144 @@
+//! Options for configuring the file store.
+use std::{
+    path::{Path, PathBuf},
+    time::Duration,
+};
+
+use super::{gc::GcConfig, meta::raw_outboard_size, temp_name};
+use crate::Hash;
+
+/// Options for directories used by the file store.
+#[derive(Debug, Clone)]
+pub struct PathOptions {
+    /// Path to the directory where data and outboard files are stored.
+    pub data_path: PathBuf,
+    /// Path to the directory where temp files are stored.
+    /// This *must* be on the same device as `data_path`, since we need to
+    /// atomically move temp files into place.
+    pub temp_path: PathBuf,
+}
+
+impl PathOptions {
+    pub fn new(root: &Path) -> Self {
+        Self {
+            data_path: root.join("data"),
+            temp_path: root.join("temp"),
+        }
+    }
+
+    pub fn data_path(&self, hash: &Hash) -> PathBuf {
+        self.data_path.join(format!("{}.data", hash.to_hex()))
+    }
+
+    pub fn outboard_path(&self, hash: &Hash) -> PathBuf {
+        self.data_path.join(format!("{}.obao4", hash.to_hex()))
+    }
+
+    pub fn sizes_path(&self, hash: &Hash) -> PathBuf {
+        self.data_path.join(format!("{}.sizes4", hash.to_hex()))
+    }
+
+    pub fn bitfield_path(&self, hash: &Hash) -> PathBuf {
+        self.data_path.join(format!("{}.bitfield", hash.to_hex()))
+    }
+
+    pub fn temp_file_name(&self) -> PathBuf {
+        self.temp_path.join(temp_name())
+    }
+}
+
+/// Options for inlining small complete data or outboards.
+#[derive(Debug, Clone)]
+pub struct InlineOptions {
+    /// Maximum data size to inline.
+    pub max_data_inlined: u64,
+    /// Maximum outboard size to inline.
+    pub max_outboard_inlined: u64,
+}
+
+impl InlineOptions {
+    /// Do not inline anything, ever.
+    pub const NO_INLINE: Self = Self {
+        max_data_inlined: 0,
+        max_outboard_inlined: 0,
+    };
+    /// Always inline everything
+    pub const ALWAYS_INLINE: Self = Self {
+        max_data_inlined: u64::MAX,
+        max_outboard_inlined: u64::MAX,
+    };
+}
+
+impl Default for InlineOptions {
+    fn default() -> Self {
+        Self {
+            max_data_inlined: 1024 * 16,
+            max_outboard_inlined: 1024 * 16,
+        }
+    }
+}
+
+/// Options for transaction batching.
+#[derive(Debug, Clone)]
+pub struct BatchOptions {
+    /// Maximum number of actor messages to batch before creating a new read transaction.
+    pub max_read_batch: usize,
+    /// Maximum duration to wait before committing a read transaction.
+    pub max_read_duration: Duration,
+    /// Maximum number of actor messages to batch before committing write transaction.
+    pub max_write_batch: usize,
+    /// Maximum duration to wait before committing a write transaction.
+    pub max_write_duration: Duration,
+}
+
+impl Default for BatchOptions {
+    fn default() -> Self {
+        Self {
+            max_read_batch: 10000,
+            max_read_duration: Duration::from_secs(1),
+            max_write_batch: 1000,
+            max_write_duration: Duration::from_millis(500),
+        }
+    }
+}
+
+/// Options for the file store.
+#[derive(Debug, Clone)]
+pub struct Options {
+    /// Path options.
+    pub path: PathOptions,
+    /// Inline storage options.
+    pub inline: InlineOptions,
+    /// Transaction batching options.
+    pub batch: BatchOptions,
+    /// Gc configuration.
+    pub gc: Option<GcConfig>,
+}
+
+impl Options {
+    /// Create new optinos with the given root path and everything else default.
+    pub fn new(root: &Path) -> Self {
+        Self {
+            path: PathOptions::new(root),
+            inline: InlineOptions::default(),
+            batch: BatchOptions::default(),
+            gc: None,
+        }
+    }
+
+    // check if the data will be inlined, based on the size of the data
+    pub fn is_inlined_data(&self, data_size: u64) -> bool {
+        data_size <= self.inline.max_data_inlined
+    }
+
+    // check if the outboard will be inlined, based on the size of the *outboard*
+    pub fn is_inlined_outboard(&self, outboard_size: u64) -> bool {
+        outboard_size <= self.inline.max_outboard_inlined
+    }
+
+    // check if both the data and outboard will be inlined, based on the size of the data
+    pub fn is_inlined_all(&self, data_size: u64) -> bool {
+        let outboard_size = raw_outboard_size(data_size);
+        self.is_inlined_data(data_size) && self.is_inlined_outboard(outboard_size)
+    }
+}
diff --git a/src/store/fs/test_support.rs b/src/store/fs/test_support.rs
deleted file mode 100644
index 9cc62bb86..000000000
--- a/src/store/fs/test_support.rs
+++ /dev/null
@@ -1,401 +0,0 @@
-//! DB functions to support testing
-//!
-//! For some tests we need to modify the state of the store in ways that are not
-//! possible through the public API. This module provides functions to do that.
-use std::{
-    io,
-    path::{Path, PathBuf},
-};
-
-use redb::ReadableTable;
-
-use super::{
-    tables::{ReadableTables, Tables},
-    ActorError, ActorMessage, ActorResult, ActorState, DataLocation, EntryState, FilterPredicate,
-    OutboardLocation, OuterResult, Store, StoreInner,
-};
-use crate::{
-    store::{mutable_mem_storage::SizeInfo, DbIter},
-    util::raw_outboard_size,
-    Hash,
-};
-
-/// The full state of an entry, including the data.
-#[derive(derive_more::Debug)]
-pub enum EntryData {
-    /// Complete
-    Complete {
-        /// Data
-        #[debug("data")]
-        data: Vec<u8>,
-        /// Outboard
-        #[debug("outboard")]
-        outboard: Vec<u8>,
-    },
-    /// Partial
-    Partial {
-        /// Data
-        #[debug("data")]
-        data: Vec<u8>,
-        /// Outboard
-        #[debug("outboard")]
-        outboard: Vec<u8>,
-        /// Sizes
-        #[debug("sizes")]
-        sizes: Vec<u8>,
-    },
-}
-
-impl Store {
-    /// Get the complete state of an entry, both in memory and in redb.
-    #[cfg(test)]
-    pub(crate) async fn entry_state(&self, hash: Hash) -> io::Result<EntryStateResponse> {
-        Ok(self.0.entry_state(hash).await?)
-    }
-
-    async fn all_blobs(&self) -> io::Result<DbIter<Hash>> {
-        Ok(Box::new(self.0.all_blobs().await?.into_iter()))
-    }
-
-    /// Transform all entries in the store. This is for testing and can be used to get the store
-    /// in a wrong state.
-    pub async fn transform_entries(
-        &self,
-        transform: impl Fn(Hash, EntryData) -> Option<EntryData> + Send + Sync,
-    ) -> io::Result<()> {
-        let blobs = self.all_blobs().await?;
-        for blob in blobs {
-            let hash = blob?;
-            let entry = self.get_full_entry_state(hash).await?;
-            if let Some(entry) = entry {
-                let entry1 = transform(hash, entry);
-                self.set_full_entry_state(hash, entry1).await?;
-            }
-        }
-        Ok(())
-    }
-
-    /// Set the full entry state for a hash. This is for testing and can be used to get the store
-    /// in a wrong state.
-    pub(crate) async fn set_full_entry_state(
-        &self,
-        hash: Hash,
-        entry: Option<EntryData>,
-    ) -> io::Result<()> {
-        Ok(self.0.set_full_entry_state(hash, entry).await?)
-    }
-
-    /// Set the full entry state for a hash. This is for testing and can be used to get the store
-    /// in a wrong state.
-    pub(crate) async fn get_full_entry_state(&self, hash: Hash) -> io::Result<Option<EntryData>> {
-        Ok(self.0.get_full_entry_state(hash).await?)
-    }
-
-    /// Owned data path
-    pub fn owned_data_path(&self, hash: &Hash) -> PathBuf {
-        self.0.path_options.owned_data_path(hash)
-    }
-
-    /// Owned outboard path
-    pub fn owned_outboard_path(&self, hash: &Hash) -> PathBuf {
-        self.0.path_options.owned_outboard_path(hash)
-    }
-}
-
-impl StoreInner {
-    #[cfg(test)]
-    async fn entry_state(&self, hash: Hash) -> OuterResult<EntryStateResponse> {
-        let (tx, rx) = oneshot::channel();
-        self.tx.send(ActorMessage::EntryState { hash, tx }).await?;
-        Ok(rx.await??)
-    }
-
-    async fn set_full_entry_state(&self, hash: Hash, entry: Option<EntryData>) -> OuterResult<()> {
-        let (tx, rx) = oneshot::channel();
-        self.tx
-            .send(ActorMessage::SetFullEntryState { hash, entry, tx })
-            .await?;
-        Ok(rx.await??)
-    }
-
-    async fn get_full_entry_state(&self, hash: Hash) -> OuterResult<Option<EntryData>> {
-        let (tx, rx) = oneshot::channel();
-        self.tx
-            .send(ActorMessage::GetFullEntryState { hash, tx })
-            .await?;
-        Ok(rx.await??)
-    }
-
-    async fn all_blobs(&self) -> OuterResult<Vec<io::Result<Hash>>> {
-        let (tx, rx) = oneshot::channel();
-        let filter: FilterPredicate<Hash, EntryState> =
-            Box::new(|_i, k, v| Some((k.value(), v.value())));
-        self.tx.send(ActorMessage::Blobs { filter, tx }).await?;
-        let blobs = rx.await?;
-        let res = blobs?
-            .into_iter()
-            .map(|r| {
-                r.map(|(hash, _)| hash)
-                    .map_err(|e| ActorError::from(e).into())
-            })
-            .collect::<Vec<_>>();
-        Ok(res)
-    }
-}
-
-#[cfg(test)]
-#[derive(Debug)]
-pub(crate) struct EntryStateResponse {
-    pub mem: Option<crate::store::bao_file::BaoFileHandle>,
-    pub db: Option<EntryState<Vec<u8>>>,
-}
-
-impl ActorState {
-    pub(super) fn get_full_entry_state(
-        &mut self,
-        tables: &impl ReadableTables,
-        hash: Hash,
-    ) -> ActorResult<Option<EntryData>> {
-        let data_path = self.options.path.owned_data_path(&hash);
-        let outboard_path = self.options.path.owned_outboard_path(&hash);
-        let sizes_path = self.options.path.owned_sizes_path(&hash);
-        let entry = match tables.blobs().get(hash)? {
-            Some(guard) => match guard.value() {
-                EntryState::Complete {
-                    data_location,
-                    outboard_location,
-                } => {
-                    let data = match data_location {
-                        DataLocation::External(paths, size) => {
-                            let path = paths.first().ok_or_else(|| {
-                                ActorError::Inconsistent("external data missing".to_owned())
-                            })?;
-                            let res = std::fs::read(path)?;
-                            if res.len() != size as usize {
-                                return Err(ActorError::Inconsistent(
-                                    "external data size mismatch".to_owned(),
-                                ));
-                            }
-                            res
-                        }
-                        DataLocation::Owned(size) => {
-                            let res = std::fs::read(data_path)?;
-                            if res.len() != size as usize {
-                                return Err(ActorError::Inconsistent(
-                                    "owned data size mismatch".to_owned(),
-                                ));
-                            }
-                            res
-                        }
-                        DataLocation::Inline(_) => {
-                            let data = tables.inline_data().get(hash)?.ok_or_else(|| {
-                                ActorError::Inconsistent("inline data missing".to_owned())
-                            })?;
-                            data.value().to_vec()
-                        }
-                    };
-                    let expected_outboard_size = raw_outboard_size(data.len() as u64);
-                    let outboard = match outboard_location {
-                        OutboardLocation::Owned => std::fs::read(outboard_path)?,
-                        OutboardLocation::Inline(_) => tables
-                            .inline_outboard()
-                            .get(hash)?
-                            .ok_or_else(|| {
-                                ActorError::Inconsistent("inline outboard missing".to_owned())
-                            })?
-                            .value()
-                            .to_vec(),
-                        OutboardLocation::NotNeeded => Vec::new(),
-                    };
-                    if outboard.len() != expected_outboard_size as usize {
-                        return Err(ActorError::Inconsistent(
-                            "outboard size mismatch".to_owned(),
-                        ));
-                    }
-                    Some(EntryData::Complete { data, outboard })
-                }
-                EntryState::Partial { .. } => {
-                    let data = std::fs::read(data_path)?;
-                    let outboard = std::fs::read(outboard_path)?;
-                    let sizes = std::fs::read(sizes_path)?;
-                    Some(EntryData::Partial {
-                        data,
-                        outboard,
-                        sizes,
-                    })
-                }
-            },
-            None => None,
-        };
-        Ok(entry)
-    }
-
-    pub(super) fn set_full_entry_state(
-        &mut self,
-        tables: &mut Tables,
-        hash: Hash,
-        entry: Option<EntryData>,
-    ) -> ActorResult<()> {
-        let data_path = self.options.path.owned_data_path(&hash);
-        let outboard_path = self.options.path.owned_outboard_path(&hash);
-        let sizes_path = self.options.path.owned_sizes_path(&hash);
-        // tabula rasa
-        std::fs::remove_file(&outboard_path).ok();
-        std::fs::remove_file(&data_path).ok();
-        std::fs::remove_file(&sizes_path).ok();
-        tables.inline_data.remove(&hash)?;
-        tables.inline_outboard.remove(&hash)?;
-        let Some(entry) = entry else {
-            tables.blobs.remove(&hash)?;
-            return Ok(());
-        };
-        // write the new data and determine the new state
-        let entry = match entry {
-            EntryData::Complete { data, outboard } => {
-                let data_size = data.len() as u64;
-                let data_location = if data_size > self.options.inline.max_data_inlined {
-                    std::fs::write(data_path, &data)?;
-                    DataLocation::Owned(data_size)
-                } else {
-                    tables.inline_data.insert(hash, data.as_slice())?;
-                    DataLocation::Inline(())
-                };
-                let outboard_size = outboard.len() as u64;
-                let outboard_location = if outboard_size > self.options.inline.max_outboard_inlined
-                {
-                    std::fs::write(outboard_path, &outboard)?;
-                    OutboardLocation::Owned
-                } else if outboard_size > 0 {
-                    tables.inline_outboard.insert(hash, outboard.as_slice())?;
-                    OutboardLocation::Inline(())
-                } else {
-                    OutboardLocation::NotNeeded
-                };
-                EntryState::Complete {
-                    data_location,
-                    outboard_location,
-                }
-            }
-            EntryData::Partial {
-                data,
-                outboard,
-                sizes,
-            } => {
-                std::fs::write(data_path, data)?;
-                std::fs::write(outboard_path, outboard)?;
-                std::fs::write(sizes_path, sizes)?;
-                EntryState::Partial { size: None }
-            }
-        };
-        // finally, write the state
-        tables.blobs.insert(hash, entry)?;
-        Ok(())
-    }
-
-    #[cfg(test)]
-    pub(super) fn entry_state(
-        &mut self,
-        tables: &impl ReadableTables,
-        hash: Hash,
-    ) -> ActorResult<EntryStateResponse> {
-        let mem = self.handles.get(&hash).and_then(|weak| weak.upgrade());
-        let db = match tables.blobs().get(hash)? {
-            Some(entry) => Some({
-                match entry.value() {
-                    EntryState::Complete {
-                        data_location,
-                        outboard_location,
-                    } => {
-                        let data_location = match data_location {
-                            DataLocation::Inline(()) => {
-                                let data = tables.inline_data().get(hash)?.ok_or_else(|| {
-                                    ActorError::Inconsistent("inline data missing".to_owned())
-                                })?;
-                                DataLocation::Inline(data.value().to_vec())
-                            }
-                            DataLocation::Owned(x) => DataLocation::Owned(x),
-                            DataLocation::External(p, s) => DataLocation::External(p, s),
-                        };
-                        let outboard_location = match outboard_location {
-                            OutboardLocation::Inline(()) => {
-                                let outboard =
-                                    tables.inline_outboard().get(hash)?.ok_or_else(|| {
-                                        ActorError::Inconsistent(
-                                            "inline outboard missing".to_owned(),
-                                        )
-                                    })?;
-                                OutboardLocation::Inline(outboard.value().to_vec())
-                            }
-                            OutboardLocation::Owned => OutboardLocation::Owned,
-                            OutboardLocation::NotNeeded => OutboardLocation::NotNeeded,
-                        };
-                        EntryState::Complete {
-                            data_location,
-                            outboard_location,
-                        }
-                    }
-                    EntryState::Partial { size } => EntryState::Partial { size },
-                }
-            }),
-            None => None,
-        };
-        Ok(EntryStateResponse { mem, db })
-    }
-}
-
-/// What do to with a file pair when making partial files
-#[derive(Debug)]
-pub enum MakePartialResult {
-    /// leave the file as is
-    Retain,
-    /// remove it entirely
-    Remove,
-    /// truncate the data file to the given size
-    Truncate(u64),
-}
-
-/// Open a database and make it partial.
-pub fn make_partial(
-    path: &Path,
-    f: impl Fn(Hash, u64) -> MakePartialResult + Send + Sync,
-) -> io::Result<()> {
-    tracing::info!("starting runtime for make_partial");
-    let rt = tokio::runtime::Builder::new_current_thread()
-        .enable_all()
-        .build()?;
-    rt.block_on(async move {
-        let blobs_path = path.join("blobs");
-        let store = Store::load(blobs_path).await?;
-        store
-            .transform_entries(|hash, entry| match &entry {
-                EntryData::Complete { data, outboard } => {
-                    let res = f(hash, data.len() as u64);
-                    tracing::info!("make_partial: {} {:?}", hash, res);
-                    match res {
-                        MakePartialResult::Retain => Some(entry),
-                        MakePartialResult::Remove => None,
-                        MakePartialResult::Truncate(size) => {
-                            let current_size = data.len() as u64;
-                            if size < current_size {
-                                let size = size as usize;
-                                let sizes = SizeInfo::complete(current_size).to_vec();
-                                Some(EntryData::Partial {
-                                    data: data[..size].to_vec(),
-                                    outboard: outboard.to_vec(),
-                                    sizes,
-                                })
-                            } else {
-                                Some(entry)
-                            }
-                        }
-                    }
-                }
-                EntryData::Partial { .. } => Some(entry),
-            })
-            .await?;
-        std::io::Result::Ok(())
-    })?;
-    drop(rt);
-    tracing::info!("done with make_partial");
-    Ok(())
-}
diff --git a/src/store/fs/tests.rs b/src/store/fs/tests.rs
deleted file mode 100644
index 85540eb89..000000000
--- a/src/store/fs/tests.rs
+++ /dev/null
@@ -1,811 +0,0 @@
-use std::io::Cursor;
-
-use bao_tree::ChunkRanges;
-use iroh_io::AsyncSliceReaderExt;
-
-use crate::{
-    store::{
-        bao_file::test_support::{
-            decode_response_into_batch, make_wire_data, random_test_data, simulate_remote, validate,
-        },
-        Map as _, MapEntryMut, MapMut, ReadableStore, Store as _,
-    },
-    util::raw_outboard,
-    IROH_BLOCK_SIZE,
-};
-
-macro_rules! assert_matches {
-        ($expression:expr, $pattern:pat) => {
-            match $expression {
-                $pattern => (),
-                _ => panic!("assertion failed: `(expr matches pattern)` \
-                             expression: `{:?}`, pattern: `{}`", $expression, stringify!($pattern)),
-            }
-        };
-        ($expression:expr, $pattern:pat, $($arg:tt)+) => {
-            match $expression {
-                $pattern => (),
-                _ => panic!("{}: expression: `{:?}`, pattern: `{}`", format_args!($($arg)+), $expression, stringify!($pattern)),
-            }
-        };
-    }
-
-use super::*;
-
-/// Helper to simulate a slow request.
-pub fn to_stream(
-    data: &[u8],
-    mtu: usize,
-    delay: std::time::Duration,
-) -> impl Stream<Item = io::Result<Bytes>> + 'static {
-    let parts = data
-        .chunks(mtu)
-        .map(Bytes::copy_from_slice)
-        .collect::<Vec<_>>();
-    futures_lite::stream::iter(parts)
-        .then(move |part| async move {
-            tokio::time::sleep(delay).await;
-            io::Result::Ok(part)
-        })
-        .boxed()
-}
-
-async fn create_test_db() -> (tempfile::TempDir, Store) {
-    let _ = tracing_subscriber::fmt::try_init();
-    let testdir = tempfile::tempdir().unwrap();
-    let db_path = testdir.path().join("db.redb");
-    let options = Options {
-        path: PathOptions::new(testdir.path()),
-        batch: Default::default(),
-        inline: Default::default(),
-    };
-    let db = Store::new(db_path, options).await.unwrap();
-    (testdir, db)
-}
-
-/// small file that does not have outboard at all
-const SMALL_SIZE: u64 = 1024;
-/// medium file that has inline outboard but file data
-const MID_SIZE: u64 = 1024 * 32;
-/// large file that has file outboard and file data
-const LARGE_SIZE: u64 = 1024 * 1024 * 10;
-
-#[tokio::test]
-async fn get_cases() {
-    let np = IgnoreProgressSender::<ImportProgress>::default;
-    let (tempdir, db) = create_test_db().await;
-    {
-        let small = Bytes::from(random_test_data(SMALL_SIZE as usize));
-        let (_outboard, hash) = raw_outboard(&small);
-        let res = db.get(&hash).await.unwrap();
-        assert_matches!(res, None);
-        let tt = db
-            .import_bytes(small.clone(), BlobFormat::Raw)
-            .await
-            .unwrap();
-        let res = db.get(&hash).await.unwrap();
-        let entry = res.expect("entry not found");
-        let actual = entry.data_reader().read_to_end().await.unwrap();
-        assert_eq!(actual, small);
-        drop(tt);
-    }
-    {
-        let mid = Bytes::from(random_test_data(MID_SIZE as usize));
-        let (_outboard, hash) = raw_outboard(&mid);
-        let res = db.get(&hash).await.unwrap();
-        assert_matches!(res, None);
-        let tt = db.import_bytes(mid.clone(), BlobFormat::Raw).await.unwrap();
-        let res = db.get(&hash).await.unwrap();
-        let entry = res.expect("entry not found");
-        let actual = entry.data_reader().read_to_end().await.unwrap();
-        assert_eq!(actual, mid);
-        drop(tt);
-    }
-    {
-        let large = Bytes::from(random_test_data(LARGE_SIZE as usize));
-        let (_outboard, hash) = raw_outboard(&large);
-        let res = db.get(&hash).await.unwrap();
-        assert_matches!(res, None);
-        let tt = db
-            .import_bytes(large.clone(), BlobFormat::Raw)
-            .await
-            .unwrap();
-        let res = db.get(&hash).await.unwrap();
-        let entry = res.expect("entry not found");
-        let actual = entry.data_reader().read_to_end().await.unwrap();
-        assert_eq!(actual, large);
-        drop(tt);
-    }
-    {
-        let mid = random_test_data(MID_SIZE as usize);
-        let path = tempdir.path().join("mid.data");
-        std::fs::write(&path, &mid).unwrap();
-        let (_outboard, hash) = raw_outboard(&mid);
-        let res = db.get(&hash).await.unwrap();
-        assert_matches!(res, None);
-        let tt = db
-            .import_file(path, ImportMode::TryReference, BlobFormat::Raw, np())
-            .await
-            .unwrap();
-        let res = db.get(&hash).await.unwrap();
-        let entry = res.expect("entry not found");
-        let actual = entry.data_reader().read_to_end().await.unwrap();
-        assert_eq!(actual, mid);
-        drop(tt);
-    }
-}
-
-#[tokio::test]
-async fn get_or_create_cases() {
-    let (_tempdir, db) = create_test_db().await;
-    {
-        const SIZE: u64 = SMALL_SIZE;
-        let data = random_test_data(SIZE as usize);
-        let (hash, reader) = simulate_remote(&data);
-        let entry = db.get_or_create(hash, 0).await.unwrap();
-        {
-            let state = db.entry_state(hash).await.unwrap();
-            assert_eq!(state.db, None);
-            assert_matches!(state.mem, Some(_));
-        }
-        let writer = entry.batch_writer().await.unwrap();
-        decode_response_into_batch(hash, IROH_BLOCK_SIZE, ChunkRanges::all(), reader, writer)
-            .await
-            .unwrap();
-        {
-            let state = db.entry_state(hash).await.unwrap();
-            assert_matches!(state.mem, Some(_));
-            assert_matches!(state.db, None);
-        }
-        db.insert_complete(entry.clone()).await.unwrap();
-        {
-            let state = db.entry_state(hash).await.unwrap();
-            assert_matches!(state.mem, Some(_));
-            assert_matches!(
-                state.db,
-                Some(EntryState::Complete {
-                    data_location: DataLocation::Inline(_),
-                    ..
-                })
-            );
-        }
-        drop(entry);
-        // sync so we know the msg sent on drop is processed
-        db.sync().await.unwrap();
-        let state = db.entry_state(hash).await.unwrap();
-        assert_matches!(state.mem, None);
-    }
-    {
-        const SIZE: u64 = MID_SIZE;
-        let data = random_test_data(SIZE as usize);
-        let (hash, reader) = simulate_remote(&data);
-        let entry = db.get_or_create(hash, 0).await.unwrap();
-        {
-            let state = db.entry_state(hash).await.unwrap();
-            assert_eq!(state.db, None);
-            assert_matches!(state.mem, Some(_));
-        }
-        let writer = entry.batch_writer().await.unwrap();
-        decode_response_into_batch(hash, IROH_BLOCK_SIZE, ChunkRanges::all(), reader, writer)
-            .await
-            .unwrap();
-        {
-            let state = db.entry_state(hash).await.unwrap();
-            assert_matches!(state.mem, Some(_));
-            assert_matches!(state.db, Some(EntryState::Partial { .. }));
-        }
-        db.insert_complete(entry.clone()).await.unwrap();
-        {
-            let state = db.entry_state(hash).await.unwrap();
-            assert_matches!(state.mem, Some(_));
-            assert_matches!(
-                state.db,
-                Some(EntryState::Complete {
-                    data_location: DataLocation::Owned(SIZE),
-                    ..
-                })
-            );
-        }
-        drop(entry);
-        // // sync so we know the msg sent on drop is processed
-        // db.sync().await.unwrap();
-        // let state = db.entry_state(hash).await.unwrap();
-        // assert_matches!(state.mem, None);
-    }
-}
-
-/// Import mem cases, small (data inline, outboard none), mid (data file, outboard inline), large (data file, outboard file)
-#[tokio::test]
-async fn import_mem_cases() {
-    let (_tempdir, db) = create_test_db().await;
-    {
-        const SIZE: u64 = SMALL_SIZE;
-        let small = random_test_data(SIZE as usize);
-        let (outboard, hash) = raw_outboard(&small);
-        let tt = db
-            .import_bytes(small.clone().into(), BlobFormat::Raw)
-            .await
-            .unwrap();
-        let actual = db.entry_state(*tt.hash()).await.unwrap();
-        let expected = EntryState::Complete {
-            data_location: DataLocation::Inline(small),
-            outboard_location: OutboardLocation::NotNeeded,
-        };
-        assert_eq!(tt.hash(), &hash);
-        assert_eq!(actual.db, Some(expected));
-        assert!(outboard.is_empty());
-    }
-    {
-        const SIZE: u64 = MID_SIZE;
-        let mid = Bytes::from(random_test_data(SIZE as usize));
-        let (outboard, hash) = raw_outboard(&mid);
-        let tt = db.import_bytes(mid.clone(), BlobFormat::Raw).await.unwrap();
-        let actual = db.entry_state(*tt.hash()).await.unwrap();
-        let expected = EntryState::Complete {
-            data_location: DataLocation::Owned(SIZE),
-            outboard_location: OutboardLocation::Inline(outboard),
-        };
-        assert_eq!(tt.hash(), &hash);
-        assert_eq!(actual.db, Some(expected));
-        assert_eq!(mid, std::fs::read(db.owned_data_path(&hash)).unwrap());
-    }
-    {
-        const SIZE: u64 = LARGE_SIZE;
-        let large = Bytes::from(random_test_data(SIZE as usize));
-        let (outboard, hash) = raw_outboard(&large);
-        let tt = db
-            .import_bytes(large.clone(), BlobFormat::Raw)
-            .await
-            .unwrap();
-        let actual = db.entry_state(*tt.hash()).await.unwrap();
-        let expected = EntryState::Complete {
-            data_location: DataLocation::Owned(SIZE),
-            outboard_location: OutboardLocation::Owned,
-        };
-        assert_eq!(tt.hash(), &hash);
-        assert_eq!(actual.db, Some(expected));
-        assert_eq!(large, std::fs::read(db.owned_data_path(&hash)).unwrap());
-        assert_eq!(
-            outboard,
-            tokio::fs::read(db.owned_outboard_path(&hash))
-                .await
-                .unwrap()
-        );
-    }
-}
-
-/// Import mem cases, small (data inline, outboard none), mid (data file, outboard inline), large (data file, outboard file)
-#[tokio::test]
-async fn import_stream_cases() {
-    let np = IgnoreProgressSender::<ImportProgress>::default;
-    let (_tempdir, db) = create_test_db().await;
-    {
-        const SIZE: u64 = SMALL_SIZE;
-        let small = random_test_data(SIZE as usize);
-        let (outboard, hash) = raw_outboard(&small);
-        let (tt, size) = db
-            .import_stream(
-                to_stream(&small, 100, Duration::from_millis(1)),
-                BlobFormat::Raw,
-                np(),
-            )
-            .await
-            .unwrap();
-        let actual = db.entry_state(*tt.hash()).await.unwrap();
-        let expected = EntryState::Complete {
-            data_location: DataLocation::Inline(small),
-            outboard_location: OutboardLocation::NotNeeded,
-        };
-        assert_eq!(size, SIZE);
-        assert_eq!(tt.hash(), &hash);
-        assert_eq!(actual.db, Some(expected));
-        assert!(outboard.is_empty());
-    }
-    {
-        const SIZE: u64 = MID_SIZE;
-        let mid = Bytes::from(random_test_data(SIZE as usize));
-        let (outboard, hash) = raw_outboard(&mid);
-        let (tt, size) = db
-            .import_stream(
-                to_stream(&mid, 1000, Duration::from_millis(1)),
-                BlobFormat::Raw,
-                np(),
-            )
-            .await
-            .unwrap();
-        let actual = db.entry_state(*tt.hash()).await.unwrap();
-        let expected = EntryState::Complete {
-            data_location: DataLocation::Owned(SIZE),
-            outboard_location: OutboardLocation::Inline(outboard),
-        };
-        assert_eq!(size, SIZE);
-        assert_eq!(tt.hash(), &hash);
-        assert_eq!(actual.db, Some(expected));
-        assert_eq!(mid, std::fs::read(db.owned_data_path(&hash)).unwrap());
-    }
-    {
-        const SIZE: u64 = LARGE_SIZE;
-        let large = Bytes::from(random_test_data(SIZE as usize));
-        let (outboard, hash) = raw_outboard(&large);
-        let (tt, size) = db
-            .import_stream(
-                to_stream(&large, 100000, Duration::from_millis(1)),
-                BlobFormat::Raw,
-                np(),
-            )
-            .await
-            .unwrap();
-        let actual = db.entry_state(*tt.hash()).await.unwrap();
-        let expected = EntryState::Complete {
-            data_location: DataLocation::Owned(SIZE),
-            outboard_location: OutboardLocation::Owned,
-        };
-        assert_eq!(size, SIZE);
-        assert_eq!(tt.hash(), &hash);
-        assert_eq!(actual.db, Some(expected));
-        assert_eq!(large, std::fs::read(db.owned_data_path(&hash)).unwrap());
-        assert_eq!(
-            outboard,
-            tokio::fs::read(db.owned_outboard_path(&hash))
-                .await
-                .unwrap()
-        );
-    }
-}
-
-/// Import file cases, small (data inline, outboard none), mid (data file, outboard inline), large (data file, outboard file)
-#[tokio::test]
-async fn import_file_cases() {
-    let np = IgnoreProgressSender::<ImportProgress>::default;
-    let (tempdir, db) = create_test_db().await;
-    {
-        const SIZE: u64 = SMALL_SIZE;
-        let small = random_test_data(SIZE as usize);
-        let path = tempdir.path().join("small.data");
-        std::fs::write(&path, &small).unwrap();
-        let (outboard, hash) = raw_outboard(&small);
-        let (tt, size) = db
-            .import_file(path, ImportMode::Copy, BlobFormat::Raw, np())
-            .await
-            .unwrap();
-        let actual = db.entry_state(*tt.hash()).await.unwrap();
-        let expected = EntryState::Complete {
-            data_location: DataLocation::Inline(small),
-            outboard_location: OutboardLocation::NotNeeded,
-        };
-        assert_eq!(size, SIZE);
-        assert_eq!(tt.hash(), &hash);
-        assert_eq!(actual.db, Some(expected));
-        assert!(outboard.is_empty());
-    }
-    {
-        const SIZE: u64 = MID_SIZE;
-        let mid = Bytes::from(random_test_data(SIZE as usize));
-        let path = tempdir.path().join("mid.data");
-        std::fs::write(&path, &mid).unwrap();
-        let (outboard, hash) = raw_outboard(&mid);
-        let (tt, size) = db
-            .import_file(path, ImportMode::Copy, BlobFormat::Raw, np())
-            .await
-            .unwrap();
-        let actual = db.entry_state(*tt.hash()).await.unwrap();
-        let expected = EntryState::Complete {
-            data_location: DataLocation::Owned(SIZE),
-            outboard_location: OutboardLocation::Inline(outboard),
-        };
-        assert_eq!(size, SIZE);
-        assert_eq!(tt.hash(), &hash);
-        assert_eq!(actual.db, Some(expected));
-        assert_eq!(mid, std::fs::read(db.owned_data_path(&hash)).unwrap());
-    }
-    {
-        const SIZE: u64 = LARGE_SIZE;
-        let large = Bytes::from(random_test_data(SIZE as usize));
-        let path = tempdir.path().join("mid.data");
-        std::fs::write(&path, &large).unwrap();
-        let (outboard, hash) = raw_outboard(&large);
-        let (tt, size) = db
-            .import_file(path, ImportMode::Copy, BlobFormat::Raw, np())
-            .await
-            .unwrap();
-        let actual = db.entry_state(*tt.hash()).await.unwrap();
-        let expected = EntryState::Complete {
-            data_location: DataLocation::Owned(SIZE),
-            outboard_location: OutboardLocation::Owned,
-        };
-        assert_eq!(size, SIZE);
-        assert_eq!(tt.hash(), &hash);
-        assert_eq!(actual.db, Some(expected));
-        assert_eq!(large, std::fs::read(db.owned_data_path(&hash)).unwrap());
-        assert_eq!(
-            outboard,
-            tokio::fs::read(db.owned_outboard_path(&hash))
-                .await
-                .unwrap()
-        );
-    }
-}
-
-#[tokio::test]
-async fn import_file_reference_cases() {
-    let np = IgnoreProgressSender::<ImportProgress>::default;
-    let (tempdir, db) = create_test_db().await;
-    {
-        const SIZE: u64 = SMALL_SIZE;
-        let small = random_test_data(SIZE as usize);
-        let path = tempdir.path().join("small.data");
-        std::fs::write(&path, &small).unwrap();
-        let (outboard, hash) = raw_outboard(&small);
-        let (tt, size) = db
-            .import_file(path, ImportMode::TryReference, BlobFormat::Raw, np())
-            .await
-            .unwrap();
-        let actual = db.entry_state(*tt.hash()).await.unwrap();
-        let expected = EntryState::Complete {
-            data_location: DataLocation::Inline(small),
-            outboard_location: OutboardLocation::NotNeeded,
-        };
-        assert_eq!(size, SIZE);
-        assert_eq!(tt.hash(), &hash);
-        assert_eq!(actual.db, Some(expected));
-        assert!(outboard.is_empty());
-    }
-    {
-        const SIZE: u64 = MID_SIZE;
-        let mid = random_test_data(SIZE as usize);
-        let path = tempdir.path().join("mid.data");
-        std::fs::write(&path, &mid).unwrap();
-        let (outboard, hash) = raw_outboard(&mid);
-        let (tt, size) = db
-            .import_file(
-                path.clone(),
-                ImportMode::TryReference,
-                BlobFormat::Raw,
-                np(),
-            )
-            .await
-            .unwrap();
-        let actual = db.entry_state(*tt.hash()).await.unwrap();
-        let expected = EntryState::Complete {
-            data_location: DataLocation::External(vec![path.clone()], SIZE),
-            outboard_location: OutboardLocation::Inline(outboard),
-        };
-        assert_eq!(size, SIZE);
-        assert_eq!(tt.hash(), &hash);
-        assert_eq!(actual.db, Some(expected));
-        assert_eq!(mid, std::fs::read(path).unwrap());
-        assert!(!db.owned_data_path(&hash).exists());
-    }
-}
-
-#[tokio::test]
-async fn import_file_error_cases() {
-    let np = IgnoreProgressSender::<ImportProgress>::default;
-    let (tempdir, db) = create_test_db().await;
-    // relative path is not allowed
-    {
-        let path = PathBuf::from("relativepath.data");
-        let cause = db
-            .import_file(path, ImportMode::Copy, BlobFormat::Raw, np())
-            .await
-            .unwrap_err();
-        assert_eq!(cause.kind(), io::ErrorKind::InvalidInput);
-    }
-    // file does not exist
-    {
-        let path = tempdir.path().join("pathdoesnotexist.data");
-        let cause = db
-            .import_file(path, ImportMode::Copy, BlobFormat::Raw, np())
-            .await
-            .unwrap_err();
-        assert_eq!(cause.kind(), io::ErrorKind::InvalidInput);
-    }
-    // file is a directory
-    {
-        let path = tempdir.path().join("pathisdir.data");
-        std::fs::create_dir_all(&path).unwrap();
-        let cause = db
-            .import_file(path, ImportMode::Copy, BlobFormat::Raw, np())
-            .await
-            .unwrap_err();
-        assert_eq!(cause.kind(), io::ErrorKind::InvalidInput);
-    }
-}
-
-#[tokio::test]
-async fn import_file_tempdir_is_file() {
-    let np = IgnoreProgressSender::<ImportProgress>::default;
-    let (tempdir, db) = create_test_db().await;
-    // temp dir is readonly, this is a bit mean since we mess with the internals of the store
-    {
-        let temp_dir = db.0.temp_file_name().parent().unwrap().to_owned();
-        std::fs::remove_dir_all(&temp_dir).unwrap();
-        std::fs::write(temp_dir, []).unwrap();
-        // std::fs::set_permissions(temp_dir, std::os::unix::fs::PermissionsExt::from_mode(0o0))
-        //     .unwrap();
-        let path = tempdir.path().join("mid.data");
-        let data = random_test_data(MID_SIZE as usize);
-        std::fs::write(&path, &data).unwrap();
-        let _cause = db
-            .import_file(path, ImportMode::Copy, BlobFormat::Raw, np())
-            .await
-            .unwrap_err();
-        // cause is NotADirectory, but unstable
-        // assert_eq!(cause.kind(), io::ErrorKind::NotADirectory);
-    }
-    drop(tempdir);
-}
-
-#[tokio::test]
-async fn import_file_datadir_is_file() {
-    let np = IgnoreProgressSender::<ImportProgress>::default;
-    let (tempdir, db) = create_test_db().await;
-    // temp dir is readonly, this is a bit mean since we mess with the internals of the store
-    {
-        let data_dir = db.0.path_options.data_path.to_owned();
-        std::fs::remove_dir_all(&data_dir).unwrap();
-        std::fs::write(data_dir, []).unwrap();
-        let path = tempdir.path().join("mid.data");
-        let data = random_test_data(MID_SIZE as usize);
-        std::fs::write(&path, &data).unwrap();
-        let _cause = db
-            .import_file(path, ImportMode::Copy, BlobFormat::Raw, np())
-            .await
-            .unwrap_err();
-        // cause is NotADirectory, but unstable
-        // assert_eq!(cause.kind(), io::ErrorKind::NotADirectory);
-    }
-    drop(tempdir);
-}
-
-/// tests that owned wins over external in both cases
-#[tokio::test]
-async fn import_file_overwrite() {
-    let np = IgnoreProgressSender::<ImportProgress>::default;
-    let (tempdir, db) = create_test_db().await;
-    // overwrite external with owned
-    {
-        let path = tempdir.path().join("mid.data");
-        let data = random_test_data(MID_SIZE as usize);
-        let (_outboard, hash) = raw_outboard(&data);
-        std::fs::write(&path, &data).unwrap();
-        let (tt1, size1) = db
-            .import_file(path.clone(), ImportMode::Copy, BlobFormat::Raw, np())
-            .await
-            .unwrap();
-        assert_eq!(size1, MID_SIZE);
-        assert_eq!(tt1.hash(), &hash);
-        let state = db.entry_state(hash).await.unwrap();
-        assert_matches!(
-            state.db,
-            Some(EntryState::Complete {
-                data_location: DataLocation::Owned(_),
-                ..
-            })
-        );
-        let (tt2, size2) = db
-            .import_file(path, ImportMode::TryReference, BlobFormat::Raw, np())
-            .await
-            .unwrap();
-        assert_eq!(size2, MID_SIZE);
-        assert_eq!(tt2.hash(), &hash);
-        let state = db.entry_state(hash).await.unwrap();
-        assert_matches!(
-            state.db,
-            Some(EntryState::Complete {
-                data_location: DataLocation::Owned(_),
-                ..
-            })
-        );
-    }
-    {
-        let path = tempdir.path().join("mid2.data");
-        let data = random_test_data(MID_SIZE as usize);
-        let (_outboard, hash) = raw_outboard(&data);
-        std::fs::write(&path, &data).unwrap();
-        let (tt1, size1) = db
-            .import_file(
-                path.clone(),
-                ImportMode::TryReference,
-                BlobFormat::Raw,
-                np(),
-            )
-            .await
-            .unwrap();
-        let state = db.entry_state(hash).await.unwrap();
-        assert_eq!(size1, MID_SIZE);
-        assert_eq!(tt1.hash(), &hash);
-        assert_matches!(
-            state.db,
-            Some(EntryState::Complete {
-                data_location: DataLocation::External(_, _),
-                ..
-            })
-        );
-        let (tt2, size2) = db
-            .import_file(path, ImportMode::Copy, BlobFormat::Raw, np())
-            .await
-            .unwrap();
-        let state = db.entry_state(hash).await.unwrap();
-        assert_eq!(size2, MID_SIZE);
-        assert_eq!(tt2.hash(), &hash);
-        assert_matches!(
-            state.db,
-            Some(EntryState::Complete {
-                data_location: DataLocation::Owned(_),
-                ..
-            })
-        );
-    }
-}
-
-/// tests that export works in copy mode
-#[tokio::test]
-async fn export_copy_cases() {
-    let np = || Box::new(|_: u64| io::Result::Ok(()));
-    let (tempdir, db) = create_test_db().await;
-    let small = random_test_data(SMALL_SIZE as usize);
-    let mid = random_test_data(MID_SIZE as usize);
-    let large = random_test_data(LARGE_SIZE as usize);
-    let small_tt = db
-        .import_bytes(small.clone().into(), BlobFormat::Raw)
-        .await
-        .unwrap();
-    let mid_tt = db
-        .import_bytes(mid.clone().into(), BlobFormat::Raw)
-        .await
-        .unwrap();
-    let large_tt = db
-        .import_bytes(large.clone().into(), BlobFormat::Raw)
-        .await
-        .unwrap();
-    let small_path = tempdir.path().join("small.data");
-    let mid_path = tempdir.path().join("mid.data");
-    let large_path = tempdir.path().join("large.data");
-    db.export(*small_tt.hash(), small_path.clone(), ExportMode::Copy, np())
-        .await
-        .unwrap();
-    assert_eq!(small.to_vec(), std::fs::read(&small_path).unwrap());
-    db.export(*mid_tt.hash(), mid_path.clone(), ExportMode::Copy, np())
-        .await
-        .unwrap();
-    assert_eq!(mid.to_vec(), std::fs::read(&mid_path).unwrap());
-    db.export(*large_tt.hash(), large_path.clone(), ExportMode::Copy, np())
-        .await
-        .unwrap();
-    assert_eq!(large.to_vec(), std::fs::read(&large_path).unwrap());
-    let state = db.entry_state(*small_tt.hash()).await.unwrap();
-    assert_eq!(
-        state.db,
-        Some(EntryState::Complete {
-            data_location: DataLocation::Inline(small),
-            outboard_location: OutboardLocation::NotNeeded,
-        })
-    );
-    let state = db.entry_state(*mid_tt.hash()).await.unwrap();
-    assert_eq!(
-        state.db,
-        Some(EntryState::Complete {
-            data_location: DataLocation::Owned(MID_SIZE),
-            outboard_location: OutboardLocation::Inline(raw_outboard(&mid).0),
-        })
-    );
-    let state = db.entry_state(*large_tt.hash()).await.unwrap();
-    assert_eq!(
-        state.db,
-        Some(EntryState::Complete {
-            data_location: DataLocation::Owned(LARGE_SIZE),
-            outboard_location: OutboardLocation::Owned,
-        })
-    );
-}
-
-/// tests that export works in reference mode
-#[tokio::test]
-async fn export_reference_cases() {
-    let np = || Box::new(|_: u64| io::Result::Ok(()));
-    let (tempdir, db) = create_test_db().await;
-    let small = random_test_data(SMALL_SIZE as usize);
-    let mid = random_test_data(MID_SIZE as usize);
-    let large = random_test_data(LARGE_SIZE as usize);
-    let small_tt = db
-        .import_bytes(small.clone().into(), BlobFormat::Raw)
-        .await
-        .unwrap();
-    let mid_tt = db
-        .import_bytes(mid.clone().into(), BlobFormat::Raw)
-        .await
-        .unwrap();
-    let large_tt = db
-        .import_bytes(large.clone().into(), BlobFormat::Raw)
-        .await
-        .unwrap();
-    let small_path = tempdir.path().join("small.data");
-    let mid_path = tempdir.path().join("mid.data");
-    let large_path = tempdir.path().join("large.data");
-    db.export(
-        *small_tt.hash(),
-        small_path.clone(),
-        ExportMode::TryReference,
-        np(),
-    )
-    .await
-    .unwrap();
-    assert_eq!(small.to_vec(), std::fs::read(&small_path).unwrap());
-    db.export(
-        *mid_tt.hash(),
-        mid_path.clone(),
-        ExportMode::TryReference,
-        np(),
-    )
-    .await
-    .unwrap();
-    assert_eq!(mid.to_vec(), std::fs::read(&mid_path).unwrap());
-    db.export(
-        *large_tt.hash(),
-        large_path.clone(),
-        ExportMode::TryReference,
-        np(),
-    )
-    .await
-    .unwrap();
-    assert_eq!(large.to_vec(), std::fs::read(&large_path).unwrap());
-    let state = db.entry_state(*small_tt.hash()).await.unwrap();
-    // small entries will never use external references
-    assert_eq!(
-        state.db,
-        Some(EntryState::Complete {
-            data_location: DataLocation::Inline(small),
-            outboard_location: OutboardLocation::NotNeeded,
-        })
-    );
-    // mid entries should now use external references
-    let state = db.entry_state(*mid_tt.hash()).await.unwrap();
-    assert_eq!(
-        state.db,
-        Some(EntryState::Complete {
-            data_location: DataLocation::External(vec![mid_path], MID_SIZE),
-            outboard_location: OutboardLocation::Inline(raw_outboard(&mid).0),
-        })
-    );
-    // large entries should now use external references
-    let state = db.entry_state(*large_tt.hash()).await.unwrap();
-    assert_eq!(
-        state.db,
-        Some(EntryState::Complete {
-            data_location: DataLocation::External(vec![large_path], LARGE_SIZE),
-            outboard_location: OutboardLocation::Owned,
-        })
-    );
-}
-
-#[tokio::test]
-async fn actor_store_smoke() {
-    let testdir = tempfile::tempdir().unwrap();
-    let db_path = testdir.path().join("test.redb");
-    let options = Options {
-        path: PathOptions::new(testdir.path()),
-        batch: Default::default(),
-        inline: Default::default(),
-    };
-    let db = Store::new(db_path, options).await.unwrap();
-    db.dump().await.unwrap();
-    let data = random_test_data(1024 * 1024);
-    #[allow(clippy::single_range_in_vec_init)]
-    let ranges = [0..data.len() as u64];
-    let (hash, chunk_ranges, wire_data) = make_wire_data(&data, &ranges);
-    let handle = db.get_or_create(hash, 0).await.unwrap();
-    decode_response_into_batch(
-        hash,
-        IROH_BLOCK_SIZE,
-        chunk_ranges.clone(),
-        Cursor::new(wire_data.as_slice()),
-        handle.batch_writer().await.unwrap(),
-    )
-    .await
-    .unwrap();
-    validate(&handle, &data, &ranges).await;
-    db.insert_complete(handle).await.unwrap();
-    db.sync().await.unwrap();
-    db.dump().await.unwrap();
-}
diff --git a/src/store/fs/util.rs b/src/store/fs/util.rs
index b747d9d7b..f2949a7cc 100644
--- a/src/store/fs/util.rs
+++ b/src/store/fs/util.rs
@@ -1,57 +1,17 @@
-use std::{
-    fs::OpenOptions,
-    io::{self, Write},
-    path::Path,
-};
+use std::future::Future;
 
-/// overwrite a file with the given data.
-///
-/// This is almost like `std::fs::write`, but it does not truncate the file.
-///
-/// So if you overwrite a file with less data than it had before, the file will
-/// still have the same size as before.
-///
-/// Also, if you overwrite a file with the same data as it had before, the
-/// file will be unchanged even if the overwrite operation is interrupted.
-pub fn overwrite_and_sync(path: &Path, data: &[u8]) -> io::Result<std::fs::File> {
-    tracing::trace!(
-        "overwriting file {} with {} bytes",
-        path.display(),
-        data.len()
-    );
-    // std::fs::create_dir_all(path.parent().unwrap()).unwrap();
-    // tracing::error!("{}", path.parent().unwrap().display());
-    // tracing::error!("{}", path.parent().unwrap().metadata().unwrap().is_dir());
-    let mut file = OpenOptions::new()
-        .write(true)
-        .create(true)
-        .truncate(false)
-        .open(path)?;
-    file.write_all(data)?;
-    // todo: figure out if it is safe to not sync here
-    file.sync_all()?;
-    Ok(file)
-}
-
-/// Read a file into memory and then delete it.
-pub fn read_and_remove(path: &Path) -> io::Result<Vec<u8>> {
-    let data = std::fs::read(path)?;
-    // todo: should we fail here or just log a warning?
-    // remove could fail e.g. on windows if the file is still open
-    std::fs::remove_file(path)?;
-    Ok(data)
-}
+use tokio::{select, sync::mpsc};
 
-/// A wrapper for a flume receiver that allows peeking at the next message.
+/// A wrapper for a tokio mpsc receiver that allows peeking at the next message.
 #[derive(Debug)]
-pub(super) struct PeekableFlumeReceiver<T> {
+pub struct PeekableReceiver<T> {
     msg: Option<T>,
-    recv: async_channel::Receiver<T>,
+    recv: mpsc::Receiver<T>,
 }
 
 #[allow(dead_code)]
-impl<T> PeekableFlumeReceiver<T> {
-    pub fn new(recv: async_channel::Receiver<T>) -> Self {
+impl<T> PeekableReceiver<T> {
+    pub fn new(recv: mpsc::Receiver<T>) -> Self {
         Self { msg: None, recv }
     }
 
@@ -59,11 +19,34 @@ impl<T> PeekableFlumeReceiver<T> {
     ///
     /// Will block if there are no messages.
     /// Returns None only if there are no more messages (sender is dropped).
+    ///
+    /// Cancel safe because the only async operation is the recv() call, which is cancel safe.
     pub async fn recv(&mut self) -> Option<T> {
         if let Some(msg) = self.msg.take() {
             return Some(msg);
         }
-        self.recv.recv().await.ok()
+        self.recv.recv().await
+    }
+
+    /// Receive the next message, but only if it passes the filter.
+    ///
+    /// Cancel safe because the only async operation is the [Self::recv] call, which is cancel safe.
+    pub async fn extract<U>(
+        &mut self,
+        f: impl Fn(T) -> std::result::Result<U, T>,
+        timeout: impl Future + Unpin,
+    ) -> Option<U> {
+        let msg = select! {
+            x = self.recv() => x?,
+            _ = timeout => return None,
+        };
+        match f(msg) {
+            Ok(u) => Some(u),
+            Err(msg) => {
+                self.msg = Some(msg);
+                None
+            }
+        }
     }
 
     /// Push back a message. This will only work if there is room for it.
diff --git a/src/store/fs/validate.rs b/src/store/fs/validate.rs
deleted file mode 100644
index ae1870471..000000000
--- a/src/store/fs/validate.rs
+++ /dev/null
@@ -1,492 +0,0 @@
-//! Validation of the store's contents.
-use std::collections::BTreeSet;
-
-use redb::ReadableTable;
-
-use super::{
-    raw_outboard_size, tables::Tables, ActorResult, ActorState, DataLocation, EntryState, Hash,
-    OutboardLocation,
-};
-use crate::{
-    store::{fs::tables::BaoFilePart, ConsistencyCheckProgress, ReportLevel},
-    util::progress::BoxedProgressSender,
-};
-
-impl ActorState {
-    //! This performs a full consistency check. Eventually it will also validate
-    //! file content again, but that part is not yet implemented.
-    //!
-    //! Currently the following checks are performed for complete entries:
-    //!
-    //! Check that the data in the entries table is consistent with the data in
-    //! the inline_data and inline_outboard tables.
-    //!
-    //! For every entry where data_location is inline, the inline_data table
-    //! must contain the data. For every entry where
-    //! data_location is not inline, the inline_data table must not contain data.
-    //! Instead, the data must exist as a file in the data directory or be
-    //! referenced to one or many external files.
-    //!
-    //! For every entry where outboard_location is inline, the inline_outboard
-    //! table must contain the outboard. For every entry where outboard_location
-    //! is not inline, the inline_outboard table must not contain data, and the
-    //! outboard must exist as a file in the data directory. Outboards are never
-    //! external.
-    //!
-    //! In addition to these consistency checks, it is checked that the size of
-    //! the outboard is consistent with the size of the data.
-    //!
-    //! For partial entries, it is checked that the data and outboard files
-    //! exist.
-    //!
-    //! In addition to the consistency checks, it is checked that there are no
-    //! orphaned or unexpected files in the data directory. Also, all entries of
-    //! all tables are dumped at trace level. This is helpful for debugging and
-    //! also ensures that the data can be read.
-    //!
-    //! Note that during validation, a set of all hashes will be kept in memory.
-    //! So to validate exceedingly large stores, the validation process will
-    //! consume a lot of memory.
-    //!
-    //! In addition, validation is a blocking operation that will make the store
-    //! unresponsive for the duration of the validation.
-    pub(super) fn consistency_check(
-        &mut self,
-        db: &redb::Database,
-        repair: bool,
-        progress: BoxedProgressSender<ConsistencyCheckProgress>,
-    ) -> ActorResult<()> {
-        use crate::util::progress::ProgressSender;
-        let mut invalid_entries = BTreeSet::new();
-        macro_rules! send {
-            ($level:expr, $entry:expr, $($arg:tt)*) => {
-                if let Err(_) = progress.blocking_send(ConsistencyCheckProgress::Update { message: format!($($arg)*), level: $level, entry: $entry }) {
-                    return Ok(());
-                }
-            };
-        }
-        macro_rules! trace {
-            ($($arg:tt)*) => {
-                send!(ReportLevel::Trace, None, $($arg)*)
-            };
-        }
-        macro_rules! info {
-            ($($arg:tt)*) => {
-                send!(ReportLevel::Info, None, $($arg)*)
-            };
-        }
-        macro_rules! warn {
-            ($($arg:tt)*) => {
-                send!(ReportLevel::Warn, None, $($arg)*)
-            };
-        }
-        macro_rules! entry_warn {
-            ($hash:expr, $($arg:tt)*) => {
-                send!(ReportLevel::Warn, Some($hash), $($arg)*)
-            };
-        }
-        macro_rules! entry_info {
-            ($hash:expr, $($arg:tt)*) => {
-                send!(ReportLevel::Info, Some($hash), $($arg)*)
-            };
-        }
-        macro_rules! error {
-            ($($arg:tt)*) => {
-                send!(ReportLevel::Error, None, $($arg)*)
-            };
-        }
-        macro_rules! entry_error {
-            ($hash:expr, $($arg:tt)*) => {
-                invalid_entries.insert($hash);
-                send!(ReportLevel::Error, Some($hash), $($arg)*)
-            };
-        }
-        let mut delete_after_commit = Default::default();
-        let txn = db.begin_write()?;
-        {
-            let mut tables = Tables::new(&txn, &mut delete_after_commit)?;
-            let blobs = &mut tables.blobs;
-            let inline_data = &mut tables.inline_data;
-            let inline_outboard = &mut tables.inline_outboard;
-            let tags = &mut tables.tags;
-            let mut orphaned_inline_data = BTreeSet::new();
-            let mut orphaned_inline_outboard = BTreeSet::new();
-            let mut orphaned_data = BTreeSet::new();
-            let mut orphaned_outboardard = BTreeSet::new();
-            let mut orphaned_sizes = BTreeSet::new();
-            // first, dump the entire data content at trace level
-            trace!("dumping blobs");
-            match blobs.iter() {
-                Ok(iter) => {
-                    for item in iter {
-                        match item {
-                            Ok((k, v)) => {
-                                let hash = k.value();
-                                let entry = v.value();
-                                trace!("blob {} -> {:?}", hash.to_hex(), entry);
-                            }
-                            Err(cause) => {
-                                error!("failed to access blob item: {}", cause);
-                            }
-                        }
-                    }
-                }
-                Err(cause) => {
-                    error!("failed to iterate blobs: {}", cause);
-                }
-            }
-            trace!("dumping inline_data");
-            match inline_data.iter() {
-                Ok(iter) => {
-                    for item in iter {
-                        match item {
-                            Ok((k, v)) => {
-                                let hash = k.value();
-                                let data = v.value();
-                                trace!("inline_data {} -> {:?}", hash.to_hex(), data.len());
-                            }
-                            Err(cause) => {
-                                error!("failed to access inline data item: {}", cause);
-                            }
-                        }
-                    }
-                }
-                Err(cause) => {
-                    error!("failed to iterate inline_data: {}", cause);
-                }
-            }
-            trace!("dumping inline_outboard");
-            match inline_outboard.iter() {
-                Ok(iter) => {
-                    for item in iter {
-                        match item {
-                            Ok((k, v)) => {
-                                let hash = k.value();
-                                let data = v.value();
-                                trace!("inline_outboard {} -> {:?}", hash.to_hex(), data.len());
-                            }
-                            Err(cause) => {
-                                error!("failed to access inline outboard item: {}", cause);
-                            }
-                        }
-                    }
-                }
-                Err(cause) => {
-                    error!("failed to iterate inline_outboard: {}", cause);
-                }
-            }
-            trace!("dumping tags");
-            match tags.iter() {
-                Ok(iter) => {
-                    for item in iter {
-                        match item {
-                            Ok((k, v)) => {
-                                let tag = k.value();
-                                let value = v.value();
-                                trace!("tags {} -> {:?}", tag, value);
-                            }
-                            Err(cause) => {
-                                error!("failed to access tag item: {}", cause);
-                            }
-                        }
-                    }
-                }
-                Err(cause) => {
-                    error!("failed to iterate tags: {}", cause);
-                }
-            }
-
-            // perform consistency check for each entry
-            info!("validating blobs");
-            // set of a all hashes that are referenced by the blobs table
-            let mut entries = BTreeSet::new();
-            match blobs.iter() {
-                Ok(iter) => {
-                    for item in iter {
-                        let Ok((hash, entry)) = item else {
-                            error!("failed to access blob item");
-                            continue;
-                        };
-                        let hash = hash.value();
-                        entries.insert(hash);
-                        entry_info!(hash, "validating blob");
-                        let entry = entry.value();
-                        match entry {
-                            EntryState::Complete {
-                                data_location,
-                                outboard_location,
-                            } => {
-                                let data_size = match data_location {
-                                    DataLocation::Inline(_) => {
-                                        let Ok(inline_data) = inline_data.get(hash) else {
-                                            entry_error!(hash, "inline data can not be accessed");
-                                            continue;
-                                        };
-                                        let Some(inline_data) = inline_data else {
-                                            entry_error!(hash, "inline data missing");
-                                            continue;
-                                        };
-                                        inline_data.value().len() as u64
-                                    }
-                                    DataLocation::Owned(size) => {
-                                        let path = self.options.path.owned_data_path(&hash);
-                                        let Ok(metadata) = path.metadata() else {
-                                            entry_error!(hash, "owned data file does not exist");
-                                            continue;
-                                        };
-                                        if metadata.len() != size {
-                                            entry_error!(
-                                                hash,
-                                                "owned data file size mismatch: {}",
-                                                path.display()
-                                            );
-                                            continue;
-                                        }
-                                        size
-                                    }
-                                    DataLocation::External(paths, size) => {
-                                        for path in paths {
-                                            let Ok(metadata) = path.metadata() else {
-                                                entry_error!(
-                                                    hash,
-                                                    "external data file does not exist: {}",
-                                                    path.display()
-                                                );
-                                                invalid_entries.insert(hash);
-                                                continue;
-                                            };
-                                            if metadata.len() != size {
-                                                entry_error!(
-                                                    hash,
-                                                    "external data file size mismatch: {}",
-                                                    path.display()
-                                                );
-                                                invalid_entries.insert(hash);
-                                                continue;
-                                            }
-                                        }
-                                        size
-                                    }
-                                };
-                                match outboard_location {
-                                    OutboardLocation::Inline(_) => {
-                                        let Ok(inline_outboard) = inline_outboard.get(hash) else {
-                                            entry_error!(
-                                                hash,
-                                                "inline outboard can not be accessed"
-                                            );
-                                            continue;
-                                        };
-                                        let Some(inline_outboard) = inline_outboard else {
-                                            entry_error!(hash, "inline outboard missing");
-                                            continue;
-                                        };
-                                        let outboard_size = inline_outboard.value().len() as u64;
-                                        if outboard_size != raw_outboard_size(data_size) {
-                                            entry_error!(hash, "inline outboard size mismatch");
-                                        }
-                                    }
-                                    OutboardLocation::Owned => {
-                                        let Ok(metadata) =
-                                            self.options.path.owned_outboard_path(&hash).metadata()
-                                        else {
-                                            entry_error!(
-                                                hash,
-                                                "owned outboard file does not exist"
-                                            );
-                                            continue;
-                                        };
-                                        let outboard_size = metadata.len();
-                                        if outboard_size != raw_outboard_size(data_size) {
-                                            entry_error!(hash, "owned outboard size mismatch");
-                                        }
-                                    }
-                                    OutboardLocation::NotNeeded => {
-                                        if raw_outboard_size(data_size) != 0 {
-                                            entry_error!(
-                                                hash,
-                                                "outboard not needed but data size is not zero"
-                                            );
-                                        }
-                                    }
-                                }
-                            }
-                            EntryState::Partial { .. } => {
-                                if !self.options.path.owned_data_path(&hash).exists() {
-                                    entry_error!(hash, "persistent partial entry has no data");
-                                }
-                                if !self.options.path.owned_outboard_path(&hash).exists() {
-                                    entry_error!(hash, "persistent partial entry has no outboard");
-                                }
-                            }
-                        }
-                    }
-                }
-                Err(cause) => {
-                    error!("failed to iterate blobs: {}", cause);
-                }
-            };
-            if repair {
-                info!("repairing - removing invalid entries found so far");
-                for hash in &invalid_entries {
-                    blobs.remove(hash)?;
-                }
-            }
-            info!("checking for orphaned inline data");
-            match inline_data.iter() {
-                Ok(iter) => {
-                    for item in iter {
-                        let Ok((hash, _)) = item else {
-                            error!("failed to access inline data item");
-                            continue;
-                        };
-                        let hash = hash.value();
-                        if !entries.contains(&hash) {
-                            orphaned_inline_data.insert(hash);
-                            entry_error!(hash, "orphaned inline data");
-                        }
-                    }
-                }
-                Err(cause) => {
-                    error!("failed to iterate inline_data: {}", cause);
-                }
-            };
-            info!("checking for orphaned inline outboard data");
-            match inline_outboard.iter() {
-                Ok(iter) => {
-                    for item in iter {
-                        let Ok((hash, _)) = item else {
-                            error!("failed to access inline outboard item");
-                            continue;
-                        };
-                        let hash = hash.value();
-                        if !entries.contains(&hash) {
-                            orphaned_inline_outboard.insert(hash);
-                            entry_error!(hash, "orphaned inline outboard");
-                        }
-                    }
-                }
-                Err(cause) => {
-                    error!("failed to iterate inline_outboard: {}", cause);
-                }
-            };
-            info!("checking for unexpected or orphaned files");
-            for entry in self.options.path.data_path.read_dir()? {
-                let entry = entry?;
-                let path = entry.path();
-                if !path.is_file() {
-                    warn!("unexpected entry in data directory: {}", path.display());
-                    continue;
-                }
-                match path.extension().and_then(|x| x.to_str()) {
-                    Some("data") => match path.file_stem().and_then(|x| x.to_str()) {
-                        Some(stem) => {
-                            let mut hash = [0u8; 32];
-                            let Ok(_) = hex::decode_to_slice(stem, &mut hash) else {
-                                warn!("unexpected data file in data directory: {}", path.display());
-                                continue;
-                            };
-                            let hash = Hash::from(hash);
-                            if !entries.contains(&hash) {
-                                orphaned_data.insert(hash);
-                                entry_warn!(hash, "orphaned data file");
-                            }
-                        }
-                        None => {
-                            warn!("unexpected data file in data directory: {}", path.display());
-                        }
-                    },
-                    Some("obao4") => match path.file_stem().and_then(|x| x.to_str()) {
-                        Some(stem) => {
-                            let mut hash = [0u8; 32];
-                            let Ok(_) = hex::decode_to_slice(stem, &mut hash) else {
-                                warn!(
-                                    "unexpected outboard file in data directory: {}",
-                                    path.display()
-                                );
-                                continue;
-                            };
-                            let hash = Hash::from(hash);
-                            if !entries.contains(&hash) {
-                                orphaned_outboardard.insert(hash);
-                                entry_warn!(hash, "orphaned outboard file");
-                            }
-                        }
-                        None => {
-                            warn!(
-                                "unexpected outboard file in data directory: {}",
-                                path.display()
-                            );
-                        }
-                    },
-                    Some("sizes4") => match path.file_stem().and_then(|x| x.to_str()) {
-                        Some(stem) => {
-                            let mut hash = [0u8; 32];
-                            let Ok(_) = hex::decode_to_slice(stem, &mut hash) else {
-                                warn!(
-                                    "unexpected outboard file in data directory: {}",
-                                    path.display()
-                                );
-                                continue;
-                            };
-                            let hash = Hash::from(hash);
-                            if !entries.contains(&hash) {
-                                orphaned_sizes.insert(hash);
-                                entry_warn!(hash, "orphaned outboard file");
-                            }
-                        }
-                        None => {
-                            warn!(
-                                "unexpected outboard file in data directory: {}",
-                                path.display()
-                            );
-                        }
-                    },
-                    _ => {
-                        warn!("unexpected file in data directory: {}", path.display());
-                    }
-                }
-            }
-            if repair {
-                info!("repairing - removing orphaned files and inline data");
-                for hash in orphaned_inline_data {
-                    entry_info!(hash, "deleting orphaned inline data");
-                    inline_data.remove(&hash)?;
-                }
-                for hash in orphaned_inline_outboard {
-                    entry_info!(hash, "deleting orphaned inline outboard");
-                    inline_outboard.remove(&hash)?;
-                }
-                for hash in orphaned_data {
-                    tables.delete_after_commit.insert(hash, [BaoFilePart::Data]);
-                }
-                for hash in orphaned_outboardard {
-                    tables
-                        .delete_after_commit
-                        .insert(hash, [BaoFilePart::Outboard]);
-                }
-                for hash in orphaned_sizes {
-                    tables
-                        .delete_after_commit
-                        .insert(hash, [BaoFilePart::Sizes]);
-                }
-            }
-        }
-        txn.commit()?;
-        if repair {
-            info!("repairing - deleting orphaned files");
-            for (hash, part) in delete_after_commit.into_inner() {
-                let path = match part {
-                    BaoFilePart::Data => self.options.path.owned_data_path(&hash),
-                    BaoFilePart::Outboard => self.options.path.owned_outboard_path(&hash),
-                    BaoFilePart::Sizes => self.options.path.owned_sizes_path(&hash),
-                };
-                entry_info!(hash, "deleting orphaned file: {}", path.display());
-                if let Err(cause) = std::fs::remove_file(&path) {
-                    entry_error!(hash, "failed to delete orphaned file: {}", cause);
-                }
-            }
-        }
-        Ok(())
-    }
-}
diff --git a/src/store/mem.rs b/src/store/mem.rs
index 105b8ddd5..df9783ab2 100644
--- a/src/store/mem.rs
+++ b/src/store/mem.rs
@@ -1,510 +1,1046 @@
-//! A full in memory database for iroh-blobs
+//! Mutable in-memory blob store.
 //!
-//! Main entry point is [Store].
+//! Being a memory store, this store has to import all data into memory before it can
+//! serve it. So the amount of data you can serve is limited by your available memory.
+//! Other than that this is a fully featured store that provides all features such as
+//! tags and garbage collection.
+//!
+//! For many use cases this can be quite useful, since it does not require write access
+//! to the file system.
 use std::{
-    collections::{BTreeMap, BTreeSet},
+    collections::{BTreeMap, HashMap, HashSet},
     future::Future,
-    io,
-    path::PathBuf,
-    sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard},
+    io::{self, Write},
+    num::NonZeroU64,
+    ops::Deref,
+    sync::Arc,
     time::SystemTime,
 };
 
 use bao_tree::{
-    io::{fsm::Outboard, outboard::PreOrderOutboard, sync::WriteAt},
-    BaoTree,
+    blake3,
+    io::{
+        mixed::{traverse_ranges_validated, EncodedItem, ReadBytesAt},
+        outboard::PreOrderMemOutboard,
+        sync::{Outboard, ReadAt, WriteAt},
+        BaoContentItem, Leaf,
+    },
+    BaoTree, ChunkNum, ChunkRanges, TreeNode,
 };
-use bytes::{Bytes, BytesMut};
-use futures_lite::{Stream, StreamExt};
-use iroh_io::AsyncSliceReader;
-use tracing::info;
-
-use super::{
-    temp_name, BaoBatchWriter, ConsistencyCheckProgress, ExportMode, ExportProgressCb, ImportMode,
-    ImportProgress, Map, TempCounterMap,
+use bytes::Bytes;
+use irpc::channel::mpsc;
+use n0_future::future::yield_now;
+use range_collections::range_set::RangeSetRange;
+use tokio::{
+    io::AsyncReadExt,
+    sync::watch,
+    task::{JoinError, JoinSet},
 };
+use tracing::{error, info, instrument, trace, Instrument};
+
+use super::util::{BaoTreeSender, PartialMemStorage};
 use crate::{
+    api::{
+        self,
+        blobs::{AddProgressItem, Bitfield, BlobStatus, ExportProgressItem},
+        proto::{
+            BatchMsg, BatchResponse, BlobDeleteRequest, BlobStatusMsg, BlobStatusRequest, Command,
+            CreateTagMsg, CreateTagRequest, CreateTempTagMsg, DeleteBlobsMsg, DeleteTagsMsg,
+            DeleteTagsRequest, ExportBaoMsg, ExportBaoRequest, ExportPathMsg, ExportPathRequest,
+            ExportRangesItem, ExportRangesMsg, ExportRangesRequest, ImportBaoMsg, ImportBaoRequest,
+            ImportByteStreamMsg, ImportByteStreamUpdate, ImportBytesMsg, ImportBytesRequest,
+            ImportPathMsg, ImportPathRequest, ListBlobsMsg, ListTagsMsg, ListTagsRequest,
+            ObserveMsg, ObserveRequest, RenameTagMsg, RenameTagRequest, Scope, SetTagMsg,
+            SetTagRequest, ShutdownMsg, SyncDbMsg,
+        },
+        tags::TagInfo,
+        ApiClient,
+    },
     store::{
-        mutable_mem_storage::MutableMemStorage, BaoBlobSize, MapEntry, MapEntryMut, ReadableStore,
+        util::{SizeInfo, SparseMemFile, Tag},
+        HashAndFormat, IROH_BLOCK_SIZE,
     },
     util::{
-        progress::{BoxedProgressSender, IdGenerator, IgnoreProgressSender, ProgressSender},
-        TagCounter, TagDrop,
+        temp_tag::{TagDrop, TempTagScope, TempTags},
+        ChunkRangesExt,
     },
-    BlobFormat, Hash, HashAndFormat, Tag, TempTag, IROH_BLOCK_SIZE,
+    BlobFormat, Hash,
 };
 
-/// A fully featured in memory database for iroh-blobs, including support for
-/// partial blobs.
-#[derive(Debug, Clone, Default)]
-pub struct Store {
-    inner: Arc<StoreInner>,
+#[derive(Debug, Default)]
+pub struct Options {}
+
+#[derive(Debug, Clone)]
+#[repr(transparent)]
+pub struct MemStore {
+    client: ApiClient,
 }
 
-#[derive(Debug, Default)]
-struct StoreInner(RwLock<StateInner>);
+impl AsRef<crate::api::Store> for MemStore {
+    fn as_ref(&self) -> &crate::api::Store {
+        crate::api::Store::ref_from_sender(&self.client)
+    }
+}
+
+impl Deref for MemStore {
+    type Target = crate::api::Store;
 
-impl TagDrop for StoreInner {
-    fn on_drop(&self, inner: &HashAndFormat) {
-        tracing::trace!("temp tag drop: {:?}", inner);
-        let mut state = self.0.write().unwrap();
-        state.temp.dec(inner);
+    fn deref(&self) -> &Self::Target {
+        crate::api::Store::ref_from_sender(&self.client)
     }
 }
 
-impl TagCounter for StoreInner {
-    fn on_create(&self, inner: &HashAndFormat) {
-        tracing::trace!("temp tagging: {:?}", inner);
-        let mut state = self.0.write().unwrap();
-        state.temp.inc(inner);
+impl Default for MemStore {
+    fn default() -> Self {
+        Self::new()
     }
 }
 
-impl Store {
-    /// Create a new in memory store
+#[derive(derive_more::From)]
+enum TaskResult {
+    Unit(()),
+    Import(anyhow::Result<ImportEntry>),
+    Scope(Scope),
+}
+
+impl MemStore {
+    pub fn from_sender(client: ApiClient) -> Self {
+        Self { client }
+    }
+
     pub fn new() -> Self {
-        Self::default()
+        let (sender, receiver) = tokio::sync::mpsc::channel(32);
+        tokio::spawn(
+            Actor {
+                commands: receiver,
+                tasks: JoinSet::new(),
+                state: State {
+                    data: HashMap::new(),
+                    tags: BTreeMap::new(),
+                },
+                options: Arc::new(Options::default()),
+                temp_tags: Default::default(),
+                protected: Default::default(),
+            }
+            .run(),
+        );
+        Self::from_sender(sender.into())
     }
+}
 
-    /// Take a write lock on the store
-    fn write_lock(&self) -> RwLockWriteGuard<'_, StateInner> {
-        self.inner.0.write().unwrap()
+struct Actor {
+    commands: tokio::sync::mpsc::Receiver<Command>,
+    tasks: JoinSet<TaskResult>,
+    state: State,
+    #[allow(dead_code)]
+    options: Arc<Options>,
+    // temp tags
+    temp_tags: TempTags,
+    protected: HashSet<Hash>,
+}
+
+impl Actor {
+    fn spawn<F, T>(&mut self, f: F)
+    where
+        F: Future<Output = T> + Send + 'static,
+        T: Into<TaskResult>,
+    {
+        let span = tracing::Span::current();
+        let fut = async move { f.await.into() }.instrument(span);
+        self.tasks.spawn(fut);
     }
 
-    /// Take a read lock on the store
-    fn read_lock(&self) -> RwLockReadGuard<'_, StateInner> {
-        self.inner.0.read().unwrap()
+    async fn handle_command(&mut self, cmd: Command) -> Option<ShutdownMsg> {
+        match cmd {
+            Command::ImportBao(ImportBaoMsg {
+                inner: ImportBaoRequest { hash, size },
+                rx: data,
+                tx,
+                ..
+            }) => {
+                let entry = self.get_or_create_entry(hash);
+                self.spawn(import_bao(entry, size, data, tx));
+            }
+            Command::Observe(ObserveMsg {
+                inner: ObserveRequest { hash },
+                tx,
+                ..
+            }) => {
+                let entry = self.get_or_create_entry(hash);
+                self.spawn(observe(entry, tx));
+            }
+            Command::ImportBytes(ImportBytesMsg {
+                inner:
+                    ImportBytesRequest {
+                        data,
+                        scope,
+                        format,
+                        ..
+                    },
+                tx,
+                ..
+            }) => {
+                self.spawn(import_bytes(data, scope, format, tx));
+            }
+            Command::ImportByteStream(ImportByteStreamMsg { inner, tx, rx, .. }) => {
+                self.spawn(import_byte_stream(inner.scope, inner.format, rx, tx));
+            }
+            Command::ImportPath(cmd) => {
+                self.spawn(import_path(cmd));
+            }
+            Command::ExportBao(ExportBaoMsg {
+                inner: ExportBaoRequest { hash, ranges },
+                tx,
+                ..
+            }) => {
+                let entry = self.get_or_create_entry(hash);
+                self.spawn(export_bao(entry, ranges, tx));
+            }
+            Command::ExportPath(cmd) => {
+                let entry = self.state.data.get(&cmd.hash).cloned();
+                self.spawn(export_path(entry, cmd));
+            }
+            Command::DeleteTags(cmd) => {
+                let DeleteTagsMsg {
+                    inner: DeleteTagsRequest { from, to },
+                    tx,
+                    ..
+                } = cmd;
+                info!("deleting tags from {:?} to {:?}", from, to);
+                // state.tags.remove(&from.unwrap());
+                // todo: more efficient impl
+                self.state.tags.retain(|tag, _| {
+                    if let Some(from) = &from {
+                        if tag < from {
+                            return true;
+                        }
+                    }
+                    if let Some(to) = &to {
+                        if tag >= to {
+                            return true;
+                        }
+                    }
+                    info!("    removing {:?}", tag);
+                    false
+                });
+                tx.send(Ok(())).await.ok();
+            }
+            Command::RenameTag(cmd) => {
+                let RenameTagMsg {
+                    inner: RenameTagRequest { from, to },
+                    tx,
+                    ..
+                } = cmd;
+                let tags = &mut self.state.tags;
+                let value = match tags.remove(&from) {
+                    Some(value) => value,
+                    None => {
+                        tx.send(Err(api::Error::io(
+                            io::ErrorKind::NotFound,
+                            format!("tag not found: {from:?}"),
+                        )))
+                        .await
+                        .ok();
+                        return None;
+                    }
+                };
+                tags.insert(to, value);
+                tx.send(Ok(())).await.ok();
+                return None;
+            }
+            Command::ListTags(cmd) => {
+                let ListTagsMsg {
+                    inner:
+                        ListTagsRequest {
+                            from,
+                            to,
+                            raw,
+                            hash_seq,
+                        },
+                    tx,
+                    ..
+                } = cmd;
+                let tags = self
+                    .state
+                    .tags
+                    .iter()
+                    .filter(move |(tag, value)| {
+                        if let Some(from) = &from {
+                            if tag < &from {
+                                return false;
+                            }
+                        }
+                        if let Some(to) = &to {
+                            if tag >= &to {
+                                return false;
+                            }
+                        }
+                        raw && value.format.is_raw() || hash_seq && value.format.is_hash_seq()
+                    })
+                    .map(|(tag, value)| TagInfo {
+                        name: tag.clone(),
+                        hash: value.hash,
+                        format: value.format,
+                    })
+                    .map(Ok);
+                tx.send(tags.collect()).await.ok();
+            }
+            Command::SetTag(SetTagMsg {
+                inner: SetTagRequest { name: tag, value },
+                tx,
+                ..
+            }) => {
+                self.state.tags.insert(tag, value);
+                tx.send(Ok(())).await.ok();
+            }
+            Command::CreateTag(CreateTagMsg {
+                inner: CreateTagRequest { value },
+                tx,
+                ..
+            }) => {
+                let tag = Tag::auto(SystemTime::now(), |tag| self.state.tags.contains_key(tag));
+                self.state.tags.insert(tag.clone(), value);
+                tx.send(Ok(tag)).await.ok();
+            }
+            Command::CreateTempTag(cmd) => {
+                trace!("{cmd:?}");
+                self.create_temp_tag(cmd).await;
+            }
+            Command::ListTempTags(cmd) => {
+                trace!("{cmd:?}");
+                let tts = self.temp_tags.list();
+                cmd.tx.send(tts).await.ok();
+            }
+            Command::ListBlobs(cmd) => {
+                let ListBlobsMsg { tx, .. } = cmd;
+                let blobs = self.state.data.keys().cloned().collect::<Vec<Hash>>();
+                self.spawn(async move {
+                    for blob in blobs {
+                        if tx.send(Ok(blob)).await.is_err() {
+                            break;
+                        }
+                    }
+                });
+            }
+            Command::BlobStatus(cmd) => {
+                trace!("{cmd:?}");
+                let BlobStatusMsg {
+                    inner: BlobStatusRequest { hash },
+                    tx,
+                    ..
+                } = cmd;
+                let res = match self.state.data.get(&hash) {
+                    None => api::blobs::BlobStatus::NotFound,
+                    Some(x) => {
+                        let bitfield = x.0.state.borrow().bitfield();
+                        if bitfield.is_complete() {
+                            BlobStatus::Complete {
+                                size: bitfield.size,
+                            }
+                        } else {
+                            BlobStatus::Partial {
+                                size: bitfield.validated_size(),
+                            }
+                        }
+                    }
+                };
+                tx.send(res).await.ok();
+            }
+            Command::DeleteBlobs(cmd) => {
+                trace!("{cmd:?}");
+                let DeleteBlobsMsg {
+                    inner: BlobDeleteRequest { hashes, force },
+                    tx,
+                    ..
+                } = cmd;
+                for hash in hashes {
+                    if !force && self.protected.contains(&hash) {
+                        continue;
+                    }
+                    self.state.data.remove(&hash);
+                }
+                tx.send(Ok(())).await.ok();
+            }
+            Command::Batch(cmd) => {
+                trace!("{cmd:?}");
+                let (id, scope) = self.temp_tags.create_scope();
+                self.spawn(handle_batch(cmd, id, scope));
+            }
+            Command::ClearProtected(cmd) => {
+                self.protected.clear();
+                cmd.tx.send(Ok(())).await.ok();
+            }
+            Command::ExportRanges(cmd) => {
+                let entry = self.get_or_create_entry(cmd.hash);
+                self.spawn(export_ranges(cmd, entry.clone()));
+            }
+            Command::SyncDb(SyncDbMsg { tx, .. }) => {
+                tx.send(Ok(())).await.ok();
+            }
+            Command::Shutdown(cmd) => {
+                return Some(cmd);
+            }
+        }
+        None
+    }
+
+    fn get_or_create_entry(&mut self, hash: Hash) -> BaoFileHandle {
+        self.state
+            .data
+            .entry(hash)
+            .or_insert_with(|| BaoFileHandle::new_partial(hash))
+            .clone()
+    }
+
+    async fn create_temp_tag(&mut self, cmd: CreateTempTagMsg) {
+        let CreateTempTagMsg { tx, inner, .. } = cmd;
+        let mut tt = self.temp_tags.create(inner.scope, inner.value);
+        if tx.is_rpc() {
+            tt.leak();
+        }
+        tx.send(tt).await.ok();
     }
 
-    fn import_bytes_sync(
-        &self,
-        id: u64,
-        bytes: Bytes,
-        format: BlobFormat,
-        progress: impl ProgressSender<Msg = ImportProgress> + IdGenerator,
-    ) -> io::Result<TempTag> {
-        progress.blocking_send(ImportProgress::OutboardProgress { id, offset: 0 })?;
-        let progress2 = progress.clone();
-        let cb = move |offset| {
-            progress2
-                .try_send(ImportProgress::OutboardProgress { id, offset })
-                .ok();
+    async fn finish_import(&mut self, res: anyhow::Result<ImportEntry>) {
+        let import_data = match res {
+            Ok(entry) => entry,
+            Err(e) => {
+                error!("import failed: {e}");
+                return;
+            }
         };
-        let (storage, hash) = MutableMemStorage::complete(bytes, cb);
-        progress.blocking_send(ImportProgress::OutboardDone { id, hash })?;
-        use super::Store;
-        let tag = self.temp_tag(HashAndFormat { hash, format });
-        let entry = Entry {
-            inner: Arc::new(EntryInner {
+        let hash = import_data.outboard.root().into();
+        let entry = self.get_or_create_entry(hash);
+        entry
+            .0
+            .state
+            .send_if_modified(|state: &mut BaoFileStorage| {
+                let BaoFileStorage::Partial(_) = state.deref() else {
+                    return false;
+                };
+                *state =
+                    CompleteStorage::new(import_data.data, import_data.outboard.data.into()).into();
+                true
+            });
+        let tt = self.temp_tags.create(
+            import_data.scope,
+            HashAndFormat {
                 hash,
-                data: RwLock::new(storage),
-            }),
-            complete: true,
-        };
-        self.write_lock().entries.insert(hash, entry);
-        Ok(tag)
-    }
-
-    fn export_sync(
-        &self,
-        hash: Hash,
-        target: PathBuf,
-        _mode: ExportMode,
-        progress: impl Fn(u64) -> io::Result<()> + Send + Sync + 'static,
-    ) -> io::Result<()> {
-        tracing::trace!("exporting {} to {}", hash, target.display());
-
-        if !target.is_absolute() {
-            return Err(io::Error::new(
-                io::ErrorKind::InvalidInput,
-                "target path must be absolute",
-            ));
+                format: import_data.format,
+            },
+        );
+        import_data.tx.send(AddProgressItem::Done(tt)).await.ok();
+    }
+
+    fn log_task_result(&self, res: Result<TaskResult, JoinError>) -> Option<TaskResult> {
+        match res {
+            Ok(x) => Some(x),
+            Err(e) => {
+                if e.is_cancelled() {
+                    trace!("task cancelled: {e}");
+                } else {
+                    error!("task failed: {e}");
+                }
+                None
+            }
         }
-        let parent = target.parent().ok_or_else(|| {
-            io::Error::new(
-                io::ErrorKind::InvalidInput,
-                "target path has no parent directory",
-            )
-        })?;
-        // create the directory in which the target file is
-        std::fs::create_dir_all(parent)?;
-        let state = self.read_lock();
-        let entry = state
-            .entries
-            .get(&hash)
-            .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "hash not found"))?;
-        let reader = &entry.inner.data;
-        let size = reader.read().unwrap().current_size();
-        let mut file = std::fs::File::create(target)?;
-        for offset in (0..size).step_by(1024 * 1024) {
-            let bytes = reader.read().unwrap().read_data_at(offset, 1024 * 1024);
-            file.write_at(offset, &bytes)?;
-            progress(offset)?;
+    }
+
+    pub async fn run(mut self) {
+        let shutdown = loop {
+            tokio::select! {
+                cmd = self.commands.recv() => {
+                    let Some(cmd) = cmd else {
+                        // last sender has been dropped.
+                        // exit immediately.
+                        break None;
+                    };
+                    if let Some(cmd) = self.handle_command(cmd).await {
+                        break Some(cmd);
+                    }
+                }
+                Some(res) = self.tasks.join_next(), if !self.tasks.is_empty() => {
+                    let Some(res) = self.log_task_result(res) else {
+                        continue;
+                    };
+                    match res {
+                        TaskResult::Import(res) => {
+                            self.finish_import(res).await;
+                        }
+                        TaskResult::Scope(scope) => {
+                            self.temp_tags.end_scope(scope);
+                        }
+                        TaskResult::Unit(_) => {}
+                    }
+                }
+            }
+        };
+        if let Some(shutdown) = shutdown {
+            shutdown.tx.send(()).await.ok();
         }
-        std::io::Write::flush(&mut file)?;
-        drop(file);
-        Ok(())
     }
 }
 
-impl super::Store for Store {
-    async fn import_file(
-        &self,
-        path: std::path::PathBuf,
-        _mode: ImportMode,
-        format: BlobFormat,
-        progress: impl ProgressSender<Msg = ImportProgress> + IdGenerator,
-    ) -> io::Result<(TempTag, u64)> {
-        let this = self.clone();
-        tokio::task::spawn_blocking(move || {
-            let id = progress.new_id();
-            progress.blocking_send(ImportProgress::Found {
-                id,
-                name: path.to_string_lossy().to_string(),
-            })?;
-            progress.try_send(ImportProgress::CopyProgress { id, offset: 0 })?;
-            // todo: provide progress for reading into mem
-            let bytes: Bytes = std::fs::read(path)?.into();
-            let size = bytes.len() as u64;
-            progress.blocking_send(ImportProgress::Size { id, size })?;
-            let tag = this.import_bytes_sync(id, bytes, format, progress)?;
-            Ok((tag, size))
-        })
-        .await?
-    }
-
-    async fn import_stream(
-        &self,
-        mut data: impl Stream<Item = io::Result<Bytes>> + Unpin + Send + 'static,
-        format: BlobFormat,
-        progress: impl ProgressSender<Msg = ImportProgress> + IdGenerator,
-    ) -> io::Result<(TempTag, u64)> {
-        let this = self.clone();
-        let id = progress.new_id();
-        let name = temp_name();
-        progress.send(ImportProgress::Found { id, name }).await?;
-        let mut bytes = BytesMut::new();
-        while let Some(chunk) = data.next().await {
-            bytes.extend_from_slice(&chunk?);
-            progress
-                .try_send(ImportProgress::CopyProgress {
-                    id,
-                    offset: bytes.len() as u64,
-                })
-                .ok();
+async fn handle_batch(cmd: BatchMsg, id: Scope, scope: Arc<TempTagScope>) -> Scope {
+    if let Err(cause) = handle_batch_impl(cmd, id, &scope).await {
+        error!("batch failed: {cause}");
+    }
+    id
+}
+
+async fn handle_batch_impl(cmd: BatchMsg, id: Scope, scope: &Arc<TempTagScope>) -> api::Result<()> {
+    let BatchMsg { tx, mut rx, .. } = cmd;
+    trace!("created scope {}", id);
+    tx.send(id).await.map_err(api::Error::other)?;
+    while let Some(msg) = rx.recv().await? {
+        match msg {
+            BatchResponse::Drop(msg) => scope.on_drop(&msg),
+            BatchResponse::Ping => {}
         }
-        let bytes = bytes.freeze();
-        let size = bytes.len() as u64;
-        progress.blocking_send(ImportProgress::Size { id, size })?;
-        let tag = this.import_bytes_sync(id, bytes, format, progress)?;
-        Ok((tag, size))
-    }
-
-    async fn import_bytes(&self, bytes: Bytes, format: BlobFormat) -> io::Result<TempTag> {
-        let this = self.clone();
-        tokio::task::spawn_blocking(move || {
-            this.import_bytes_sync(0, bytes, format, IgnoreProgressSender::default())
-        })
-        .await?
-    }
-
-    async fn rename_tag(&self, from: Tag, to: Tag) -> io::Result<()> {
-        let mut state = self.write_lock();
-        let value = state.tags.remove(&from).ok_or_else(|| {
-            io::Error::new(
-                io::ErrorKind::NotFound,
-                format!("tag not found: {:?}", from),
-            )
-        })?;
-        state.tags.insert(to, value);
-        Ok(())
     }
+    Ok(())
+}
 
-    async fn set_tag(&self, name: Tag, value: HashAndFormat) -> io::Result<()> {
-        let mut state = self.write_lock();
-        state.tags.insert(name, value);
-        Ok(())
+async fn export_ranges(mut cmd: ExportRangesMsg, entry: BaoFileHandle) {
+    if let Err(cause) = export_ranges_impl(cmd.inner, &mut cmd.tx, entry).await {
+        cmd.tx
+            .send(ExportRangesItem::Error(cause.into()))
+            .await
+            .ok();
     }
+}
 
-    async fn delete_tags(&self, from: Option<Tag>, to: Option<Tag>) -> io::Result<()> {
-        let mut state = self.write_lock();
-        info!("deleting tags from {:?} to {:?}", from, to);
-        // state.tags.remove(&from.unwrap());
-        // todo: more efficient impl
-        state.tags.retain(|tag, _| {
-            if let Some(from) = &from {
-                if tag < from {
-                    return true;
+async fn export_ranges_impl(
+    cmd: ExportRangesRequest,
+    tx: &mut mpsc::Sender<ExportRangesItem>,
+    entry: BaoFileHandle,
+) -> io::Result<()> {
+    let ExportRangesRequest { ranges, hash } = cmd;
+    let bitfield = entry.bitfield();
+    trace!(
+        "exporting ranges: {hash} {ranges:?} size={}",
+        bitfield.size()
+    );
+    debug_assert!(entry.hash() == hash, "hash mismatch");
+    let data = entry.data_reader();
+    let size = bitfield.size();
+    for range in ranges.iter() {
+        let range = match range {
+            RangeSetRange::Range(range) => size.min(*range.start)..size.min(*range.end),
+            RangeSetRange::RangeFrom(range) => size.min(*range.start)..size,
+        };
+        let requested = ChunkRanges::bytes(range.start..range.end);
+        if !bitfield.ranges.is_superset(&requested) {
+            return Err(io::Error::other(format!(
+                "missing range: {requested:?}, present: {bitfield:?}",
+            )));
+        }
+        let bs = 1024;
+        let mut offset = range.start;
+        loop {
+            let end: u64 = (offset + bs).min(range.end);
+            let size = (end - offset) as usize;
+            tx.send(
+                Leaf {
+                    offset,
+                    data: data.read_bytes_at(offset, size)?,
                 }
+                .into(),
+            )
+            .await?;
+            offset = end;
+            if offset >= range.end {
+                break;
             }
-            if let Some(to) = &to {
-                if tag >= to {
-                    return true;
+        }
+    }
+    Ok(())
+}
+
+fn chunk_range(leaf: &Leaf) -> ChunkRanges {
+    let start = ChunkNum::chunks(leaf.offset);
+    let end = ChunkNum::chunks(leaf.offset + leaf.data.len() as u64);
+    (start..end).into()
+}
+
+async fn import_bao(
+    entry: BaoFileHandle,
+    size: NonZeroU64,
+    mut stream: mpsc::Receiver<BaoContentItem>,
+    tx: irpc::channel::oneshot::Sender<api::Result<()>>,
+) {
+    let size = size.get();
+    entry
+        .0
+        .state
+        .send_if_modified(|state: &mut BaoFileStorage| {
+            let BaoFileStorage::Partial(entry) = state else {
+                // entry was already completed, no need to write
+                return false;
+            };
+            entry.size.write(0, size);
+            false
+        });
+    let tree = BaoTree::new(size, IROH_BLOCK_SIZE);
+    while let Some(item) = stream.recv().await.unwrap() {
+        entry.0.state.send_if_modified(|state| {
+            let BaoFileStorage::Partial(partial) = state else {
+                // entry was already completed, no need to write
+                return false;
+            };
+            match item {
+                BaoContentItem::Parent(parent) => {
+                    if let Some(offset) = tree.pre_order_offset(parent.node) {
+                        let mut pair = [0u8; 64];
+                        pair[..32].copy_from_slice(parent.pair.0.as_bytes());
+                        pair[32..].copy_from_slice(parent.pair.1.as_bytes());
+                        partial
+                            .outboard
+                            .write_at(offset * 64, &pair)
+                            .expect("writing to mem can never fail");
+                    }
+                    false
+                }
+                BaoContentItem::Leaf(leaf) => {
+                    let start = leaf.offset;
+                    partial
+                        .data
+                        .write_at(start, &leaf.data)
+                        .expect("writing to mem can never fail");
+                    let added = chunk_range(&leaf);
+                    let update = partial.bitfield.update(&Bitfield::new(added.clone(), size));
+                    if update.new_state().complete {
+                        let data = std::mem::take(&mut partial.data);
+                        let outboard = std::mem::take(&mut partial.outboard);
+                        let data: Bytes = <Vec<u8>>::try_from(data).unwrap().into();
+                        let outboard: Bytes = <Vec<u8>>::try_from(outboard).unwrap().into();
+                        *state = CompleteStorage::new(data, outboard).into();
+                    }
+                    update.changed()
                 }
             }
-            info!("    removing {:?}", tag);
-            false
         });
-        Ok(())
     }
+    tx.send(Ok(())).await.ok();
+}
+
+#[instrument(skip_all, fields(hash = %entry.hash.fmt_short()))]
+async fn export_bao(
+    entry: BaoFileHandle,
+    ranges: ChunkRanges,
+    mut sender: mpsc::Sender<EncodedItem>,
+) {
+    let data = entry.data_reader();
+    let outboard = entry.outboard_reader();
+    let tx = BaoTreeSender::new(&mut sender);
+    traverse_ranges_validated(data, outboard, &ranges, tx)
+        .await
+        .ok();
+}
 
-    async fn create_tag(&self, hash: HashAndFormat) -> io::Result<Tag> {
-        let mut state = self.write_lock();
-        let tag = Tag::auto(SystemTime::now(), |x| state.tags.contains_key(x));
-        state.tags.insert(tag.clone(), hash);
-        Ok(tag)
+#[instrument(skip_all, fields(hash = %entry.hash.fmt_short()))]
+async fn observe(entry: BaoFileHandle, tx: mpsc::Sender<api::blobs::Bitfield>) {
+    entry.subscribe().forward(tx).await.ok();
+}
+
+async fn import_bytes(
+    data: Bytes,
+    scope: Scope,
+    format: BlobFormat,
+    tx: mpsc::Sender<AddProgressItem>,
+) -> anyhow::Result<ImportEntry> {
+    tx.send(AddProgressItem::Size(data.len() as u64)).await?;
+    tx.send(AddProgressItem::CopyDone).await?;
+    let outboard = PreOrderMemOutboard::create(&data, IROH_BLOCK_SIZE);
+    Ok(ImportEntry {
+        data,
+        outboard,
+        scope,
+        format,
+        tx,
+    })
+}
+
+async fn import_byte_stream(
+    scope: Scope,
+    format: BlobFormat,
+    mut rx: mpsc::Receiver<ImportByteStreamUpdate>,
+    tx: mpsc::Sender<AddProgressItem>,
+) -> anyhow::Result<ImportEntry> {
+    let mut res = Vec::new();
+    loop {
+        match rx.recv().await {
+            Ok(Some(ImportByteStreamUpdate::Bytes(data))) => {
+                res.extend_from_slice(&data);
+                tx.send(AddProgressItem::CopyProgress(res.len() as u64))
+                    .await?;
+            }
+            Ok(Some(ImportByteStreamUpdate::Done)) => {
+                break;
+            }
+            Ok(None) => {
+                return Err(api::Error::io(
+                    io::ErrorKind::UnexpectedEof,
+                    "byte stream ended unexpectedly",
+                )
+                .into());
+            }
+            Err(e) => {
+                return Err(e.into());
+            }
+        }
     }
+    import_bytes(res.into(), scope, format, tx).await
+}
 
-    fn temp_tag(&self, tag: HashAndFormat) -> TempTag {
-        self.inner.temp_tag(tag)
+#[instrument(skip_all, fields(path = %cmd.path.display()))]
+async fn import_path(cmd: ImportPathMsg) -> anyhow::Result<ImportEntry> {
+    let ImportPathMsg {
+        inner:
+            ImportPathRequest {
+                path,
+                scope,
+                format,
+                ..
+            },
+        tx,
+        ..
+    } = cmd;
+    let mut res = Vec::new();
+    let mut file = tokio::fs::File::open(path).await?;
+    let mut buf = [0u8; 1024 * 64];
+    loop {
+        let size = file.read(&mut buf).await?;
+        if size == 0 {
+            break;
+        }
+        res.extend_from_slice(&buf[..size]);
+        tx.send(AddProgressItem::CopyProgress(res.len() as u64))
+            .await?;
     }
+    import_bytes(res.into(), scope, format, tx).await
+}
 
-    async fn gc_run<G, Gut>(&self, config: super::GcConfig, protected_cb: G)
-    where
-        G: Fn() -> Gut,
-        Gut: Future<Output = BTreeSet<Hash>> + Send,
-    {
-        super::gc_run_loop(self, config, move || async { Ok(()) }, protected_cb).await
+#[instrument(skip_all, fields(hash = %cmd.hash.fmt_short(), path = %cmd.target.display()))]
+async fn export_path(entry: Option<BaoFileHandle>, cmd: ExportPathMsg) {
+    let ExportPathMsg { inner, mut tx, .. } = cmd;
+    let Some(entry) = entry else {
+        tx.send(ExportProgressItem::Error(api::Error::io(
+            io::ErrorKind::NotFound,
+            "hash not found",
+        )))
+        .await
+        .ok();
+        return;
+    };
+    match export_path_impl(entry, inner, &mut tx).await {
+        Ok(()) => tx.send(ExportProgressItem::Done).await.ok(),
+        Err(e) => tx.send(ExportProgressItem::Error(e.into())).await.ok(),
+    };
+}
+
+async fn export_path_impl(
+    entry: BaoFileHandle,
+    cmd: ExportPathRequest,
+    tx: &mut mpsc::Sender<ExportProgressItem>,
+) -> io::Result<()> {
+    let ExportPathRequest { target, .. } = cmd;
+    // todo: for partial entries make sure to only write the part that is actually present
+    let mut file = std::fs::File::create(target)?;
+    let size = entry.0.state.borrow().size();
+    tx.send(ExportProgressItem::Size(size)).await?;
+    let mut buf = [0u8; 1024 * 64];
+    for offset in (0..size).step_by(1024 * 64) {
+        let len = std::cmp::min(size - offset, 1024 * 64) as usize;
+        let buf = &mut buf[..len];
+        entry.0.state.borrow().data().read_exact_at(offset, buf)?;
+        file.write_all(buf)?;
+        tx.try_send(ExportProgressItem::CopyProgress(offset))
+            .await
+            .map_err(|_e| io::Error::other(""))?;
+        yield_now().await;
     }
+    Ok(())
+}
 
-    async fn delete(&self, hashes: Vec<Hash>) -> io::Result<()> {
-        let mut state = self.write_lock();
-        for hash in hashes {
-            if !state.temp.contains(&hash) {
-                state.entries.remove(&hash);
-            }
-        }
-        Ok(())
+struct ImportEntry {
+    scope: Scope,
+    format: BlobFormat,
+    data: Bytes,
+    outboard: PreOrderMemOutboard,
+    tx: mpsc::Sender<AddProgressItem>,
+}
+
+pub struct DataReader(BaoFileHandle);
+
+impl ReadBytesAt for DataReader {
+    fn read_bytes_at(&self, offset: u64, size: usize) -> std::io::Result<Bytes> {
+        let entry = self.0 .0.state.borrow();
+        entry.data().read_bytes_at(offset, size)
     }
+}
+
+pub struct OutboardReader {
+    hash: blake3::Hash,
+    tree: BaoTree,
+    data: BaoFileHandle,
+}
 
-    async fn shutdown(&self) {}
+impl Outboard for OutboardReader {
+    fn root(&self) -> blake3::Hash {
+        self.hash
+    }
 
-    async fn sync(&self) -> io::Result<()> {
-        Ok(())
+    fn tree(&self) -> BaoTree {
+        self.tree
+    }
+
+    fn load(&self, node: TreeNode) -> io::Result<Option<(blake3::Hash, blake3::Hash)>> {
+        let Some(offset) = self.tree.pre_order_offset(node) else {
+            return Ok(None);
+        };
+        let mut buf = [0u8; 64];
+        let size = self
+            .data
+            .0
+            .state
+            .borrow()
+            .outboard()
+            .read_at(offset * 64, &mut buf)?;
+        if size != 64 {
+            return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "short read"));
+        }
+        let left: [u8; 32] = buf[..32].try_into().unwrap();
+        let right: [u8; 32] = buf[32..].try_into().unwrap();
+        Ok(Some((left.into(), right.into())))
     }
 }
 
-#[derive(Debug, Default)]
-struct StateInner {
-    entries: BTreeMap<Hash, Entry>,
+struct State {
+    data: HashMap<Hash, BaoFileHandle>,
     tags: BTreeMap<Tag, HashAndFormat>,
-    temp: TempCounterMap,
 }
 
-/// An in memory entry
-#[derive(Debug, Clone)]
-pub struct Entry {
-    inner: Arc<EntryInner>,
-    complete: bool,
+#[derive(Debug, derive_more::From)]
+pub enum BaoFileStorage {
+    Partial(PartialMemStorage),
+    Complete(CompleteStorage),
+}
+
+impl BaoFileStorage {
+    /// Get the bitfield of the storage.
+    pub fn bitfield(&self) -> Bitfield {
+        match self {
+            Self::Partial(entry) => entry.bitfield.clone(),
+            Self::Complete(entry) => Bitfield::complete(entry.size()),
+        }
+    }
 }
 
 #[derive(Debug)]
-struct EntryInner {
+pub struct BaoFileHandleInner {
+    state: watch::Sender<BaoFileStorage>,
     hash: Hash,
-    data: RwLock<MutableMemStorage>,
 }
 
-impl MapEntry for Entry {
-    fn hash(&self) -> Hash {
-        self.inner.hash
-    }
+/// A cheaply cloneable handle to a bao file, including the hash
+#[derive(Debug, Clone, derive_more::Deref)]
+pub struct BaoFileHandle(Arc<BaoFileHandleInner>);
 
-    fn size(&self) -> BaoBlobSize {
-        let size = self.inner.data.read().unwrap().current_size();
-        BaoBlobSize::new(size, self.complete)
+impl BaoFileHandle {
+    pub fn new_partial(hash: Hash) -> Self {
+        let (state, _) = watch::channel(BaoFileStorage::Partial(PartialMemStorage {
+            data: SparseMemFile::new(),
+            outboard: SparseMemFile::new(),
+            size: SizeInfo::default(),
+            bitfield: Bitfield::empty(),
+        }));
+        Self(Arc::new(BaoFileHandleInner { state, hash }))
     }
 
-    fn is_complete(&self) -> bool {
-        self.complete
+    pub fn hash(&self) -> Hash {
+        self.hash
     }
 
-    async fn outboard(&self) -> io::Result<impl Outboard> {
-        let size = self.inner.data.read().unwrap().current_size();
-        Ok(PreOrderOutboard {
-            root: self.hash().into(),
-            tree: BaoTree::new(size, IROH_BLOCK_SIZE),
-            data: OutboardReader(self.inner.clone()),
-        })
+    pub fn bitfield(&self) -> Bitfield {
+        self.0.state.borrow().bitfield()
     }
 
-    async fn data_reader(&self) -> io::Result<impl AsyncSliceReader> {
-        Ok(DataReader(self.inner.clone()))
+    pub fn subscribe(&self) -> BaoFileStorageSubscriber {
+        BaoFileStorageSubscriber::new(self.0.state.subscribe())
     }
-}
 
-impl MapEntryMut for Entry {
-    async fn batch_writer(&self) -> io::Result<impl BaoBatchWriter> {
-        Ok(BatchWriter(self.inner.clone()))
+    pub fn data_reader(&self) -> DataReader {
+        DataReader(self.clone())
     }
-}
-
-struct DataReader(Arc<EntryInner>);
 
-impl AsyncSliceReader for DataReader {
-    async fn read_at(&mut self, offset: u64, len: usize) -> std::io::Result<Bytes> {
-        Ok(self.0.data.read().unwrap().read_data_at(offset, len))
+    pub fn outboard_reader(&self) -> OutboardReader {
+        let entry = self.0.state.borrow();
+        let hash = self.hash.into();
+        let tree = BaoTree::new(entry.size(), IROH_BLOCK_SIZE);
+        OutboardReader {
+            hash,
+            tree,
+            data: self.clone(),
+        }
     }
+}
 
-    async fn size(&mut self) -> std::io::Result<u64> {
-        Ok(self.0.data.read().unwrap().data_len())
+impl Default for BaoFileStorage {
+    fn default() -> Self {
+        Self::Partial(Default::default())
     }
 }
 
-struct OutboardReader(Arc<EntryInner>);
+impl BaoFileStorage {
+    fn data(&self) -> &[u8] {
+        match self {
+            Self::Partial(entry) => entry.data.as_ref(),
+            Self::Complete(entry) => &entry.data,
+        }
+    }
 
-impl AsyncSliceReader for OutboardReader {
-    async fn read_at(&mut self, offset: u64, len: usize) -> std::io::Result<Bytes> {
-        Ok(self.0.data.read().unwrap().read_outboard_at(offset, len))
+    fn outboard(&self) -> &[u8] {
+        match self {
+            Self::Partial(entry) => entry.outboard.as_ref(),
+            Self::Complete(entry) => &entry.outboard,
+        }
     }
 
-    async fn size(&mut self) -> std::io::Result<u64> {
-        Ok(self.0.data.read().unwrap().outboard_len())
+    fn size(&self) -> u64 {
+        match self {
+            Self::Partial(entry) => entry.current_size(),
+            Self::Complete(entry) => entry.size(),
+        }
     }
 }
 
-struct BatchWriter(Arc<EntryInner>);
+#[derive(Debug, Clone)]
+pub struct CompleteStorage {
+    pub(crate) data: Bytes,
+    pub(crate) outboard: Bytes,
+}
 
-impl super::BaoBatchWriter for BatchWriter {
-    async fn write_batch(
-        &mut self,
-        size: u64,
-        batch: Vec<bao_tree::io::fsm::BaoContentItem>,
-    ) -> io::Result<()> {
-        self.0.data.write().unwrap().write_batch(size, &batch)
+impl CompleteStorage {
+    pub fn create(data: Bytes) -> (Hash, Self) {
+        let outboard = PreOrderMemOutboard::create(&data, IROH_BLOCK_SIZE);
+        let hash = outboard.root().into();
+        let outboard = outboard.data.into();
+        let entry = Self::new(data, outboard);
+        (hash, entry)
     }
 
-    async fn sync(&mut self) -> io::Result<()> {
-        Ok(())
+    pub fn new(data: Bytes, outboard: Bytes) -> Self {
+        Self { data, outboard }
     }
-}
 
-impl super::Map for Store {
-    type Entry = Entry;
+    pub fn size(&self) -> u64 {
+        self.data.len() as u64
+    }
+}
 
-    async fn get(&self, hash: &Hash) -> std::io::Result<Option<Self::Entry>> {
-        Ok(self.inner.0.read().unwrap().entries.get(hash).cloned())
+#[allow(dead_code)]
+fn print_outboard(hashes: &[u8]) {
+    assert!(hashes.len() % 64 == 0);
+    for chunk in hashes.chunks(64) {
+        let left: [u8; 32] = chunk[..32].try_into().unwrap();
+        let right: [u8; 32] = chunk[32..].try_into().unwrap();
+        let left = blake3::Hash::from(left);
+        let right = blake3::Hash::from(right);
+        println!("l: {left:?}, r: {right:?}");
     }
 }
 
-impl super::MapMut for Store {
-    type EntryMut = Entry;
+pub struct BaoFileStorageSubscriber {
+    receiver: watch::Receiver<BaoFileStorage>,
+}
 
-    async fn get_mut(&self, hash: &Hash) -> std::io::Result<Option<Self::EntryMut>> {
-        self.get(hash).await
+impl BaoFileStorageSubscriber {
+    pub fn new(receiver: watch::Receiver<BaoFileStorage>) -> Self {
+        Self { receiver }
     }
 
-    async fn get_or_create(&self, hash: Hash, _size: u64) -> std::io::Result<Entry> {
-        let entry = Entry {
-            inner: Arc::new(EntryInner {
-                hash,
-                data: RwLock::new(MutableMemStorage::default()),
-            }),
-            complete: false,
-        };
-        Ok(entry)
+    /// Forward observed *values* to the given sender
+    ///
+    /// Returns an error if sending fails, or if the last sender is dropped
+    pub async fn forward(mut self, mut tx: mpsc::Sender<Bitfield>) -> anyhow::Result<()> {
+        let value = self.receiver.borrow().bitfield();
+        tx.send(value).await?;
+        loop {
+            self.update_or_closed(&mut tx).await?;
+            let value = self.receiver.borrow().bitfield();
+            tx.send(value.clone()).await?;
+        }
     }
 
-    async fn entry_status(&self, hash: &Hash) -> std::io::Result<crate::store::EntryStatus> {
-        self.entry_status_sync(hash)
+    /// Forward observed *deltas* to the given sender
+    ///
+    /// Returns an error if sending fails, or if the last sender is dropped
+    #[allow(dead_code)]
+    pub async fn forward_delta(mut self, mut tx: mpsc::Sender<Bitfield>) -> anyhow::Result<()> {
+        let value = self.receiver.borrow().bitfield();
+        let mut old = value.clone();
+        tx.send(value).await?;
+        loop {
+            self.update_or_closed(&mut tx).await?;
+            let new = self.receiver.borrow().bitfield();
+            let diff = old.diff(&new);
+            if diff.is_empty() {
+                continue;
+            }
+            tx.send(diff).await?;
+            old = new;
+        }
     }
 
-    fn entry_status_sync(&self, hash: &Hash) -> std::io::Result<crate::store::EntryStatus> {
-        Ok(match self.inner.0.read().unwrap().entries.get(hash) {
-            Some(entry) => {
-                if entry.complete {
-                    crate::store::EntryStatus::Complete
-                } else {
-                    crate::store::EntryStatus::Partial
-                }
+    async fn update_or_closed(&mut self, tx: &mut mpsc::Sender<Bitfield>) -> anyhow::Result<()> {
+        tokio::select! {
+            _ = tx.closed() => {
+                // the sender is closed, we are done
+                Err(irpc::channel::SendError::ReceiverClosed.into())
             }
-            None => crate::store::EntryStatus::NotFound,
-        })
-    }
-
-    async fn insert_complete(&self, mut entry: Entry) -> std::io::Result<()> {
-        let hash = entry.hash();
-        let mut inner = self.inner.0.write().unwrap();
-        let complete = inner
-            .entries
-            .get(&hash)
-            .map(|x| x.complete)
-            .unwrap_or_default();
-        if !complete {
-            entry.complete = true;
-            inner.entries.insert(hash, entry);
+            e = self.receiver.changed() => Ok(e?),
         }
-        Ok(())
     }
 }
 
-impl ReadableStore for Store {
-    async fn blobs(&self) -> io::Result<crate::store::DbIter<Hash>> {
-        let entries = self.read_lock().entries.clone();
-        Ok(Box::new(
-            entries
-                .into_values()
-                .filter(|x| x.complete)
-                .map(|x| Ok(x.hash())),
-        ))
-    }
-
-    async fn partial_blobs(&self) -> io::Result<crate::store::DbIter<Hash>> {
-        let entries = self.read_lock().entries.clone();
-        Ok(Box::new(
-            entries
-                .into_values()
-                .filter(|x| !x.complete)
-                .map(|x| Ok(x.hash())),
-        ))
-    }
-
-    async fn tags(
-        &self,
-        from: Option<Tag>,
-        to: Option<Tag>,
-    ) -> io::Result<crate::store::DbIter<(crate::Tag, crate::HashAndFormat)>> {
-        #[allow(clippy::mutable_key_type)]
-        let tags = self.read_lock().tags.clone();
-        let tags = tags
-            .into_iter()
-            .filter(move |(tag, _)| {
-                if let Some(from) = &from {
-                    if tag < from {
-                        return false;
-                    }
-                }
-                if let Some(to) = &to {
-                    if tag >= to {
-                        return false;
-                    }
-                }
-                true
-            })
-            .map(Ok);
-        Ok(Box::new(tags))
-    }
+#[cfg(test)]
+mod tests {
+    use n0_future::StreamExt;
+    use testresult::TestResult;
 
-    fn temp_tags(&self) -> Box<dyn Iterator<Item = crate::HashAndFormat> + Send + Sync + 'static> {
-        let tags = self.read_lock().temp.keys();
-        Box::new(tags)
-    }
+    use super::*;
 
-    async fn consistency_check(
-        &self,
-        _repair: bool,
-        _tx: BoxedProgressSender<ConsistencyCheckProgress>,
-    ) -> io::Result<()> {
-        todo!()
-    }
+    #[tokio::test]
+    async fn smoke() -> TestResult<()> {
+        let store = MemStore::new();
+        let tt = store.add_bytes(vec![0u8; 1024 * 64]).temp_tag().await?;
+        let hash = *tt.hash();
+        println!("hash: {hash:?}");
+        let mut stream = store.export_bao(hash, ChunkRanges::all()).stream();
+        while let Some(item) = stream.next().await {
+            println!("item: {item:?}");
+        }
+        let stream = store.export_bao(hash, ChunkRanges::all());
+        let exported = stream.bao_to_vec().await?;
 
-    async fn export(
-        &self,
-        hash: Hash,
-        target: std::path::PathBuf,
-        mode: crate::store::ExportMode,
-        progress: ExportProgressCb,
-    ) -> io::Result<()> {
-        let this = self.clone();
-        tokio::task::spawn_blocking(move || this.export_sync(hash, target, mode, progress)).await?
+        let store2 = MemStore::new();
+        let mut or = store2.observe(hash).stream().await?;
+        tokio::spawn(async move {
+            while let Some(event) = or.next().await {
+                println!("event: {event:?}");
+            }
+        });
+        store2
+            .import_bao_bytes(hash, ChunkRanges::all(), exported.clone())
+            .await?;
+
+        let exported2 = store2
+            .export_bao(hash, ChunkRanges::all())
+            .bao_to_vec()
+            .await?;
+        assert_eq!(exported, exported2);
+
+        Ok(())
     }
 }
diff --git a/src/store/mod.rs b/src/store/mod.rs
new file mode 100644
index 000000000..3e1a3748f
--- /dev/null
+++ b/src/store/mod.rs
@@ -0,0 +1,17 @@
+//! Store implementations
+//!
+//! Use the [`mem`] store for sharing a small amount of mutable data,
+//! the [`readonly_mem`] store for sharing static data, and the [`fs`] store
+//! for when you want to efficiently share more than the available memory and
+//! have access to a writeable filesystem.
+use bao_tree::BlockSize;
+pub mod fs;
+pub mod mem;
+pub mod readonly_mem;
+mod test;
+pub(crate) mod util;
+
+use crate::hash::{Hash, HashAndFormat};
+
+/// Block size used by iroh, 2^4*1024 = 16KiB
+pub const IROH_BLOCK_SIZE: BlockSize = BlockSize::from_chunk_log(4);
diff --git a/src/store/mutable_mem_storage.rs b/src/store/mutable_mem_storage.rs
deleted file mode 100644
index f100476fa..000000000
--- a/src/store/mutable_mem_storage.rs
+++ /dev/null
@@ -1,128 +0,0 @@
-use bao_tree::{
-    io::{fsm::BaoContentItem, sync::WriteAt},
-    BaoTree,
-};
-use bytes::Bytes;
-
-use crate::{
-    util::{compute_outboard, copy_limited_slice, SparseMemFile},
-    IROH_BLOCK_SIZE,
-};
-
-/// Mutable in memory storage for a bao file.
-///
-/// This is used for incomplete files if they are not big enough to warrant
-/// writing to disk. We must keep track of ranges in both data and outboard
-/// that have been written to, and track the most precise known size.
-#[derive(Debug, Default)]
-pub struct MutableMemStorage {
-    /// Data file, can be any size.
-    pub data: SparseMemFile,
-    /// Outboard file, must be a multiple of 64 bytes.
-    pub outboard: SparseMemFile,
-    /// Size that was announced as we wrote that chunk
-    pub sizes: SizeInfo,
-}
-
-/// Keep track of the most precise size we know of.
-///
-/// When in memory, we don't have to write the size for every chunk to a separate
-/// slot, but can just keep the best one.
-#[derive(Debug, Default)]
-pub struct SizeInfo {
-    pub offset: u64,
-    pub size: u64,
-}
-
-impl SizeInfo {
-    /// Create a new size info for a complete file of size `size`.
-    pub(crate) fn complete(size: u64) -> Self {
-        let mask = (1 << IROH_BLOCK_SIZE.chunk_log()) - 1;
-        // offset of the last bao chunk in a file of size `size`
-        let last_chunk_offset = size & mask;
-        Self {
-            offset: last_chunk_offset,
-            size,
-        }
-    }
-
-    /// Write a size at the given offset. The size at the highest offset is going to be kept.
-    fn write(&mut self, offset: u64, size: u64) {
-        // >= instead of > because we want to be able to update size 0, the initial value.
-        if offset >= self.offset {
-            self.offset = offset;
-            self.size = size;
-        }
-    }
-
-    /// The current size, representing the most correct size we know.
-    pub fn current_size(&self) -> u64 {
-        self.size
-    }
-}
-
-impl MutableMemStorage {
-    /// Create a new mutable mem storage from the given data
-    pub fn complete(bytes: Bytes, cb: impl Fn(u64) + Send + Sync + 'static) -> (Self, crate::Hash) {
-        let (hash, outboard) = compute_outboard(&bytes[..], bytes.len() as u64, move |offset| {
-            cb(offset);
-            Ok(())
-        })
-        .unwrap();
-        let outboard = outboard.unwrap_or_default();
-        let res = Self {
-            data: bytes.to_vec().into(),
-            outboard: outboard.into(),
-            sizes: SizeInfo::complete(bytes.len() as u64),
-        };
-        (res, hash)
-    }
-
-    pub(super) fn current_size(&self) -> u64 {
-        self.sizes.current_size()
-    }
-
-    pub(super) fn read_data_at(&self, offset: u64, len: usize) -> Bytes {
-        copy_limited_slice(&self.data, offset, len)
-    }
-
-    pub(super) fn data_len(&self) -> u64 {
-        self.data.len() as u64
-    }
-
-    pub(super) fn read_outboard_at(&self, offset: u64, len: usize) -> Bytes {
-        copy_limited_slice(&self.outboard, offset, len)
-    }
-
-    pub(super) fn outboard_len(&self) -> u64 {
-        self.outboard.len() as u64
-    }
-
-    pub(super) fn write_batch(
-        &mut self,
-        size: u64,
-        batch: &[BaoContentItem],
-    ) -> std::io::Result<()> {
-        let tree = BaoTree::new(size, IROH_BLOCK_SIZE);
-        for item in batch {
-            match item {
-                BaoContentItem::Parent(parent) => {
-                    if let Some(offset) = tree.pre_order_offset(parent.node) {
-                        let o0 = offset
-                            .checked_mul(64)
-                            .expect("u64 overflow multiplying to hash pair offset");
-                        let o1 = o0.checked_add(32).expect("u64 overflow");
-                        let outboard = &mut self.outboard;
-                        outboard.write_all_at(o0, parent.pair.0.as_bytes().as_slice())?;
-                        outboard.write_all_at(o1, parent.pair.1.as_bytes().as_slice())?;
-                    }
-                }
-                BaoContentItem::Leaf(leaf) => {
-                    self.sizes.write(leaf.offset, size);
-                    self.data.write_all_at(leaf.offset, leaf.data.as_ref())?;
-                }
-            }
-        }
-        Ok(())
-    }
-}
diff --git a/src/store/readonly_mem.rs b/src/store/readonly_mem.rs
index ec8f1b5f3..a00cf82e6 100644
--- a/src/store/readonly_mem.rs
+++ b/src/store/readonly_mem.rs
@@ -1,357 +1,390 @@
-//! A readonly in memory database for iroh-blobs, usable for testing and sharing static data.
+//! Readonly in-memory store.
 //!
-//! Main entry point is [Store].
+//! This can only serve data that is provided at creation time. It is much simpler
+//! than the mutable in-memory store and the file system store, and can serve as a
+//! good starting point for custom implementations.
+//!
+//! It can also be useful as a lightweight store for tests.
 use std::{
-    collections::{BTreeMap, BTreeSet, HashMap},
-    future::Future,
-    io,
+    collections::HashMap,
+    io::{self, Write},
+    ops::Deref,
     path::PathBuf,
-    sync::Arc,
 };
 
 use bao_tree::{
-    blake3,
-    io::{outboard::PreOrderMemOutboard, sync::Outboard},
+    io::{
+        mixed::{traverse_ranges_validated, EncodedItem, ReadBytesAt},
+        outboard::PreOrderMemOutboard,
+        sync::ReadAt,
+        Leaf,
+    },
+    BaoTree, ChunkRanges,
 };
 use bytes::Bytes;
-use futures_lite::Stream;
-use iroh_io::AsyncSliceReader;
-use tokio::io::AsyncWriteExt;
+use irpc::channel::mpsc;
+use n0_future::future::{self, yield_now};
+use range_collections::range_set::RangeSetRange;
+use ref_cast::RefCast;
+use tokio::task::{JoinError, JoinSet};
 
-use super::{BaoBatchWriter, BaoBlobSize, ConsistencyCheckProgress, DbIter, ExportProgressCb};
+use super::util::BaoTreeSender;
 use crate::{
-    store::{
-        EntryStatus, ExportMode, ImportMode, ImportProgress, Map, MapEntry, MapEntryMut,
-        ReadableStore,
+    api::{
+        self,
+        blobs::{Bitfield, ExportProgressItem},
+        proto::{
+            self, BlobStatus, Command, ExportBaoMsg, ExportBaoRequest, ExportPathMsg,
+            ExportPathRequest, ExportRangesItem, ExportRangesMsg, ExportRangesRequest,
+            ImportBaoMsg, ImportByteStreamMsg, ImportBytesMsg, ImportPathMsg, ObserveMsg,
+            ObserveRequest,
+        },
+        ApiClient, TempTag,
     },
-    util::{
-        progress::{BoxedProgressSender, IdGenerator, ProgressSender},
-        Tag,
-    },
-    BlobFormat, Hash, HashAndFormat, TempTag, IROH_BLOCK_SIZE,
+    store::{mem::CompleteStorage, IROH_BLOCK_SIZE},
+    util::ChunkRangesExt,
+    Hash,
 };
 
-/// A readonly in memory database for iroh-blobs.
-///
-/// This is basically just a HashMap, so it does not allow for any modifications
-/// unless you have a mutable reference to it.
-///
-/// It is therefore useful mostly for testing and sharing static data.
-#[derive(Debug, Clone, Default)]
-pub struct Store(Arc<HashMap<Hash, (PreOrderMemOutboard<Bytes>, Bytes)>>);
-
-impl<K, V> FromIterator<(K, V)> for Store
-where
-    K: Into<String>,
-    V: AsRef<[u8]>,
-{
-    fn from_iter<T: IntoIterator<Item = (K, V)>>(iter: T) -> Self {
-        let (db, _m) = Self::new(iter);
-        db
-    }
-}
-
-impl Store {
-    /// Create a new [Store] from a sequence of entries.
-    ///
-    /// Returns the database and a map of names to computed blake3 hashes.
-    /// In case of duplicate names, the last entry is used.
-    pub fn new(
-        entries: impl IntoIterator<Item = (impl Into<String>, impl AsRef<[u8]>)>,
-    ) -> (Self, BTreeMap<String, blake3::Hash>) {
-        let mut names = BTreeMap::new();
-        let mut res = HashMap::new();
-        for (name, data) in entries.into_iter() {
-            let name = name.into();
-            let data: &[u8] = data.as_ref();
-            // wrap into the right types
-            let outboard = PreOrderMemOutboard::create(data, IROH_BLOCK_SIZE).map_data(Bytes::from);
-            let hash = outboard.root();
-            // add the name, this assumes that names are unique
-            names.insert(name, hash);
-            let data = Bytes::from(data.to_vec());
-            let hash = Hash::from(hash);
-            res.insert(hash, (outboard, data));
-        }
-        (Self(Arc::new(res)), names)
-    }
-
-    /// Insert a new entry into the database, and return the hash of the entry.
-    ///
-    /// If the database was shared before, this will make a copy.
-    pub fn insert(&mut self, data: impl AsRef<[u8]>) -> Hash {
-        let inner = Arc::make_mut(&mut self.0);
-        let data: &[u8] = data.as_ref();
-        // wrap into the right types
-        let outboard = PreOrderMemOutboard::create(data, IROH_BLOCK_SIZE).map_data(Bytes::from);
-        let hash = outboard.root();
-        let data = Bytes::from(data.to_vec());
-        let hash = Hash::from(hash);
-        inner.insert(hash, (outboard, data));
-        hash
-    }
-
-    /// Insert multiple entries into the database, and return the hash of the last entry.
-    pub fn insert_many(
-        &mut self,
-        items: impl IntoIterator<Item = impl AsRef<[u8]>>,
-    ) -> Option<Hash> {
-        let mut hash = None;
-        for item in items.into_iter() {
-            hash = Some(self.insert(item));
-        }
-        hash
-    }
-
-    /// Get the bytes associated with a hash, if they exist.
-    pub fn get_content(&self, hash: &Hash) -> Option<Bytes> {
-        let entry = self.0.get(hash)?;
-        Some(entry.1.clone())
-    }
-
-    async fn export_impl(
-        &self,
-        hash: Hash,
-        target: PathBuf,
-        _mode: ExportMode,
-        progress: impl Fn(u64) -> io::Result<()> + Send + Sync + 'static,
-    ) -> io::Result<()> {
-        tracing::trace!("exporting {} to {}", hash, target.display());
-
-        if !target.is_absolute() {
-            return Err(io::Error::new(
-                io::ErrorKind::InvalidInput,
-                "target path must be absolute",
-            ));
-        }
-        let parent = target.parent().ok_or_else(|| {
-            io::Error::new(
-                io::ErrorKind::InvalidInput,
-                "target path has no parent directory",
-            )
-        })?;
-        // create the directory in which the target file is
-        tokio::fs::create_dir_all(parent).await?;
-        let data = self
-            .get_content(&hash)
-            .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "hash not found"))?;
-
-        let mut offset = 0u64;
-        let mut file = tokio::fs::File::create(&target).await?;
-        for chunk in data.chunks(1024 * 1024) {
-            progress(offset)?;
-            file.write_all(chunk).await?;
-            offset += chunk.len() as u64;
-        }
-        file.sync_all().await?;
-        drop(file);
-        Ok(())
-    }
-}
-
-/// The [MapEntry] implementation for [Store].
 #[derive(Debug, Clone)]
-pub struct Entry {
-    outboard: PreOrderMemOutboard<Bytes>,
-    data: Bytes,
+pub struct ReadonlyMemStore {
+    client: ApiClient,
 }
 
-impl MapEntry for Entry {
-    fn hash(&self) -> Hash {
-        self.outboard.root().into()
-    }
+impl Deref for ReadonlyMemStore {
+    type Target = crate::api::Store;
 
-    fn size(&self) -> BaoBlobSize {
-        BaoBlobSize::Verified(self.data.len() as u64)
-    }
-
-    async fn outboard(&self) -> io::Result<impl bao_tree::io::fsm::Outboard> {
-        Ok(self.outboard.clone())
-    }
-
-    async fn data_reader(&self) -> io::Result<impl AsyncSliceReader> {
-        Ok(self.data.clone())
-    }
-
-    fn is_complete(&self) -> bool {
-        true
+    fn deref(&self) -> &Self::Target {
+        crate::api::Store::ref_from_sender(&self.client)
     }
 }
 
-impl Map for Store {
-    type Entry = Entry;
-
-    async fn get(&self, hash: &Hash) -> io::Result<Option<Self::Entry>> {
-        Ok(self.0.get(hash).map(|(o, d)| Entry {
-            outboard: o.clone(),
-            data: d.clone(),
-        }))
-    }
+struct Actor {
+    commands: tokio::sync::mpsc::Receiver<proto::Command>,
+    tasks: JoinSet<()>,
+    data: HashMap<Hash, CompleteStorage>,
 }
 
-impl super::MapMut for Store {
-    type EntryMut = Entry;
-
-    async fn get_mut(&self, hash: &Hash) -> io::Result<Option<Self::EntryMut>> {
-        self.get(hash).await
-    }
-
-    async fn get_or_create(&self, _hash: Hash, _size: u64) -> io::Result<Entry> {
-        Err(io::Error::new(
-            io::ErrorKind::Other,
-            "cannot create temp entry in readonly database",
-        ))
+impl Actor {
+    fn new(
+        commands: tokio::sync::mpsc::Receiver<proto::Command>,
+        data: HashMap<Hash, CompleteStorage>,
+    ) -> Self {
+        Self {
+            data,
+            commands,
+            tasks: JoinSet::new(),
+        }
     }
 
-    fn entry_status_sync(&self, hash: &Hash) -> io::Result<EntryStatus> {
-        Ok(match self.0.contains_key(hash) {
-            true => EntryStatus::Complete,
-            false => EntryStatus::NotFound,
-        })
+    async fn handle_command(&mut self, cmd: Command) -> Option<irpc::channel::oneshot::Sender<()>> {
+        match cmd {
+            Command::ImportBao(ImportBaoMsg { tx, .. }) => {
+                tx.send(Err(api::Error::Io(io::Error::other(
+                    "import not supported",
+                ))))
+                .await
+                .ok();
+            }
+            Command::ImportBytes(ImportBytesMsg { tx, .. }) => {
+                tx.send(io::Error::other("import not supported").into())
+                    .await
+                    .ok();
+            }
+            Command::ImportByteStream(ImportByteStreamMsg { tx, .. }) => {
+                tx.send(io::Error::other("import not supported").into())
+                    .await
+                    .ok();
+            }
+            Command::ImportPath(ImportPathMsg { tx, .. }) => {
+                tx.send(io::Error::other("import not supported").into())
+                    .await
+                    .ok();
+            }
+            Command::Observe(ObserveMsg {
+                inner: ObserveRequest { hash },
+                tx,
+                ..
+            }) => {
+                let size = self.data.get_mut(&hash).map(|x| x.data.len() as u64);
+                self.tasks.spawn(async move {
+                    if let Some(size) = size {
+                        tx.send(Bitfield::complete(size)).await.ok();
+                    } else {
+                        tx.send(Bitfield::empty()).await.ok();
+                        future::pending::<()>().await;
+                    };
+                });
+            }
+            Command::ExportBao(ExportBaoMsg {
+                inner: ExportBaoRequest { hash, ranges },
+                tx,
+                ..
+            }) => {
+                let entry = self.data.get(&hash).cloned();
+                self.tasks.spawn(export_bao(hash, entry, ranges, tx));
+            }
+            Command::ExportPath(ExportPathMsg {
+                inner: ExportPathRequest { hash, target, .. },
+                tx,
+                ..
+            }) => {
+                let entry = self.data.get(&hash).cloned();
+                self.tasks.spawn(export_path(entry, target, tx));
+            }
+            Command::Batch(_cmd) => {}
+            Command::ClearProtected(cmd) => {
+                cmd.tx.send(Ok(())).await.ok();
+            }
+            Command::CreateTag(cmd) => {
+                cmd.tx
+                    .send(Err(io::Error::other("create tag not supported").into()))
+                    .await
+                    .ok();
+            }
+            Command::CreateTempTag(cmd) => {
+                cmd.tx.send(TempTag::new(cmd.inner.value, None)).await.ok();
+            }
+            Command::RenameTag(cmd) => {
+                cmd.tx
+                    .send(Err(io::Error::other("rename tag not supported").into()))
+                    .await
+                    .ok();
+            }
+            Command::DeleteTags(cmd) => {
+                cmd.tx
+                    .send(Err(io::Error::other("delete tags not supported").into()))
+                    .await
+                    .ok();
+            }
+            Command::DeleteBlobs(cmd) => {
+                cmd.tx
+                    .send(Err(io::Error::other("delete blobs not supported").into()))
+                    .await
+                    .ok();
+            }
+            Command::ListBlobs(cmd) => {
+                let hashes: Vec<Hash> = self.data.keys().cloned().collect();
+                self.tasks.spawn(async move {
+                    for hash in hashes {
+                        cmd.tx.send(Ok(hash)).await.ok();
+                    }
+                });
+            }
+            Command::BlobStatus(cmd) => {
+                let hash = cmd.inner.hash;
+                let entry = self.data.get(&hash);
+                let status = if let Some(entry) = entry {
+                    BlobStatus::Complete {
+                        size: entry.data.len() as u64,
+                    }
+                } else {
+                    BlobStatus::NotFound
+                };
+                cmd.tx.send(status).await.ok();
+            }
+            Command::ListTags(cmd) => {
+                cmd.tx.send(Vec::new()).await.ok();
+            }
+            Command::SetTag(cmd) => {
+                cmd.tx
+                    .send(Err(io::Error::other("set tag not supported").into()))
+                    .await
+                    .ok();
+            }
+            Command::ListTempTags(cmd) => {
+                cmd.tx.send(Vec::new()).await.ok();
+            }
+            Command::SyncDb(cmd) => {
+                cmd.tx.send(Ok(())).await.ok();
+            }
+            Command::Shutdown(cmd) => {
+                return Some(cmd.tx);
+            }
+            Command::ExportRanges(cmd) => {
+                let entry = self.data.get(&cmd.inner.hash).cloned();
+                self.tasks.spawn(export_ranges(cmd, entry));
+            }
+        }
+        None
     }
 
-    async fn entry_status(&self, hash: &Hash) -> io::Result<EntryStatus> {
-        self.entry_status_sync(hash)
+    fn log_unit_task(&self, res: Result<(), JoinError>) {
+        if let Err(e) = res {
+            tracing::error!("task failed: {e}");
+        }
     }
 
-    async fn insert_complete(&self, _entry: Entry) -> io::Result<()> {
-        // this is unreachable, since we cannot create partial entries
-        unreachable!()
+    async fn run(mut self) {
+        loop {
+            tokio::select! {
+                Some(cmd) = self.commands.recv() => {
+                    if let Some(shutdown) = self.handle_command(cmd).await {
+                        shutdown.send(()).await.ok();
+                        break;
+                    }
+                },
+                Some(res) = self.tasks.join_next(), if !self.tasks.is_empty() => {
+                    self.log_unit_task(res);
+                },
+                else => break,
+            }
+        }
     }
 }
 
-impl ReadableStore for Store {
-    async fn blobs(&self) -> io::Result<DbIter<Hash>> {
-        Ok(Box::new(
-            self.0
-                .keys()
-                .copied()
-                .map(Ok)
-                .collect::<Vec<_>>()
-                .into_iter(),
-        ))
-    }
-
-    async fn tags(
-        &self,
-        _from: Option<Tag>,
-        _to: Option<Tag>,
-    ) -> io::Result<DbIter<(Tag, HashAndFormat)>> {
-        Ok(Box::new(std::iter::empty()))
-    }
-
-    fn temp_tags(&self) -> Box<dyn Iterator<Item = HashAndFormat> + Send + Sync + 'static> {
-        Box::new(std::iter::empty())
-    }
-
-    async fn consistency_check(
-        &self,
-        _repair: bool,
-        _tx: BoxedProgressSender<ConsistencyCheckProgress>,
-    ) -> io::Result<()> {
-        Ok(())
-    }
-
-    async fn export(
-        &self,
-        hash: Hash,
-        target: PathBuf,
-        mode: ExportMode,
-        progress: ExportProgressCb,
-    ) -> io::Result<()> {
-        self.export_impl(hash, target, mode, progress).await
-    }
+async fn export_bao(
+    hash: Hash,
+    entry: Option<CompleteStorage>,
+    ranges: ChunkRanges,
+    mut sender: mpsc::Sender<EncodedItem>,
+) {
+    let entry = match entry {
+        Some(entry) => entry,
+        None => {
+            sender
+                .send(EncodedItem::Error(bao_tree::io::EncodeError::Io(
+                    io::Error::new(
+                        io::ErrorKind::UnexpectedEof,
+                        "export task ended unexpectedly",
+                    ),
+                )))
+                .await
+                .ok();
+            return;
+        }
+    };
+    let data = entry.data;
+    let outboard = entry.outboard;
+    let size = data.as_ref().len() as u64;
+    let tree = BaoTree::new(size, IROH_BLOCK_SIZE);
+    let outboard = PreOrderMemOutboard {
+        root: hash.into(),
+        tree,
+        data: outboard,
+    };
+    let sender = BaoTreeSender::ref_cast_mut(&mut sender);
+    traverse_ranges_validated(data.as_ref(), outboard, &ranges, sender)
+        .await
+        .ok();
+}
 
-    async fn partial_blobs(&self) -> io::Result<DbIter<Hash>> {
-        Ok(Box::new(std::iter::empty()))
+async fn export_ranges(mut cmd: ExportRangesMsg, entry: Option<CompleteStorage>) {
+    let Some(entry) = entry else {
+        cmd.tx
+            .send(ExportRangesItem::Error(api::Error::io(
+                io::ErrorKind::NotFound,
+                "hash not found",
+            )))
+            .await
+            .ok();
+        return;
+    };
+    if let Err(cause) = export_ranges_impl(cmd.inner, &mut cmd.tx, entry).await {
+        cmd.tx
+            .send(ExportRangesItem::Error(cause.into()))
+            .await
+            .ok();
     }
 }
 
-impl MapEntryMut for Entry {
-    async fn batch_writer(&self) -> io::Result<impl BaoBatchWriter> {
-        enum Bar {}
-        impl BaoBatchWriter for Bar {
-            async fn write_batch(
-                &mut self,
-                _size: u64,
-                _batch: Vec<bao_tree::io::fsm::BaoContentItem>,
-            ) -> io::Result<()> {
-                unreachable!()
-            }
-
-            async fn sync(&mut self) -> io::Result<()> {
-                unreachable!()
+async fn export_ranges_impl(
+    cmd: ExportRangesRequest,
+    tx: &mut mpsc::Sender<ExportRangesItem>,
+    entry: CompleteStorage,
+) -> io::Result<()> {
+    let ExportRangesRequest { ranges, .. } = cmd;
+    let data = entry.data;
+    let size = data.len() as u64;
+    let bitfield = Bitfield::complete(size);
+    for range in ranges.iter() {
+        let range = match range {
+            RangeSetRange::Range(range) => size.min(*range.start)..size.min(*range.end),
+            RangeSetRange::RangeFrom(range) => size.min(*range.start)..size,
+        };
+        let requested = ChunkRanges::bytes(range.start..range.end);
+        if !bitfield.ranges.is_superset(&requested) {
+            return Err(io::Error::other(format!(
+                "missing range: {requested:?}, present: {bitfield:?}",
+            )));
+        }
+        let bs = 1024;
+        let mut offset = range.start;
+        loop {
+            let end: u64 = (offset + bs).min(range.end);
+            let size = (end - offset) as usize;
+            tx.send(
+                Leaf {
+                    offset,
+                    data: data.read_bytes_at(offset, size)?,
+                }
+                .into(),
+            )
+            .await?;
+            offset = end;
+            if offset >= range.end {
+                break;
             }
         }
-
-        #[allow(unreachable_code)]
-        Ok(unreachable!() as Bar)
     }
+    Ok(())
 }
 
-impl super::Store for Store {
-    async fn import_file(
-        &self,
-        data: PathBuf,
-        mode: ImportMode,
-        format: BlobFormat,
-        progress: impl ProgressSender<Msg = ImportProgress> + IdGenerator,
-    ) -> io::Result<(TempTag, u64)> {
-        let _ = (data, mode, progress, format);
-        Err(io::Error::new(io::ErrorKind::Other, "not implemented"))
-    }
-
-    /// import a byte slice
-    async fn import_bytes(&self, bytes: Bytes, format: BlobFormat) -> io::Result<TempTag> {
-        let _ = (bytes, format);
-        Err(io::Error::new(io::ErrorKind::Other, "not implemented"))
-    }
-
-    async fn rename_tag(&self, _from: Tag, _to: Tag) -> io::Result<()> {
-        Err(io::Error::new(io::ErrorKind::Other, "not implemented"))
-    }
-
-    async fn import_stream(
-        &self,
-        data: impl Stream<Item = io::Result<Bytes>> + Unpin + Send,
-        format: BlobFormat,
-        progress: impl ProgressSender<Msg = ImportProgress> + IdGenerator,
-    ) -> io::Result<(TempTag, u64)> {
-        let _ = (data, format, progress);
-        Err(io::Error::new(io::ErrorKind::Other, "not implemented"))
-    }
-
-    async fn set_tag(&self, _name: Tag, _hash: HashAndFormat) -> io::Result<()> {
-        Err(io::Error::new(io::ErrorKind::Other, "not implemented"))
-    }
-
-    async fn delete_tags(&self, _from: Option<Tag>, _to: Option<Tag>) -> io::Result<()> {
-        Err(io::Error::new(io::ErrorKind::Other, "not implemented"))
-    }
-
-    async fn create_tag(&self, _hash: HashAndFormat) -> io::Result<Tag> {
-        Err(io::Error::new(io::ErrorKind::Other, "not implemented"))
-    }
-
-    fn temp_tag(&self, inner: HashAndFormat) -> TempTag {
-        TempTag::new(inner, None)
-    }
-
-    async fn gc_run<G, Gut>(&self, config: super::GcConfig, protected_cb: G)
-    where
-        G: Fn() -> Gut,
-        Gut: Future<Output = BTreeSet<Hash>> + Send,
-    {
-        super::gc_run_loop(self, config, move || async { Ok(()) }, protected_cb).await
-    }
-
-    async fn delete(&self, _hashes: Vec<Hash>) -> io::Result<()> {
-        Err(io::Error::new(io::ErrorKind::Other, "not implemented"))
+impl ReadonlyMemStore {
+    pub fn new(items: impl IntoIterator<Item = impl AsRef<[u8]>>) -> Self {
+        let mut entries = HashMap::new();
+        for item in items {
+            let data = Bytes::copy_from_slice(item.as_ref());
+            let (hash, entry) = CompleteStorage::create(data);
+            entries.insert(hash, entry);
+        }
+        let (sender, receiver) = tokio::sync::mpsc::channel(1);
+        let actor = Actor::new(receiver, entries);
+        tokio::spawn(actor.run());
+        let local = irpc::LocalSender::from(sender);
+        Self {
+            client: local.into(),
+        }
     }
+}
 
-    async fn shutdown(&self) {}
+async fn export_path(
+    entry: Option<CompleteStorage>,
+    target: PathBuf,
+    mut tx: mpsc::Sender<ExportProgressItem>,
+) {
+    let Some(entry) = entry else {
+        tx.send(api::Error::io(io::ErrorKind::NotFound, "hash not found").into())
+            .await
+            .ok();
+        return;
+    };
+    match export_path_impl(entry, target, &mut tx).await {
+        Ok(()) => tx.send(ExportProgressItem::Done).await.ok(),
+        Err(cause) => tx.send(api::Error::from(cause).into()).await.ok(),
+    };
+}
 
-    async fn sync(&self) -> io::Result<()> {
-        Ok(())
-    }
+async fn export_path_impl(
+    entry: CompleteStorage,
+    target: PathBuf,
+    tx: &mut mpsc::Sender<ExportProgressItem>,
+) -> io::Result<()> {
+    let data = entry.data;
+    // todo: for partial entries make sure to only write the part that is actually present
+    let mut file = std::fs::File::create(&target)?;
+    let size = data.len() as u64;
+    tx.send(ExportProgressItem::Size(size)).await?;
+    let mut buf = [0u8; 1024 * 64];
+    for offset in (0..size).step_by(1024 * 64) {
+        let len = std::cmp::min(size - offset, 1024 * 64) as usize;
+        let buf = &mut buf[..len];
+        data.as_ref().read_exact_at(offset, buf)?;
+        file.write_all(buf)?;
+        tx.try_send(ExportProgressItem::CopyProgress(offset))
+            .await
+            .map_err(|_e| io::Error::other("error"))?;
+        yield_now().await;
+    }
+    Ok(())
 }
diff --git a/src/store/test.rs b/src/store/test.rs
new file mode 100644
index 000000000..411cc2848
--- /dev/null
+++ b/src/store/test.rs
@@ -0,0 +1 @@
+//! Test harness for store implementations.
diff --git a/src/store/traits.rs b/src/store/traits.rs
deleted file mode 100644
index 1d0df2325..000000000
--- a/src/store/traits.rs
+++ /dev/null
@@ -1,978 +0,0 @@
-//! Traits for in-memory or persistent maps of blob with bao encoded outboards.
-use std::{collections::BTreeSet, future::Future, io, path::PathBuf, time::Duration};
-
-pub use bao_tree;
-use bao_tree::{
-    io::fsm::{BaoContentItem, Outboard},
-    BaoTree, ChunkRanges,
-};
-use bytes::Bytes;
-use futures_lite::{Stream, StreamExt};
-use genawaiter::rc::{Co, Gen};
-use iroh_io::AsyncSliceReader;
-pub use range_collections;
-use serde::{Deserialize, Serialize};
-use tokio::io::AsyncRead;
-
-use crate::{
-    hashseq::parse_hash_seq,
-    protocol::RangeSpec,
-    util::{
-        local_pool::{self, LocalPool},
-        progress::{BoxedProgressSender, IdGenerator, ProgressSender},
-        Tag,
-    },
-    BlobFormat, Hash, HashAndFormat, TempTag, IROH_BLOCK_SIZE,
-};
-
-/// A fallible but owned iterator over the entries in a store.
-pub type DbIter<T> = Box<dyn Iterator<Item = io::Result<T>> + Send + Sync + 'static>;
-
-/// Export trogress callback
-pub type ExportProgressCb = Box<dyn Fn(u64) -> io::Result<()> + Send + Sync + 'static>;
-
-/// The availability status of an entry in a store.
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub enum EntryStatus {
-    /// The entry is completely available.
-    Complete,
-    /// The entry is partially available.
-    Partial,
-    /// The entry is not in the store.
-    NotFound,
-}
-
-/// The size of a bao file
-#[derive(Debug, Clone, Copy, Serialize, Deserialize, Eq, PartialEq)]
-pub enum BaoBlobSize {
-    /// A remote side told us the size, but we have insufficient data to verify it.
-    Unverified(u64),
-    /// We have verified the size.
-    Verified(u64),
-}
-
-impl BaoBlobSize {
-    /// Create a new `BaoFileSize` with the given size and verification status.
-    pub fn new(size: u64, verified: bool) -> Self {
-        if verified {
-            BaoBlobSize::Verified(size)
-        } else {
-            BaoBlobSize::Unverified(size)
-        }
-    }
-
-    /// Get just the value, no matter if it is verified or not.
-    pub fn value(&self) -> u64 {
-        match self {
-            BaoBlobSize::Unverified(size) => *size,
-            BaoBlobSize::Verified(size) => *size,
-        }
-    }
-}
-
-/// An entry for one hash in a bao map
-///
-/// The entry has the ability to provide you with an (outboard, data)
-/// reader pair. Creating the reader is async and may fail. The futures that
-/// create the readers must be `Send`, but the readers themselves don't have to
-/// be.
-pub trait MapEntry: std::fmt::Debug + Clone + Send + Sync + 'static {
-    /// The hash of the entry.
-    fn hash(&self) -> Hash;
-    /// The size of the entry.
-    fn size(&self) -> BaoBlobSize;
-    /// Returns `true` if the entry is complete.
-    ///
-    /// Note that this does not actually verify if the bytes on disk are complete,
-    /// it only checks if the entry was marked as complete in the store.
-    fn is_complete(&self) -> bool;
-    /// A future that resolves to a reader that can be used to read the outboard
-    fn outboard(&self) -> impl Future<Output = io::Result<impl Outboard>> + Send;
-    /// A future that resolves to a reader that can be used to read the data
-    fn data_reader(&self) -> impl Future<Output = io::Result<impl AsyncSliceReader>> + Send;
-}
-
-/// A generic map from hashes to bao blobs (blobs with bao outboards).
-///
-/// This is the readonly view. To allow updates, a concrete implementation must
-/// also implement [`MapMut`].
-///
-/// Entries are *not* guaranteed to be complete for all implementations.
-/// They are also not guaranteed to be immutable, since this could be the
-/// readonly view of a mutable store.
-pub trait Map: Clone + Send + Sync + 'static {
-    /// The entry type. An entry is a cheaply cloneable handle that can be used
-    /// to open readers for both the data and the outboard
-    type Entry: MapEntry;
-    /// Get an entry for a hash.
-    ///
-    /// This can also be used for a membership test by just checking if there
-    /// is an entry. Creating an entry should be cheap, any expensive ops should
-    /// be deferred to the creation of the actual readers.
-    ///
-    /// It is not guaranteed that the entry is complete.
-    fn get(&self, hash: &Hash) -> impl Future<Output = io::Result<Option<Self::Entry>>> + Send;
-}
-
-/// A partial entry
-pub trait MapEntryMut: MapEntry {
-    /// Get a batch writer
-    fn batch_writer(&self) -> impl Future<Output = io::Result<impl BaoBatchWriter>> + Send;
-}
-
-/// An async batch interface for writing bao content items to a pair of data and
-/// outboard.
-///
-/// Details like the chunk group size and the actual storage location are left
-/// to the implementation.
-pub trait BaoBatchWriter {
-    /// Write a batch of bao content items to the underlying storage.
-    ///
-    /// The batch is guaranteed to be sorted as data is received from the network.
-    /// So leaves will be sorted by offset, and parents will be sorted by pre order
-    /// traversal offset. There is no guarantee that they will be consecutive
-    /// though.
-    ///
-    /// The size is the total size of the blob that the remote side told us.
-    /// It is not guaranteed to be correct, but it is guaranteed to be
-    /// consistent with all data in the batch. The size therefore represents
-    /// an upper bound on the maximum offset of all leaf items.
-    /// So it is guaranteed that `leaf.offset + leaf.size <= size` for all
-    /// leaf items in the batch.
-    ///
-    /// Batches should not become too large. Typically, a batch is just a few
-    /// parent nodes and a leaf.
-    ///
-    /// Batch is a vec so it can be moved into a task, which is unfortunately
-    /// necessary in typical io code.
-    fn write_batch(
-        &mut self,
-        size: u64,
-        batch: Vec<BaoContentItem>,
-    ) -> impl Future<Output = io::Result<()>>;
-
-    /// Sync the written data to permanent storage, if applicable.
-    /// E.g. for a file based implementation, this would call sync_data
-    /// on all files.
-    fn sync(&mut self) -> impl Future<Output = io::Result<()>>;
-}
-
-/// Implement BaoBatchWriter for mutable references
-impl<W: BaoBatchWriter> BaoBatchWriter for &mut W {
-    async fn write_batch(&mut self, size: u64, batch: Vec<BaoContentItem>) -> io::Result<()> {
-        (**self).write_batch(size, batch).await
-    }
-
-    async fn sync(&mut self) -> io::Result<()> {
-        (**self).sync().await
-    }
-}
-
-/// A wrapper around a batch writer that calls a progress callback for one leaf
-/// per batch.
-#[derive(Debug)]
-pub(crate) struct FallibleProgressBatchWriter<W, F>(W, F);
-
-impl<W: BaoBatchWriter, F: Fn(u64, usize) -> io::Result<()> + 'static>
-    FallibleProgressBatchWriter<W, F>
-{
-    /// Create a new `FallibleProgressBatchWriter` from an inner writer and a progress callback
-    ///
-    /// The `on_write` function is called for each write, with the `offset` as the first and the
-    /// length of the data as the second param. `on_write` must return an `io::Result`.
-    /// If `on_write` returns an error, the download is aborted.
-    pub fn new(inner: W, on_write: F) -> Self {
-        Self(inner, on_write)
-    }
-}
-
-impl<W: BaoBatchWriter, F: Fn(u64, usize) -> io::Result<()> + 'static> BaoBatchWriter
-    for FallibleProgressBatchWriter<W, F>
-{
-    async fn write_batch(&mut self, size: u64, batch: Vec<BaoContentItem>) -> io::Result<()> {
-        // find the offset and length of the first (usually only) chunk
-        let chunk = batch
-            .iter()
-            .filter_map(|item| {
-                if let BaoContentItem::Leaf(leaf) = item {
-                    Some((leaf.offset, leaf.data.len()))
-                } else {
-                    None
-                }
-            })
-            .next();
-        self.0.write_batch(size, batch).await?;
-        // call the progress callback
-        if let Some((offset, len)) = chunk {
-            (self.1)(offset, len)?;
-        }
-        Ok(())
-    }
-
-    async fn sync(&mut self) -> io::Result<()> {
-        self.0.sync().await
-    }
-}
-
-/// A mutable bao map.
-///
-/// This extends the readonly [`Map`] trait with methods to create and modify entries.
-pub trait MapMut: Map {
-    /// An entry that is possibly writable
-    type EntryMut: MapEntryMut;
-
-    /// Get an existing entry as an EntryMut.
-    ///
-    /// For implementations where EntryMut and Entry are the same type, this is just an alias for
-    /// `get`.
-    fn get_mut(
-        &self,
-        hash: &Hash,
-    ) -> impl Future<Output = io::Result<Option<Self::EntryMut>>> + Send;
-
-    /// Get an existing partial entry, or create a new one.
-    ///
-    /// We need to know the size of the partial entry. This might produce an
-    /// error e.g. if there is not enough space on disk.
-    fn get_or_create(
-        &self,
-        hash: Hash,
-        size: u64,
-    ) -> impl Future<Output = io::Result<Self::EntryMut>> + Send;
-
-    /// Find out if the data behind a `hash` is complete, partial, or not present.
-    ///
-    /// Note that this does not actually verify the on-disc data, but only checks in which section
-    /// of the store the entry is present.
-    fn entry_status(&self, hash: &Hash) -> impl Future<Output = io::Result<EntryStatus>> + Send;
-
-    /// Sync version of `entry_status`, for the doc sync engine until we can get rid of it.
-    ///
-    /// Don't count on this to be efficient.
-    fn entry_status_sync(&self, hash: &Hash) -> io::Result<EntryStatus>;
-
-    /// Upgrade a partial entry to a complete entry.
-    fn insert_complete(&self, entry: Self::EntryMut)
-        -> impl Future<Output = io::Result<()>> + Send;
-}
-
-/// Extension of [`Map`] to add misc methods used by the rpc calls.
-pub trait ReadableStore: Map {
-    /// list all blobs in the database. This includes both raw blobs that have
-    /// been imported, and hash sequences that have been created internally.
-    fn blobs(&self) -> impl Future<Output = io::Result<DbIter<Hash>>> + Send;
-    /// list all tags (collections or other explicitly added things) in the database
-    fn tags(
-        &self,
-        from: Option<Tag>,
-        to: Option<Tag>,
-    ) -> impl Future<Output = io::Result<DbIter<(Tag, HashAndFormat)>>> + Send;
-
-    /// Temp tags
-    fn temp_tags(&self) -> Box<dyn Iterator<Item = HashAndFormat> + Send + Sync + 'static>;
-
-    /// Perform a consistency check on the database
-    fn consistency_check(
-        &self,
-        repair: bool,
-        tx: BoxedProgressSender<ConsistencyCheckProgress>,
-    ) -> impl Future<Output = io::Result<()>> + Send;
-
-    /// list partial blobs in the database
-    fn partial_blobs(&self) -> impl Future<Output = io::Result<DbIter<Hash>>> + Send;
-
-    /// This trait method extracts a file to a local path.
-    ///
-    /// `hash` is the hash of the file
-    /// `target` is the path to the target file
-    /// `mode` is a hint how the file should be exported.
-    /// `progress` is a callback that is called with the total number of bytes that have been written
-    fn export(
-        &self,
-        hash: Hash,
-        target: PathBuf,
-        mode: ExportMode,
-        progress: ExportProgressCb,
-    ) -> impl Future<Output = io::Result<()>> + Send;
-}
-
-/// The mutable part of a Bao store.
-pub trait Store: ReadableStore + MapMut + std::fmt::Debug {
-    /// This trait method imports a file from a local path.
-    ///
-    /// `data` is the path to the file.
-    /// `mode` is a hint how the file should be imported.
-    /// `progress` is a sender that provides a way for the importer to send progress messages
-    /// when importing large files. This also serves as a way to cancel the import. If the
-    /// consumer of the progress messages is dropped, subsequent attempts to send progress
-    /// will fail.
-    ///
-    /// Returns the hash of the imported file. The reason to have this method is that some database
-    /// implementations might be able to import a file without copying it.
-    fn import_file(
-        &self,
-        data: PathBuf,
-        mode: ImportMode,
-        format: BlobFormat,
-        progress: impl ProgressSender<Msg = ImportProgress> + IdGenerator,
-    ) -> impl Future<Output = io::Result<(TempTag, u64)>> + Send;
-
-    /// Import data from memory.
-    ///
-    /// It is a special case of `import` that does not use the file system.
-    fn import_bytes(
-        &self,
-        bytes: Bytes,
-        format: BlobFormat,
-    ) -> impl Future<Output = io::Result<TempTag>> + Send;
-
-    /// Import data from a stream of bytes.
-    fn import_stream(
-        &self,
-        data: impl Stream<Item = io::Result<Bytes>> + Send + Unpin + 'static,
-        format: BlobFormat,
-        progress: impl ProgressSender<Msg = ImportProgress> + IdGenerator,
-    ) -> impl Future<Output = io::Result<(TempTag, u64)>> + Send;
-
-    /// Import data from an async byte reader.
-    fn import_reader(
-        &self,
-        data: impl AsyncRead + Send + Unpin + 'static,
-        format: BlobFormat,
-        progress: impl ProgressSender<Msg = ImportProgress> + IdGenerator,
-    ) -> impl Future<Output = io::Result<(TempTag, u64)>> + Send {
-        let stream = tokio_util::io::ReaderStream::new(data);
-        self.import_stream(stream, format, progress)
-    }
-
-    /// Set a tag
-    fn set_tag(
-        &self,
-        name: Tag,
-        hash: HashAndFormat,
-    ) -> impl Future<Output = io::Result<()>> + Send;
-
-    /// Rename a tag
-    fn rename_tag(&self, from: Tag, to: Tag) -> impl Future<Output = io::Result<()>> + Send;
-
-    /// Delete a single tag
-    fn delete_tag(&self, name: Tag) -> impl Future<Output = io::Result<()>> + Send {
-        self.delete_tags(Some(name.clone()), Some(name.successor()))
-    }
-
-    /// Bulk delete tags
-    fn delete_tags(
-        &self,
-        from: Option<Tag>,
-        to: Option<Tag>,
-    ) -> impl Future<Output = io::Result<()>> + Send;
-
-    /// Create a new tag
-    fn create_tag(&self, hash: HashAndFormat) -> impl Future<Output = io::Result<Tag>> + Send;
-
-    /// Create a temporary pin for this store
-    fn temp_tag(&self, value: HashAndFormat) -> TempTag;
-
-    /// Start the GC loop
-    ///
-    /// The gc task will shut down, when dropping the returned future.
-    fn gc_run<G, Gut>(&self, config: super::GcConfig, protected_cb: G) -> impl Future<Output = ()>
-    where
-        G: Fn() -> Gut,
-        Gut: Future<Output = BTreeSet<Hash>> + Send;
-
-    /// physically delete the given hashes from the store.
-    fn delete(&self, hashes: Vec<Hash>) -> impl Future<Output = io::Result<()>> + Send;
-
-    /// Shutdown the store.
-    fn shutdown(&self) -> impl Future<Output = ()> + Send;
-
-    /// Sync the store.
-    fn sync(&self) -> impl Future<Output = io::Result<()>> + Send;
-
-    /// Validate the database
-    ///
-    /// This will check that the file and outboard content is correct for all complete
-    /// entries, and output valid ranges for all partial entries.
-    ///
-    /// It will not check the internal consistency of the database.
-    fn validate(
-        &self,
-        repair: bool,
-        tx: BoxedProgressSender<ValidateProgress>,
-    ) -> impl Future<Output = io::Result<()>> + Send {
-        validate_impl(self, repair, tx)
-    }
-}
-
-async fn validate_impl(
-    store: &impl Store,
-    repair: bool,
-    tx: BoxedProgressSender<ValidateProgress>,
-) -> io::Result<()> {
-    use futures_buffered::BufferedStreamExt;
-
-    let validate_parallelism: usize = num_cpus::get();
-    let lp = LocalPool::new(local_pool::Config {
-        threads: validate_parallelism,
-        ..Default::default()
-    });
-    let complete = store.blobs().await?.collect::<io::Result<Vec<_>>>()?;
-    let partial = store
-        .partial_blobs()
-        .await?
-        .collect::<io::Result<Vec<_>>>()?;
-    tx.send(ValidateProgress::Starting {
-        total: complete.len() as u64,
-    })
-    .await?;
-    let complete_result = futures_lite::stream::iter(complete)
-        .map(|hash| {
-            let store = store.clone();
-            let tx = tx.clone();
-            lp.spawn(move || async move {
-                let entry = store
-                    .get(&hash)
-                    .await?
-                    .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "entry not found"))?;
-                let size = entry.size().value();
-                let outboard = entry.outboard().await?;
-                let data = entry.data_reader().await?;
-                let chunk_ranges = ChunkRanges::all();
-                let mut ranges = bao_tree::io::fsm::valid_ranges(outboard, data, &chunk_ranges);
-                let id = tx.new_id();
-                tx.send(ValidateProgress::Entry {
-                    id,
-                    hash,
-                    path: None,
-                    size,
-                })
-                .await?;
-                let mut actual_chunk_ranges = ChunkRanges::empty();
-                while let Some(item) = ranges.next().await {
-                    let item = item?;
-                    let offset = item.start.to_bytes();
-                    actual_chunk_ranges |= ChunkRanges::from(item);
-                    tx.try_send(ValidateProgress::EntryProgress { id, offset })?;
-                }
-                let expected_chunk_range =
-                    ChunkRanges::from(..BaoTree::new(size, IROH_BLOCK_SIZE).chunks());
-                let incomplete = actual_chunk_ranges == expected_chunk_range;
-                let error = if incomplete {
-                    None
-                } else {
-                    Some(format!(
-                        "expected chunk ranges {:?}, got chunk ranges {:?}",
-                        expected_chunk_range, actual_chunk_ranges
-                    ))
-                };
-                tx.send(ValidateProgress::EntryDone { id, error }).await?;
-                drop(ranges);
-                drop(entry);
-                io::Result::Ok((hash, incomplete))
-            })
-        })
-        .buffered_unordered(validate_parallelism)
-        .collect::<Vec<_>>()
-        .await;
-    let partial_result = futures_lite::stream::iter(partial)
-        .map(|hash| {
-            let store = store.clone();
-            let tx = tx.clone();
-            lp.spawn(move || async move {
-                let entry = store
-                    .get(&hash)
-                    .await?
-                    .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "entry not found"))?;
-                let size = entry.size().value();
-                let outboard = entry.outboard().await?;
-                let data = entry.data_reader().await?;
-                let chunk_ranges = ChunkRanges::all();
-                let mut ranges = bao_tree::io::fsm::valid_ranges(outboard, data, &chunk_ranges);
-                let id = tx.new_id();
-                tx.send(ValidateProgress::PartialEntry {
-                    id,
-                    hash,
-                    path: None,
-                    size,
-                })
-                .await?;
-                let mut actual_chunk_ranges = ChunkRanges::empty();
-                while let Some(item) = ranges.next().await {
-                    let item = item?;
-                    let offset = item.start.to_bytes();
-                    actual_chunk_ranges |= ChunkRanges::from(item);
-                    tx.try_send(ValidateProgress::PartialEntryProgress { id, offset })?;
-                }
-                tx.send(ValidateProgress::PartialEntryDone {
-                    id,
-                    ranges: RangeSpec::new(&actual_chunk_ranges),
-                })
-                .await?;
-                drop(ranges);
-                drop(entry);
-                io::Result::Ok(())
-            })
-        })
-        .buffered_unordered(validate_parallelism)
-        .collect::<Vec<_>>()
-        .await;
-    let mut to_downgrade = Vec::new();
-    for item in complete_result {
-        let (hash, incomplete) = item??;
-        if incomplete {
-            to_downgrade.push(hash);
-        }
-    }
-    for item in partial_result {
-        item??;
-    }
-    if repair {
-        return Err(io::Error::new(
-            io::ErrorKind::Other,
-            "repair not implemented",
-        ));
-    }
-    Ok(())
-}
-
-/// Configuration for the GC mark and sweep.
-#[derive(derive_more::Debug)]
-pub struct GcConfig {
-    /// The period at which to execute the GC.
-    pub period: Duration,
-    /// An optional callback called every time a GC round finishes.
-    #[debug("done_callback")]
-    pub done_callback: Option<Box<dyn Fn() + Send>>,
-}
-
-/// Implementation of the gc loop.
-pub(super) async fn gc_run_loop<S, F, Fut, G, Gut>(
-    store: &S,
-    config: GcConfig,
-    start_cb: F,
-    protected_cb: G,
-) where
-    S: Store,
-    F: Fn() -> Fut,
-    Fut: Future<Output = io::Result<()>> + Send,
-    G: Fn() -> Gut,
-    Gut: Future<Output = BTreeSet<Hash>> + Send,
-{
-    tracing::info!("Starting GC task with interval {:?}", config.period);
-    let mut live = BTreeSet::new();
-    'outer: loop {
-        if let Err(cause) = start_cb().await {
-            tracing::debug!("unable to notify the db of GC start: {cause}. Shutting down GC loop.");
-            break;
-        }
-        // do delay before the two phases of GC
-        tokio::time::sleep(config.period).await;
-        tracing::debug!("Starting GC");
-        live.clear();
-
-        let p = protected_cb().await;
-        live.extend(p);
-
-        tracing::debug!("Starting GC mark phase");
-        let live_ref = &mut live;
-        let mut stream = Gen::new(|co| async move {
-            if let Err(e) = gc_mark_task(store, live_ref, &co).await {
-                co.yield_(GcMarkEvent::Error(e)).await;
-            }
-        });
-        while let Some(item) = stream.next().await {
-            match item {
-                GcMarkEvent::CustomDebug(text) => {
-                    tracing::debug!("{}", text);
-                }
-                GcMarkEvent::CustomWarning(text, _) => {
-                    tracing::warn!("{}", text);
-                }
-                GcMarkEvent::Error(err) => {
-                    tracing::error!("Fatal error during GC mark {}", err);
-                    continue 'outer;
-                }
-            }
-        }
-        drop(stream);
-
-        tracing::debug!("Starting GC sweep phase");
-        let live_ref = &live;
-        let mut stream = Gen::new(|co| async move {
-            if let Err(e) = gc_sweep_task(store, live_ref, &co).await {
-                co.yield_(GcSweepEvent::Error(e)).await;
-            }
-        });
-        while let Some(item) = stream.next().await {
-            match item {
-                GcSweepEvent::CustomDebug(text) => {
-                    tracing::debug!("{}", text);
-                }
-                GcSweepEvent::CustomWarning(text, _) => {
-                    tracing::warn!("{}", text);
-                }
-                GcSweepEvent::Error(err) => {
-                    tracing::error!("Fatal error during GC mark {}", err);
-                    continue 'outer;
-                }
-            }
-        }
-        if let Some(ref cb) = config.done_callback {
-            cb();
-        }
-    }
-}
-
-/// Implementation of the gc method.
-pub(super) async fn gc_mark_task<'a>(
-    store: &'a impl Store,
-    live: &'a mut BTreeSet<Hash>,
-    co: &Co<GcMarkEvent>,
-) -> anyhow::Result<()> {
-    macro_rules! debug {
-        ($($arg:tt)*) => {
-            co.yield_(GcMarkEvent::CustomDebug(format!($($arg)*))).await;
-        };
-    }
-    macro_rules! warn {
-        ($($arg:tt)*) => {
-            co.yield_(GcMarkEvent::CustomWarning(format!($($arg)*), None)).await;
-        };
-    }
-    let mut roots = BTreeSet::new();
-    debug!("traversing tags");
-    for item in store.tags(None, None).await? {
-        let (name, haf) = item?;
-        debug!("adding root {:?} {:?}", name, haf);
-        roots.insert(haf);
-    }
-    debug!("traversing temp roots");
-    for haf in store.temp_tags() {
-        debug!("adding temp pin {:?}", haf);
-        roots.insert(haf);
-    }
-    for HashAndFormat { hash, format } in roots {
-        // we need to do this for all formats except raw
-        if live.insert(hash) && !format.is_raw() {
-            let Some(entry) = store.get(&hash).await? else {
-                warn!("gc: {} not found", hash);
-                continue;
-            };
-            if !entry.is_complete() {
-                warn!("gc: {} is partial", hash);
-                continue;
-            }
-            let Ok(reader) = entry.data_reader().await else {
-                warn!("gc: {} creating data reader failed", hash);
-                continue;
-            };
-            let Ok((mut stream, count)) = parse_hash_seq(reader).await else {
-                warn!("gc: {} parse failed", hash);
-                continue;
-            };
-            debug!("parsed collection {} {:?}", hash, count);
-            loop {
-                let item = match stream.next().await {
-                    Ok(Some(item)) => item,
-                    Ok(None) => break,
-                    Err(_err) => {
-                        warn!("gc: {} parse failed", hash);
-                        break;
-                    }
-                };
-                // if format != raw we would have to recurse here by adding this to current
-                live.insert(item);
-            }
-        }
-    }
-    debug!("gc mark done. found {} live blobs", live.len());
-    Ok(())
-}
-
-async fn gc_sweep_task(
-    store: &impl Store,
-    live: &BTreeSet<Hash>,
-    co: &Co<GcSweepEvent>,
-) -> anyhow::Result<()> {
-    let blobs = store.blobs().await?.chain(store.partial_blobs().await?);
-    let mut count = 0;
-    let mut batch = Vec::new();
-    for hash in blobs {
-        let hash = hash?;
-        if !live.contains(&hash) {
-            batch.push(hash);
-            count += 1;
-        }
-        if batch.len() >= 100 {
-            store.delete(batch.clone()).await?;
-            batch.clear();
-        }
-    }
-    if !batch.is_empty() {
-        store.delete(batch).await?;
-    }
-    co.yield_(GcSweepEvent::CustomDebug(format!(
-        "deleted {} blobs",
-        count
-    )))
-    .await;
-    Ok(())
-}
-
-/// An event related to GC
-#[derive(Debug)]
-pub enum GcMarkEvent {
-    /// A custom event (info)
-    CustomDebug(String),
-    /// A custom non critical error
-    CustomWarning(String, Option<anyhow::Error>),
-    /// An unrecoverable error during GC
-    Error(anyhow::Error),
-}
-
-/// An event related to GC
-#[derive(Debug)]
-pub enum GcSweepEvent {
-    /// A custom event (debug)
-    CustomDebug(String),
-    /// A custom non critical error
-    CustomWarning(String, Option<anyhow::Error>),
-    /// An unrecoverable error during GC
-    Error(anyhow::Error),
-}
-
-/// Progress messages for an import operation
-///
-/// An import operation involves computing the outboard of a file, and then
-/// either copying or moving the file into the database.
-#[allow(missing_docs)]
-#[derive(Debug)]
-pub enum ImportProgress {
-    /// Found a path
-    ///
-    /// This will be the first message for an id
-    Found { id: u64, name: String },
-    /// Progress when copying the file to the store
-    ///
-    /// This will be omitted if the store can use the file in place
-    ///
-    /// There will be multiple of these messages for an id
-    CopyProgress { id: u64, offset: u64 },
-    /// Determined the size
-    ///
-    /// This will come after `Found` and zero or more `CopyProgress` messages.
-    /// For unstable files, determining the size will only be done once the file
-    /// is fully copied.
-    Size { id: u64, size: u64 },
-    /// Progress when computing the outboard
-    ///
-    /// There will be multiple of these messages for an id
-    OutboardProgress { id: u64, offset: u64 },
-    /// Done computing the outboard
-    ///
-    /// This comes after `Size` and zero or more `OutboardProgress` messages
-    OutboardDone { id: u64, hash: Hash },
-}
-
-/// The import mode describes how files will be imported.
-///
-/// This is a hint to the import trait method. For some implementations, this
-/// does not make any sense. E.g. an in memory implementation will always have
-/// to copy the file into memory. Also, a disk based implementation might choose
-/// to copy small files even if the mode is `Reference`.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
-pub enum ImportMode {
-    /// This mode will copy the file into the database before hashing.
-    ///
-    /// This is the safe default because the file can not be accidentally modified
-    /// after it has been imported.
-    #[default]
-    Copy,
-    /// This mode will try to reference the file in place and assume it is unchanged after import.
-    ///
-    /// This has a large performance and storage benefit, but it is less safe since
-    /// the file might be modified after it has been imported.
-    ///
-    /// Stores are allowed to ignore this mode and always copy the file, e.g.
-    /// if the file is very small or if the store does not support referencing files.
-    TryReference,
-}
-/// The import mode describes how files will be imported.
-///
-/// This is a hint to the import trait method. For some implementations, this
-/// does not make any sense. E.g. an in memory implementation will always have
-/// to copy the file into memory. Also, a disk based implementation might choose
-/// to copy small files even if the mode is `Reference`.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize, Serialize)]
-pub enum ExportMode {
-    /// This mode will copy the file to the target directory.
-    ///
-    /// This is the safe default because the file can not be accidentally modified
-    /// after it has been exported.
-    #[default]
-    Copy,
-    /// This mode will try to move the file to the target directory and then reference it from
-    /// the database.
-    ///
-    /// This has a large performance and storage benefit, but it is less safe since
-    /// the file might be modified in the target directory after it has been exported.
-    ///
-    /// Stores are allowed to ignore this mode and always copy the file, e.g.
-    /// if the file is very small or if the store does not support referencing files.
-    TryReference,
-}
-
-/// The expected format of a hash being exported.
-#[derive(Debug, Clone, Serialize, Deserialize, Default)]
-pub enum ExportFormat {
-    /// The hash refers to any blob and will be exported to a single file.
-    #[default]
-    Blob,
-    /// The hash refers to a [`crate::format::collection::Collection`] blob
-    /// and all children of the collection shall be exported to one file per child.
-    ///
-    /// If the blob can be parsed as a [`BlobFormat::HashSeq`], and the first child contains
-    /// collection metadata, all other children of the collection will be exported to
-    /// a file each, with their collection name treated as a relative path to the export
-    /// destination path.
-    ///
-    /// If the blob cannot be parsed as a collection, the operation will fail.
-    Collection,
-}
-
-#[allow(missing_docs)]
-#[derive(Debug)]
-pub enum ExportProgress {
-    /// Starting to export to a file
-    ///
-    /// This will be the first message for an id
-    Start {
-        id: u64,
-        hash: Hash,
-        path: PathBuf,
-        stable: bool,
-    },
-    /// Progress when copying the file to the target
-    ///
-    /// This will be omitted if the store can move the file or use copy on write
-    ///
-    /// There will be multiple of these messages for an id
-    Progress { id: u64, offset: u64 },
-    /// Done exporting
-    Done { id: u64 },
-}
-
-/// Level for generic validation messages
-#[derive(
-    Debug, Clone, Copy, derive_more::Display, Serialize, Deserialize, PartialOrd, Ord, PartialEq, Eq,
-)]
-pub enum ReportLevel {
-    /// Very unimportant info messages
-    Trace,
-    /// Info messages
-    Info,
-    /// Warnings, something is not quite right
-    Warn,
-    /// Errors, something is very wrong
-    Error,
-}
-
-/// Progress updates for the validate operation
-#[derive(Debug, Serialize, Deserialize)]
-pub enum ConsistencyCheckProgress {
-    /// Consistency check started
-    Start,
-    /// Consistency check update
-    Update {
-        /// The message
-        message: String,
-        /// The entry this message is about, if any
-        entry: Option<Hash>,
-        /// The level of the message
-        level: ReportLevel,
-    },
-    /// Consistency check ended
-    Done,
-    /// We got an error and need to abort.
-    Abort(serde_error::Error),
-}
-
-/// Progress updates for the validate operation
-#[derive(Debug, Serialize, Deserialize)]
-pub enum ValidateProgress {
-    /// started validating
-    Starting {
-        /// The total number of entries to validate
-        total: u64,
-    },
-    /// We started validating a complete entry
-    Entry {
-        /// a new unique id for this entry
-        id: u64,
-        /// the hash of the entry
-        hash: Hash,
-        /// location of the entry.
-        ///
-        /// In case of a file, this is the path to the file.
-        /// Otherwise it might be an url or something else to uniquely identify the entry.
-        path: Option<String>,
-        /// The size of the entry, in bytes.
-        size: u64,
-    },
-    /// We got progress ingesting item `id`.
-    EntryProgress {
-        /// The unique id of the entry.
-        id: u64,
-        /// The offset of the progress, in bytes.
-        offset: u64,
-    },
-    /// We are done with `id`
-    EntryDone {
-        /// The unique id of the entry.
-        id: u64,
-        /// An error if we failed to validate the entry.
-        error: Option<String>,
-    },
-    /// We started validating an entry
-    PartialEntry {
-        /// a new unique id for this entry
-        id: u64,
-        /// the hash of the entry
-        hash: Hash,
-        /// location of the entry.
-        ///
-        /// In case of a file, this is the path to the file.
-        /// Otherwise it might be an url or something else to uniquely identify the entry.
-        path: Option<String>,
-        /// The best known size of the entry, in bytes.
-        size: u64,
-    },
-    /// We got progress ingesting item `id`.
-    PartialEntryProgress {
-        /// The unique id of the entry.
-        id: u64,
-        /// The offset of the progress, in bytes.
-        offset: u64,
-    },
-    /// We are done with `id`
-    PartialEntryDone {
-        /// The unique id of the entry.
-        id: u64,
-        /// Available ranges.
-        ranges: RangeSpec,
-    },
-    /// We are done with the whole operation.
-    AllDone,
-    /// We got an error and need to abort.
-    Abort(serde_error::Error),
-}
-
-/// Database events
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum Event {
-    /// A GC was started
-    GcStarted,
-    /// A GC was completed
-    GcCompleted,
-}
diff --git a/src/store/util.rs b/src/store/util.rs
new file mode 100644
index 000000000..240ad233f
--- /dev/null
+++ b/src/store/util.rs
@@ -0,0 +1,388 @@
+use std::{
+    borrow::Borrow,
+    fmt,
+    fs::{File, OpenOptions},
+    io::{self, Read, Write},
+    path::Path,
+    time::SystemTime,
+};
+
+use arrayvec::ArrayString;
+use bao_tree::{blake3, io::mixed::EncodedItem};
+use bytes::Bytes;
+use derive_more::{From, Into};
+
+mod mem_or_file;
+mod sparse_mem_file;
+use irpc::channel::mpsc;
+pub use mem_or_file::{FixedSize, MemOrFile};
+use range_collections::{range_set::RangeSetEntry, RangeSetRef};
+use ref_cast::RefCast;
+use serde::{de::DeserializeOwned, Deserialize, Serialize};
+pub use sparse_mem_file::SparseMemFile;
+pub mod observer;
+mod size_info;
+pub use size_info::SizeInfo;
+mod partial_mem_storage;
+pub use partial_mem_storage::PartialMemStorage;
+
+/// A named, persistent tag.
+#[derive(Serialize, Deserialize, Clone, PartialEq, Eq, PartialOrd, Ord, From, Into)]
+pub struct Tag(pub Bytes);
+
+impl From<&[u8]> for Tag {
+    fn from(value: &[u8]) -> Self {
+        Self(Bytes::copy_from_slice(value))
+    }
+}
+
+impl AsRef<[u8]> for Tag {
+    fn as_ref(&self) -> &[u8] {
+        self.0.as_ref()
+    }
+}
+
+impl Borrow<[u8]> for Tag {
+    fn borrow(&self) -> &[u8] {
+        self.0.as_ref()
+    }
+}
+
+impl From<String> for Tag {
+    fn from(value: String) -> Self {
+        Self(Bytes::from(value))
+    }
+}
+
+impl From<&str> for Tag {
+    fn from(value: &str) -> Self {
+        Self(Bytes::from(value.to_owned()))
+    }
+}
+
+impl fmt::Display for Tag {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let bytes = self.0.as_ref();
+        match std::str::from_utf8(bytes) {
+            Ok(s) => write!(f, "\"{s}\""),
+            Err(_) => write!(f, "{}", hex::encode(bytes)),
+        }
+    }
+}
+
+impl Tag {
+    /// Create a new tag that does not exist yet.
+    pub fn auto(time: SystemTime, exists: impl Fn(&[u8]) -> bool) -> Self {
+        let now = chrono::DateTime::<chrono::Utc>::from(time);
+        let mut i = 0;
+        loop {
+            let mut text = format!("auto-{}", now.format("%Y-%m-%dT%H:%M:%S%.3fZ"));
+            if i != 0 {
+                text.push_str(&format!("-{i}"));
+            }
+            if !exists(text.as_bytes()) {
+                return Self::from(text);
+            }
+            i += 1;
+        }
+    }
+
+    /// The successor of this tag in lexicographic order.
+    pub fn successor(&self) -> Self {
+        let mut bytes = self.0.to_vec();
+        // increment_vec(&mut bytes);
+        bytes.push(0);
+        Self(bytes.into())
+    }
+
+    /// If this is a prefix, get the next prefix.
+    ///
+    /// This is like successor, except that it will return None if the prefix is all 0xFF instead of appending a 0 byte.
+    pub fn next_prefix(&self) -> Option<Self> {
+        let mut bytes = self.0.to_vec();
+        if next_prefix(&mut bytes) {
+            Some(Self(bytes.into()))
+        } else {
+            None
+        }
+    }
+}
+
+pub struct DD<T: fmt::Display>(pub T);
+
+impl<T: fmt::Display> fmt::Debug for DD<T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Display::fmt(&self.0, f)
+    }
+}
+
+impl fmt::Debug for Tag {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_tuple("Tag").field(&DD(self)).finish()
+    }
+}
+
+pub(crate) fn limited_range(offset: u64, len: usize, buf_len: usize) -> std::ops::Range<usize> {
+    if offset < buf_len as u64 {
+        let start = offset as usize;
+        let end = start.saturating_add(len).min(buf_len);
+        start..end
+    } else {
+        0..0
+    }
+}
+
+/// zero copy get a limited slice from a `Bytes` as a `Bytes`.
+#[allow(dead_code)]
+pub(crate) fn get_limited_slice(bytes: &Bytes, offset: u64, len: usize) -> Bytes {
+    bytes.slice(limited_range(offset, len, bytes.len()))
+}
+
+mod redb_support {
+    use bytes::Bytes;
+    use redb::{Key as RedbKey, Value as RedbValue};
+
+    use super::Tag;
+
+    impl RedbValue for Tag {
+        type SelfType<'a> = Self;
+
+        type AsBytes<'a> = bytes::Bytes;
+
+        fn fixed_width() -> Option<usize> {
+            None
+        }
+
+        fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a>
+        where
+            Self: 'a,
+        {
+            Self(Bytes::copy_from_slice(data))
+        }
+
+        fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a>
+        where
+            Self: 'a,
+            Self: 'b,
+        {
+            value.0.clone()
+        }
+
+        fn type_name() -> redb::TypeName {
+            redb::TypeName::new("Tag")
+        }
+    }
+
+    impl RedbKey for Tag {
+        fn compare(data1: &[u8], data2: &[u8]) -> std::cmp::Ordering {
+            data1.cmp(data2)
+        }
+    }
+}
+
+pub trait RangeSetExt<T> {
+    fn upper_bound(&self) -> Option<T>;
+}
+
+impl<T: RangeSetEntry + Clone> RangeSetExt<T> for RangeSetRef<T> {
+    /// The upper (exclusive) bound of the bitfield
+    fn upper_bound(&self) -> Option<T> {
+        let boundaries = self.boundaries();
+        if boundaries.is_empty() {
+            Some(RangeSetEntry::min_value())
+        } else if boundaries.len() % 2 == 0 {
+            Some(boundaries[boundaries.len() - 1].clone())
+        } else {
+            None
+        }
+    }
+}
+
+pub fn write_checksummed<P: AsRef<Path>, T: Serialize>(path: P, data: &T) -> io::Result<()> {
+    // Build Vec with space for hash
+    let mut buffer = Vec::with_capacity(32 + 128);
+    buffer.extend_from_slice(&[0u8; 32]);
+
+    // Serialize directly into buffer
+    postcard::to_io(data, &mut buffer).map_err(io::Error::other)?;
+
+    // Compute hash over data (skip first 32 bytes)
+    let data_slice = &buffer[32..];
+    let hash = blake3::hash(data_slice);
+    buffer[..32].copy_from_slice(hash.as_bytes());
+
+    // Write all at once
+    let mut file = File::create(&path)?;
+    file.write_all(&buffer)?;
+    file.sync_all()?;
+
+    Ok(())
+}
+
+pub fn read_checksummed_and_truncate<T: DeserializeOwned>(path: impl AsRef<Path>) -> io::Result<T> {
+    let path = path.as_ref();
+    let mut file = OpenOptions::new()
+        .read(true)
+        .write(true)
+        .truncate(false)
+        .open(path)?;
+    let mut buffer = Vec::new();
+    file.read_to_end(&mut buffer)?;
+    file.set_len(0)?;
+    file.sync_all()?;
+
+    if buffer.is_empty() {
+        return Err(io::Error::new(
+            io::ErrorKind::InvalidData,
+            "File marked dirty",
+        ));
+    }
+
+    if buffer.len() < 32 {
+        return Err(io::Error::new(io::ErrorKind::InvalidData, "File too short"));
+    }
+
+    let stored_hash = &buffer[..32];
+    let data = &buffer[32..];
+
+    let computed_hash = blake3::hash(data);
+    if computed_hash.as_bytes() != stored_hash {
+        return Err(io::Error::new(io::ErrorKind::InvalidData, "Hash mismatch"));
+    }
+
+    let deserialized = postcard::from_bytes(data).map_err(io::Error::other)?;
+
+    Ok(deserialized)
+}
+
+#[cfg(test)]
+pub fn read_checksummed<T: DeserializeOwned>(path: impl AsRef<Path>) -> io::Result<T> {
+    use tracing::info;
+
+    let path = path.as_ref();
+    let mut file = File::open(path)?;
+    let mut buffer = Vec::new();
+    file.read_to_end(&mut buffer)?;
+    info!("{} {}", path.display(), hex::encode(&buffer));
+
+    if buffer.is_empty() {
+        return Err(io::Error::new(
+            io::ErrorKind::InvalidData,
+            "File marked dirty",
+        ));
+    }
+
+    if buffer.len() < 32 {
+        return Err(io::Error::new(io::ErrorKind::InvalidData, "File too short"));
+    }
+
+    let stored_hash = &buffer[..32];
+    let data = &buffer[32..];
+
+    let computed_hash = blake3::hash(data);
+    if computed_hash.as_bytes() != stored_hash {
+        return Err(io::Error::new(io::ErrorKind::InvalidData, "Hash mismatch"));
+    }
+
+    let deserialized = postcard::from_bytes(data).map_err(io::Error::other)?;
+
+    Ok(deserialized)
+}
+
+/// Helper trait for bytes for debugging
+pub trait SliceInfoExt: AsRef<[u8]> {
+    // get the addr of the actual data, to check if data was copied
+    fn addr(&self) -> usize;
+
+    // a short symbol string for the address
+    fn addr_short(&self) -> ArrayString<12> {
+        let addr = self.addr().to_le_bytes();
+        symbol_string(&addr)
+    }
+
+    #[allow(dead_code)]
+    fn hash_short(&self) -> ArrayString<10> {
+        crate::Hash::new(self.as_ref()).fmt_short()
+    }
+}
+
+impl<T: AsRef<[u8]>> SliceInfoExt for T {
+    fn addr(&self) -> usize {
+        self.as_ref() as *const [u8] as *const u8 as usize
+    }
+
+    fn hash_short(&self) -> ArrayString<10> {
+        crate::Hash::new(self.as_ref()).fmt_short()
+    }
+}
+
+pub fn symbol_string(data: &[u8]) -> ArrayString<12> {
+    const SYMBOLS: &[char] = &[
+        '😀', '😂', '😍', '😎', '😢', '😡', '😱', '😴', '🤓', '🤔', '🤗', '🤢', '🤡', '🤖', '👽',
+        '👾', '👻', '💀', '💩', '♥', '💥', '💦', '💨', '💫', '💬', '💭', '💰', '💳', '💼', '📈',
+        '📉', '📍', '📢', '📦', '📱', '📷', '📺', '🎃', '🎄', '🎉', '🎋', '🎍', '🎒', '🎓', '🎖',
+        '🎤', '🎧', '🎮', '🎰', '🎲', '🎳', '🎴', '🎵', '🎷', '🎸', '🎹', '🎺', '🎻', '🎼', '🏀',
+        '🏁', '🏆', '🏈',
+    ];
+    const BASE: usize = SYMBOLS.len(); // 64
+
+    // Hash the input with BLAKE3
+    let hash = blake3::hash(data);
+    let bytes = hash.as_bytes(); // 32-byte hash
+
+    // Create an ArrayString with capacity 12 (bytes)
+    let mut result = ArrayString::<12>::new();
+
+    // Fill with 3 symbols
+    for byte in bytes.iter().take(3) {
+        let byte = *byte as usize;
+        let index = byte % BASE;
+        result.push(SYMBOLS[index]); // Each char can be up to 4 bytes
+    }
+
+    result
+}
+
+pub struct ValueOrPoisioned<T>(pub Option<T>);
+
+impl<T: fmt::Debug> fmt::Debug for ValueOrPoisioned<T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match &self.0 {
+            Some(x) => x.fmt(f),
+            None => f.debug_tuple("Poisoned").finish(),
+        }
+    }
+}
+
+/// Given a prefix, increment it lexographically.
+///
+/// If the prefix is all FF, this will return false because there is no
+/// higher prefix than that.
+#[allow(dead_code)]
+pub(crate) fn next_prefix(bytes: &mut [u8]) -> bool {
+    for byte in bytes.iter_mut().rev() {
+        if *byte < 255 {
+            *byte += 1;
+            return true;
+        }
+        *byte = 0;
+    }
+    false
+}
+
+#[derive(ref_cast::RefCast)]
+#[repr(transparent)]
+pub struct BaoTreeSender(mpsc::Sender<EncodedItem>);
+
+impl BaoTreeSender {
+    pub fn new(sender: &mut mpsc::Sender<EncodedItem>) -> &mut Self {
+        BaoTreeSender::ref_cast_mut(sender)
+    }
+}
+
+impl bao_tree::io::mixed::Sender for BaoTreeSender {
+    type Error = irpc::channel::SendError;
+    async fn send(&mut self, item: EncodedItem) -> std::result::Result<(), Self::Error> {
+        self.0.send(item).await
+    }
+}
diff --git a/src/store/util/mem_or_file.rs b/src/store/util/mem_or_file.rs
new file mode 100644
index 000000000..8c948c672
--- /dev/null
+++ b/src/store/util/mem_or_file.rs
@@ -0,0 +1,144 @@
+use std::{
+    fmt::Debug,
+    fs::File,
+    io::{self, Read, Seek},
+};
+
+use bao_tree::io::{
+    mixed::ReadBytesAt,
+    sync::{ReadAt, Size},
+};
+use bytes::Bytes;
+
+use super::SliceInfoExt;
+
+/// A wrapper for a file with a fixed size.
+#[derive(Debug)]
+pub struct FixedSize<T> {
+    file: T,
+    pub size: u64,
+}
+
+impl<T> FixedSize<T> {
+    pub fn new(file: T, size: u64) -> Self {
+        Self { file, size }
+    }
+}
+
+impl FixedSize<File> {
+    pub fn try_clone(&self) -> io::Result<Self> {
+        Ok(Self::new(self.file.try_clone()?, self.size))
+    }
+}
+
+impl<T: ReadAt> ReadAt for FixedSize<T> {
+    fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<usize> {
+        self.file.read_at(offset, buf)
+    }
+}
+
+impl<T: ReadBytesAt> ReadBytesAt for FixedSize<T> {
+    fn read_bytes_at(&self, offset: u64, size: usize) -> io::Result<Bytes> {
+        self.file.read_bytes_at(offset, size)
+    }
+}
+
+impl<T> Size for FixedSize<T> {
+    fn size(&self) -> io::Result<Option<u64>> {
+        Ok(Some(self.size))
+    }
+}
+
+/// This is a general purpose Either, just like Result, except that the two cases
+/// are Mem for something that is in memory, and File for something that is somewhere
+/// external and only available via io.
+#[derive(Debug)]
+pub enum MemOrFile<M, F> {
+    /// We got it all in memory
+    Mem(M),
+    /// A file
+    File(F),
+}
+
+impl<M: AsRef<[u8]>, F: Debug> MemOrFile<M, F> {
+    pub fn fmt_short(&self) -> String {
+        match self {
+            Self::Mem(mem) => format!("Mem(size={},addr={})", mem.as_ref().len(), mem.addr_short()),
+            Self::File(_) => "File".to_string(),
+        }
+    }
+}
+
+impl<T> MemOrFile<Bytes, FixedSize<T>> {
+    pub fn size(&self) -> u64 {
+        match self {
+            MemOrFile::Mem(mem) => mem.len() as u64,
+            MemOrFile::File(file) => file.size,
+        }
+    }
+}
+
+impl<A: Read, B: Read> Read for MemOrFile<A, B> {
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        match self {
+            MemOrFile::Mem(mem) => mem.read(buf),
+            MemOrFile::File(file) => file.read(buf),
+        }
+    }
+}
+
+impl<A: Seek, B: Seek> Seek for MemOrFile<A, B> {
+    fn seek(&mut self, pos: io::SeekFrom) -> io::Result<u64> {
+        match self {
+            MemOrFile::Mem(mem) => mem.seek(pos),
+            MemOrFile::File(file) => file.seek(pos),
+        }
+    }
+}
+
+impl<A: AsRef<[u8]>, B: ReadAt> ReadAt for MemOrFile<A, B> {
+    fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<usize> {
+        match self {
+            MemOrFile::Mem(mem) => mem.as_ref().read_at(offset, buf),
+            MemOrFile::File(file) => file.read_at(offset, buf),
+        }
+    }
+}
+
+impl<A: ReadBytesAt, B: ReadBytesAt> ReadBytesAt for MemOrFile<A, B> {
+    fn read_bytes_at(&self, offset: u64, size: usize) -> io::Result<Bytes> {
+        match self {
+            MemOrFile::Mem(mem) => mem.read_bytes_at(offset, size),
+            MemOrFile::File(file) => file.read_bytes_at(offset, size),
+        }
+    }
+}
+
+impl<A: Size, B: Size> Size for MemOrFile<A, B> {
+    fn size(&self) -> io::Result<Option<u64>> {
+        match self {
+            MemOrFile::Mem(mem) => mem.size(),
+            MemOrFile::File(file) => file.size(),
+        }
+    }
+}
+
+impl<M: Default, F> Default for MemOrFile<M, F> {
+    fn default() -> Self {
+        MemOrFile::Mem(Default::default())
+    }
+}
+
+impl<F> MemOrFile<Bytes, F> {
+    /// Create an empty MemOrFile, using a Bytes for the Mem part
+    pub fn empty() -> Self {
+        MemOrFile::default()
+    }
+}
+
+impl<M, F> MemOrFile<M, F> {
+    /// True if this is a Mem
+    pub fn is_mem(&self) -> bool {
+        matches!(self, MemOrFile::Mem(_))
+    }
+}
diff --git a/src/store/util/observer.rs b/src/store/util/observer.rs
new file mode 100644
index 000000000..b612e57e7
--- /dev/null
+++ b/src/store/util/observer.rs
@@ -0,0 +1,12 @@
+use std::fmt::Debug;
+
+// A commutative combine trait for updates
+pub trait Combine: Debug {
+    fn combine(self, other: Self) -> Self;
+}
+
+#[allow(dead_code)]
+pub trait CombineInPlace: Combine {
+    fn combine_with(&mut self, other: Self) -> Self;
+    fn is_neutral(&self) -> bool;
+}
diff --git a/src/store/util/partial_mem_storage.rs b/src/store/util/partial_mem_storage.rs
new file mode 100644
index 000000000..2b658bfea
--- /dev/null
+++ b/src/store/util/partial_mem_storage.rs
@@ -0,0 +1,54 @@
+use std::io;
+
+use bao_tree::{
+    io::{sync::WriteAt, BaoContentItem},
+    BaoTree,
+};
+
+use super::{size_info::SizeInfo, SparseMemFile};
+use crate::{api::blobs::Bitfield, store::IROH_BLOCK_SIZE};
+
+/// An incomplete entry, with all the logic to keep track of the state of the entry
+/// and for observing changes.
+#[derive(Debug, Default)]
+pub struct PartialMemStorage {
+    pub(crate) data: SparseMemFile,
+    pub(crate) outboard: SparseMemFile,
+    pub(crate) size: SizeInfo,
+    pub(crate) bitfield: Bitfield,
+}
+
+impl PartialMemStorage {
+    pub fn current_size(&self) -> u64 {
+        self.bitfield.size()
+    }
+
+    pub fn bitfield(&self) -> &Bitfield {
+        &self.bitfield
+    }
+
+    pub fn write_batch(&mut self, size: u64, batch: &[BaoContentItem]) -> io::Result<()> {
+        let tree = BaoTree::new(size, IROH_BLOCK_SIZE);
+        for item in batch {
+            match item {
+                BaoContentItem::Parent(parent) => {
+                    if let Some(offset) = tree.pre_order_offset(parent.node) {
+                        let o0 = offset
+                            .checked_mul(64)
+                            .expect("u64 overflow multiplying to hash pair offset");
+                        let outboard = &mut self.outboard;
+                        let mut buf = [0u8; 64];
+                        buf[..32].copy_from_slice(parent.pair.0.as_bytes());
+                        buf[32..].copy_from_slice(parent.pair.1.as_bytes());
+                        outboard.write_all_at(o0, &buf)?;
+                    }
+                }
+                BaoContentItem::Leaf(leaf) => {
+                    self.size.write(leaf.offset, size);
+                    self.data.write_all_at(leaf.offset, leaf.data.as_ref())?;
+                }
+            }
+        }
+        Ok(())
+    }
+}
diff --git a/src/store/util/size_info.rs b/src/store/util/size_info.rs
new file mode 100644
index 000000000..cdcb03786
--- /dev/null
+++ b/src/store/util/size_info.rs
@@ -0,0 +1,58 @@
+use std::io;
+
+use bao_tree::io::sync::WriteAt;
+
+use crate::store::IROH_BLOCK_SIZE;
+
+/// Keep track of the most precise size we know of.
+///
+/// When in memory, we don't have to write the size for every chunk to a separate
+/// slot, but can just keep the best one.
+#[derive(Debug, Default)]
+pub struct SizeInfo {
+    pub offset: u64,
+    pub size: u64,
+}
+
+#[allow(dead_code)]
+impl SizeInfo {
+    /// Create a new size info for a complete file of size `size`.
+    pub(crate) fn complete(size: u64) -> Self {
+        let mask = (1 << IROH_BLOCK_SIZE.chunk_log()) - 1;
+        // offset of the last bao chunk in a file of size `size`
+        let last_chunk_offset = size & mask;
+        Self {
+            offset: last_chunk_offset,
+            size,
+        }
+    }
+
+    /// Write a size at the given offset. The size at the highest offset is going to be kept.
+    pub fn write(&mut self, offset: u64, size: u64) {
+        // >= instead of > because we want to be able to update size 0, the initial value.
+        if offset >= self.offset {
+            self.offset = offset;
+            self.size = size;
+        }
+    }
+
+    /// The current size, representing the most correct size we know.
+    pub fn current_size(&self) -> u64 {
+        self.size
+    }
+
+    /// Persist into a file where each chunk has its own slot.
+    pub fn persist(&self, mut target: impl WriteAt) -> io::Result<()> {
+        let size_offset = (self.offset >> IROH_BLOCK_SIZE.chunk_log()) << 3;
+        target.write_all_at(size_offset, self.size.to_le_bytes().as_slice())?;
+        Ok(())
+    }
+
+    /// Convert to a vec in slot format.
+    #[allow(dead_code)]
+    pub fn to_vec(&self) -> Vec<u8> {
+        let mut res = Vec::new();
+        self.persist(&mut res).expect("io error writing to vec");
+        res
+    }
+}
diff --git a/src/util/sparse_mem_file.rs b/src/store/util/sparse_mem_file.rs
similarity index 87%
rename from src/util/sparse_mem_file.rs
rename to src/store/util/sparse_mem_file.rs
index ec6e43154..2f8687874 100644
--- a/src/util/sparse_mem_file.rs
+++ b/src/store/util/sparse_mem_file.rs
@@ -1,7 +1,10 @@
-use std::io;
+use std::{io, ops::Deref};
 
-use bao_tree::io::sync::{ReadAt, Size, WriteAt};
-use derive_more::Deref;
+use bao_tree::io::{
+    mixed::ReadBytesAt,
+    sync::{ReadAt, Size, WriteAt},
+};
+use bytes::Bytes;
 use range_collections::{range_set::RangeSetRange, RangeSet2};
 
 /// A file that is sparse in memory
@@ -31,11 +34,11 @@ impl From<Vec<u8>> for SparseMemFile {
     }
 }
 
-impl TryInto<Vec<u8>> for SparseMemFile {
+impl TryFrom<SparseMemFile> for Vec<u8> {
     type Error = io::Error;
 
-    fn try_into(self) -> Result<Vec<u8>, Self::Error> {
-        let (data, ranges) = self.into_parts();
+    fn try_from(value: SparseMemFile) -> Result<Self, Self::Error> {
+        let (data, ranges) = value.into_parts();
         if ranges == RangeSet2::from(0..data.len()) {
             Ok(data)
         } else {
@@ -99,6 +102,12 @@ impl ReadAt for SparseMemFile {
     }
 }
 
+impl ReadBytesAt for SparseMemFile {
+    fn read_bytes_at(&self, offset: u64, size: usize) -> io::Result<Bytes> {
+        self.data.read_bytes_at(offset, size)
+    }
+}
+
 impl WriteAt for SparseMemFile {
     fn write_at(&mut self, offset: u64, buf: &[u8]) -> io::Result<usize> {
         let start: usize = offset.try_into().map_err(|_| io::ErrorKind::InvalidInput)?;
diff --git a/src/test.rs b/src/test.rs
new file mode 100644
index 000000000..c0760a088
--- /dev/null
+++ b/src/test.rs
@@ -0,0 +1,63 @@
+use std::future::IntoFuture;
+
+use n0_future::{stream, StreamExt};
+use rand::{RngCore, SeedableRng};
+
+use crate::{
+    api::{blobs::AddBytesOptions, tags::TagInfo, RequestResult, Store},
+    hashseq::HashSeq,
+    BlobFormat,
+};
+
+pub async fn create_random_blobs<R: rand::Rng>(
+    store: &Store,
+    num_blobs: usize,
+    blob_size: impl Fn(usize, &mut R) -> usize,
+    mut rand: R,
+) -> anyhow::Result<Vec<TagInfo>> {
+    // generate sizes and seeds, non-parrallelized so it is deterministic
+    let sizes = (0..num_blobs)
+        .map(|n| (blob_size(n, &mut rand), rand.r#gen::<u64>()))
+        .collect::<Vec<_>>();
+    // generate random data and add it to the store
+    let infos = stream::iter(sizes)
+        .then(|(size, seed)| {
+            let mut rand = rand::rngs::StdRng::seed_from_u64(seed);
+            let mut data = vec![0u8; size];
+            rand.fill_bytes(&mut data);
+            store.add_bytes(data).into_future()
+        })
+        .collect::<Vec<_>>()
+        .await;
+    let infos = infos.into_iter().collect::<RequestResult<Vec<_>>>()?;
+    Ok(infos)
+}
+
+pub async fn add_hash_sequences<R: rand::Rng>(
+    store: &Store,
+    tags: &[TagInfo],
+    num_seqs: usize,
+    seq_size: impl Fn(usize, &mut R) -> usize,
+    mut rand: R,
+) -> anyhow::Result<Vec<TagInfo>> {
+    let infos = stream::iter(0..num_seqs)
+        .then(|n| {
+            let size = seq_size(n, &mut rand);
+            let hs = (0..size)
+                .map(|_| {
+                    let j = rand.gen_range(0..tags.len());
+                    tags[j].hash
+                })
+                .collect::<HashSeq>();
+            store
+                .add_bytes_with_opts(AddBytesOptions {
+                    data: hs.into(),
+                    format: BlobFormat::HashSeq,
+                })
+                .into_future()
+        })
+        .collect::<Vec<_>>()
+        .await;
+    let infos = infos.into_iter().collect::<RequestResult<Vec<_>>>()?;
+    Ok(infos)
+}
diff --git a/src/tests.rs b/src/tests.rs
new file mode 100644
index 000000000..f71b3e61c
--- /dev/null
+++ b/src/tests.rs
@@ -0,0 +1,758 @@
+use std::{collections::HashSet, io, ops::Range, path::PathBuf};
+
+use bao_tree::ChunkRanges;
+use bytes::Bytes;
+use iroh::{protocol::Router, Endpoint, NodeId, Watcher};
+use irpc::RpcMessage;
+use n0_future::{task::AbortOnDropHandle, StreamExt};
+use tempfile::TempDir;
+use testresult::TestResult;
+use tokio::sync::{mpsc, watch};
+use tracing::info;
+
+use crate::{
+    api::{blobs::Bitfield, Store},
+    get,
+    hashseq::HashSeq,
+    net_protocol::Blobs,
+    protocol::{ChunkRangesSeq, GetManyRequest, ObserveRequest, PushRequest},
+    provider::Event,
+    store::{
+        fs::{
+            tests::{create_n0_bao, test_data, INTERESTING_SIZES},
+            FsStore,
+        },
+        mem::MemStore,
+        util::observer::Combine,
+    },
+    util::sink::Drain,
+    BlobFormat, Hash, HashAndFormat,
+};
+
+// #[tokio::test]
+// #[traced_test]
+// async fn two_nodes_blobs_downloader_smoke() -> TestResult<()> {
+//     let (_testdir, (r1, store1, _), (r2, store2, _)) = two_node_test_setup().await?;
+//     let sizes = INTERESTING_SIZES;
+//     let mut tts = Vec::new();
+//     for size in sizes {
+//         tts.push(store1.add_bytes(test_data(size)).await?);
+//     }
+//     let addr1 = r1.endpoint().node_addr().await?;
+//     // test that we can download directly, without a downloader
+//     // let no_downloader_tt = store1.add_slice("test").await?;
+//     // let conn = r2.endpoint().connect(addr1.clone(), crate::ALPN).await?;
+//     // let stats = store2.download().fetch(conn, *no_downloader_tt.hash_and_format(), None).await?;
+//     // println!("stats: {:?}", stats);
+//     // return Ok(());
+
+//     // tell ep2 about the addr of ep1, so we don't need to rely on node discovery
+//     r2.endpoint().add_node_addr(addr1.clone())?;
+//     let d1 = Downloader::new(store2.clone(), r2.endpoint().clone());
+//     for (tt, size) in tts.iter().zip(sizes) {
+//         // protect the downloaded data from being deleted
+//         let _tt2 = store2.tags().temp_tag(*tt.hash()).await?;
+//         let request = DownloadRequest::new(*tt.hash_and_format(), [addr1.node_id]);
+//         let handle = d1.queue(request).await;
+//         handle.await?;
+//         assert_eq!(store2.get_bytes(*tt.hash()).await?, test_data(size));
+//     }
+//     Ok(())
+// }
+
+// #[tokio::test]
+// #[traced_test]
+// async fn two_nodes_blobs_downloader_progress() -> TestResult<()> {
+//     let (_testdir, (r1, store1, _), (r2, store2, _)) = two_node_test_setup().await?;
+//     let size = 1024 * 1024 * 8 + 1;
+//     let tt = store1.add_bytes(test_data(size)).await?;
+//     let addr1 = r1.endpoint().node_addr().await?;
+
+//     // tell ep2 about the addr of ep1, so we don't need to rely on node discovery
+//     r2.endpoint().add_node_addr(addr1.clone())?;
+//     // create progress channel - big enough to not block
+//     let (tx, rx) = mpsc::channel(1024 * 1024);
+//     let d1 = Downloader::new(store2.clone(), r2.endpoint().clone());
+//     // protect the downloaded data from being deleted
+//     let _tt2 = store2.tags().temp_tag(*tt.hash()).await?;
+//     let request = DownloadRequest::new(*tt.hash_and_format(), [addr1.node_id]).progress_sender(tx);
+//     let handle = d1.queue(request).await;
+//     handle.await?;
+//     let progress = drain(rx).await;
+//     assert!(!progress.is_empty());
+//     assert_eq!(store2.get_bytes(*tt.hash()).await?, test_data(size));
+//     Ok(())
+// }
+
+// #[tokio::test]
+// async fn three_nodes_blobs_downloader_switch() -> TestResult<()> {
+//     // tracing_subscriber::fmt::try_init().ok();
+//     let testdir = tempfile::tempdir()?;
+//     let (r1, store1, _) = node_test_setup(testdir.path().join("a")).await?;
+//     let (r2, store2, _) = node_test_setup(testdir.path().join("b")).await?;
+//     let (r3, store3, _) = node_test_setup(testdir.path().join("c")).await?;
+//     let addr1 = r1.endpoint().node_addr().await?;
+//     let addr2 = r2.endpoint().node_addr().await?;
+//     let size = 1024 * 1024 * 8 + 1;
+//     let data = test_data(size);
+//     // a has the data just partially
+//     let ranges = ChunkRanges::from(..ChunkNum(17));
+//     let (hash, bao) = create_n0_bao(&data, &ranges)?;
+//     let _tt1 = store1.tags().temp_tag(hash).await?;
+//     store1.import_bao_bytes(hash, ranges, bao).await?;
+//     // b has the data completely
+//     let _tt2: crate::api::TempTag = store2.add_bytes(data).await?;
+
+//     // tell ep3 about the addr of ep1 and ep2, so we don't need to rely on node discovery
+//     r3.endpoint().add_node_addr(addr1.clone())?;
+//     r3.endpoint().add_node_addr(addr2.clone())?;
+//     // create progress channel - big enough to not block
+//     let (tx, rx) = mpsc::channel(1024 * 1024);
+//     let d1 = Downloader::new(store3.clone(), r3.endpoint().clone());
+//     // protect the downloaded data from being deleted
+//     let _tt3 = store3.tags().temp_tag(hash).await?;
+//     let request = DownloadRequest::new(HashAndFormat::raw(hash), [addr1.node_id, addr2.node_id])
+//         .progress_sender(tx);
+//     let handle = d1.queue(request).await;
+//     handle.await?;
+//     let progress = drain(rx).await;
+//     assert!(!progress.is_empty());
+//     assert_eq!(store3.get_bytes(hash).await?, test_data(size));
+//     Ok(())
+// }
+
+// #[tokio::test]
+// async fn three_nodes_blobs_downloader_range_switch() -> TestResult<()> {
+//     // tracing_subscriber::fmt::try_init().ok();
+//     let testdir = tempfile::tempdir()?;
+//     let (r1, store1, _) = node_test_setup(testdir.path().join("a")).await?;
+//     let (r2, store2, _) = node_test_setup(testdir.path().join("b")).await?;
+//     let (r3, store3, _) = node_test_setup(testdir.path().join("c")).await?;
+//     let addr1 = r1.endpoint().node_addr().await?;
+//     let addr2 = r2.endpoint().node_addr().await?;
+//     let size = 1024 * 1024 * 8 + 1;
+//     let data = test_data(size);
+//     // a has the data just partially
+//     let ranges = ChunkRanges::chunks(..15);
+//     let (hash, bao) = create_n0_bao(&data, &ranges)?;
+//     let _tt1 = store1.tags().temp_tag(hash).await?;
+//     store1.import_bao_bytes(hash, ranges, bao).await?;
+//     // b has the data completely
+//     let _tt2: crate::api::TempTag = store2.add_bytes(data.clone()).await?;
+
+//     // tell ep3 about the addr of ep1 and ep2, so we don't need to rely on node discovery
+//     r3.endpoint().add_node_addr(addr1.clone())?;
+//     r3.endpoint().add_node_addr(addr2.clone())?;
+//     let range = 10000..20000;
+//     let request = GetRequest::builder()
+//         .root(ChunkRanges::bytes(range.clone()))
+//         .build(hash);
+//     // create progress channel - big enough to not block
+//     let (tx, rx) = mpsc::channel(1024 * 1024);
+//     let d1 = Downloader::new(store3.clone(), r3.endpoint().clone());
+//     // protect the downloaded data from being deleted
+//     let _tt3 = store3.tags().temp_tag(hash).await?;
+//     let request = DownloadRequest::new(request, [addr1.node_id, addr2.node_id]).progress_sender(tx);
+//     let handle = d1.queue(request).await;
+//     handle.await?;
+//     let progress = drain(rx).await;
+//     assert!(!progress.is_empty());
+//     let bitfield = store3.observe(hash).await?;
+//     assert_eq!(bitfield.ranges, ChunkRanges::bytes(range.clone()));
+//     let bytes = store3
+//         .export_ranges(hash, range.clone())
+//         .concatenate()
+//         .await?;
+//     assert_eq!(&bytes, &data[range.start as usize..range.end as usize]);
+//     // assert_eq!(store3.get_bytes(hash).await?, test_data(size));
+//     Ok(())
+// }
+
+// #[tokio::test]
+// async fn threee_nodes_hash_seq_downloader_switch() -> TestResult<()> {
+//     let testdir = tempfile::tempdir()?;
+//     let (r1, store1, _) = node_test_setup(testdir.path().join("a")).await?;
+//     let (r2, store2, _) = node_test_setup(testdir.path().join("b")).await?;
+//     let (r3, store3, _) = node_test_setup(testdir.path().join("c")).await?;
+//     let addr1 = r1.endpoint().node_addr().await?;
+//     let addr2 = r2.endpoint().node_addr().await?;
+//     let sizes = INTERESTING_SIZES;
+//     // store1 contains the hash seq, but just the first 3 items
+//     add_test_hash_seq_incomplete(&store1, sizes, |x| {
+//         if x < 6 {
+//             ChunkRanges::all()
+//         } else {
+//             ChunkRanges::from(..ChunkNum(1))
+//         }
+//     })
+//     .await?;
+//     let content = add_test_hash_seq(&store2, sizes).await?;
+
+//     // tell ep3 about the addr of ep1 and ep2, so we don't need to rely on node discovery
+//     r3.endpoint().add_node_addr(addr1.clone())?;
+//     r3.endpoint().add_node_addr(addr2.clone())?;
+//     // create progress channel - big enough to not block
+//     let (tx, rx) = mpsc::channel(1024 * 1024);
+//     let d1 = Downloader::new(store3.clone(), r3.endpoint().clone());
+//     // protect the downloaded data from being deleted
+//     let _tt3 = store3.tags().temp_tag(content).await?;
+//     let request = DownloadRequest::new(content, [addr1.node_id, addr2.node_id]).progress_sender(tx);
+//     let handle = d1.queue(request).await;
+//     handle.await?;
+//     let progress = drain(rx).await;
+//     assert!(!progress.is_empty());
+//     println!("progress: {:?}", progress);
+//     check_presence(&store3, &sizes).await?;
+//     Ok(())
+// }
+
+#[allow(dead_code)]
+async fn drain<T: RpcMessage>(mut rx: mpsc::Receiver<T>) -> Vec<T> {
+    let mut items = Vec::new();
+    while let Some(item) = rx.recv().await {
+        items.push(item);
+    }
+    items
+}
+
+async fn two_nodes_get_blobs(
+    r1: Router,
+    store1: &Store,
+    r2: Router,
+    store2: &Store,
+) -> TestResult<()> {
+    let sizes = INTERESTING_SIZES;
+    let mut tts = Vec::new();
+    for size in sizes {
+        tts.push(store1.add_bytes(test_data(size)).await?);
+    }
+    let addr1 = r1.endpoint().node_addr().initialized().await?;
+    let conn = r2.endpoint().connect(addr1, crate::ALPN).await?;
+    for size in sizes {
+        let hash = Hash::new(test_data(size));
+        store2.remote().fetch(conn.clone(), hash).await?;
+        let actual = store2.get_bytes(hash).await?;
+        assert_eq!(actual, test_data(size));
+    }
+    tokio::try_join!(r1.shutdown(), r2.shutdown())?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn two_nodes_get_blobs_fs() -> TestResult<()> {
+    let (_testdir, (r1, store1, _), (r2, store2, _)) = two_node_test_setup_fs().await?;
+    two_nodes_get_blobs(r1, &store1, r2, &store2).await
+}
+
+#[tokio::test]
+async fn two_nodes_get_blobs_mem() -> TestResult<()> {
+    let ((r1, store1), (r2, store2)) = two_node_test_setup_mem().await?;
+    two_nodes_get_blobs(r1, &store1, r2, &store2).await
+}
+
+async fn two_nodes_observe(
+    r1: Router,
+    store1: &Store,
+    r2: Router,
+    store2: &Store,
+) -> TestResult<()> {
+    let size = 1024 * 1024 * 8 + 1;
+    let data = test_data(size);
+    let (hash, bao) = create_n0_bao(&data, &ChunkRanges::all())?;
+    let addr1 = r1.endpoint().node_addr().initialized().await?;
+    let conn = r2.endpoint().connect(addr1, crate::ALPN).await?;
+    let mut stream = store2
+        .remote()
+        .observe(conn.clone(), ObserveRequest::new(hash));
+    let remote_observe_task = tokio::spawn(async move {
+        let mut current = Bitfield::empty();
+        while let Some(item) = stream.next().await {
+            current = current.combine(item?);
+            if current.is_validated() {
+                break;
+            }
+        }
+        io::Result::Ok(())
+    });
+    store1
+        .import_bao_bytes(hash, ChunkRanges::all(), bao)
+        .await?;
+    remote_observe_task.await??;
+    tokio::try_join!(r1.shutdown(), r2.shutdown())?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn two_nodes_observe_fs() -> TestResult<()> {
+    tracing_subscriber::fmt::try_init().ok();
+    let (_testdir, (r1, store1, _), (r2, store2, _)) = two_node_test_setup_fs().await?;
+    two_nodes_observe(r1, &store1, r2, &store2).await
+}
+
+#[tokio::test]
+async fn two_nodes_observe_mem() -> TestResult<()> {
+    tracing_subscriber::fmt::try_init().ok();
+    let ((r1, store1), (r2, store2)) = two_node_test_setup_mem().await?;
+    two_nodes_observe(r1, &store1, r2, &store2).await
+}
+
+async fn two_nodes_get_many(
+    r1: Router,
+    store1: &Store,
+    r2: Router,
+    store2: &Store,
+) -> TestResult<()> {
+    let sizes = INTERESTING_SIZES;
+    let mut tts = Vec::new();
+    for size in sizes {
+        tts.push(store1.add_bytes(test_data(size)).await?);
+    }
+    let hashes = tts.iter().map(|tt| tt.hash).collect::<Vec<_>>();
+    let addr1 = r1.endpoint().node_addr().initialized().await?;
+    let conn = r2.endpoint().connect(addr1, crate::ALPN).await?;
+    store2
+        .remote()
+        .execute_get_many(conn, GetManyRequest::new(hashes, ChunkRangesSeq::all()))
+        .await?;
+    for size in sizes {
+        let expected = test_data(size);
+        let hash = Hash::new(&expected);
+        let actual = store2.get_bytes(hash).await?;
+        assert_eq!(actual, expected);
+    }
+    tokio::try_join!(r1.shutdown(), r2.shutdown())?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn two_nodes_get_many_fs() -> TestResult<()> {
+    tracing_subscriber::fmt::try_init().ok();
+    let (_testdir, (r1, store1, _), (r2, store2, _)) = two_node_test_setup_fs().await?;
+    two_nodes_get_many(r1, &store1, r2, &store2).await
+}
+
+#[tokio::test]
+async fn two_nodes_get_many_mem() -> TestResult<()> {
+    tracing_subscriber::fmt::try_init().ok();
+    let ((r1, store1), (r2, store2)) = two_node_test_setup_mem().await?;
+    two_nodes_get_many(r1, &store1, r2, &store2).await
+}
+
+fn event_handler(
+    allowed_nodes: impl IntoIterator<Item = NodeId>,
+) -> (
+    mpsc::Sender<Event>,
+    watch::Receiver<usize>,
+    AbortOnDropHandle<()>,
+) {
+    let (count_tx, count_rx) = tokio::sync::watch::channel(0usize);
+    let (events_tx, mut events_rx) = mpsc::channel::<Event>(16);
+    let allowed_nodes = allowed_nodes.into_iter().collect::<HashSet<_>>();
+    let task = AbortOnDropHandle::new(tokio::task::spawn(async move {
+        while let Some(event) = events_rx.recv().await {
+            match event {
+                Event::ClientConnected {
+                    node_id, permitted, ..
+                } => {
+                    permitted.send(allowed_nodes.contains(&node_id)).await.ok();
+                }
+                Event::PushRequestReceived { permitted, .. } => {
+                    permitted.send(true).await.ok();
+                }
+                Event::TransferCompleted { .. } => {
+                    count_tx.send_modify(|count| *count += 1);
+                }
+                _ => {}
+            }
+        }
+    }));
+    (events_tx, count_rx, task)
+}
+
+async fn two_nodes_push_blobs(
+    r1: Router,
+    store1: &Store,
+    r2: Router,
+    store2: &Store,
+    mut count_rx: tokio::sync::watch::Receiver<usize>,
+) -> TestResult<()> {
+    let sizes = INTERESTING_SIZES;
+    let mut tts = Vec::new();
+    for size in sizes {
+        tts.push(store1.add_bytes(test_data(size)).await?);
+    }
+    let addr2 = r2.endpoint().node_addr().initialized().await?;
+    let conn = r1.endpoint().connect(addr2, crate::ALPN).await?;
+    for size in sizes {
+        let hash = Hash::new(test_data(size));
+        // let data = get::request::get_blob(conn.clone(), hash).bytes().await?;
+        store1
+            .remote()
+            .execute_push_sink(
+                conn.clone(),
+                PushRequest::new(hash, ChunkRangesSeq::root()),
+                Drain,
+            )
+            .await?;
+        count_rx.changed().await?;
+        let actual = store2.get_bytes(hash).await?;
+        assert_eq!(actual, test_data(size));
+    }
+    tokio::try_join!(r1.shutdown(), r2.shutdown())?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn two_nodes_push_blobs_fs() -> TestResult<()> {
+    tracing_subscriber::fmt::try_init().ok();
+    let testdir = tempfile::tempdir()?;
+    let (r1, store1, _) = node_test_setup_fs(testdir.path().join("a")).await?;
+    let (events_tx, count_rx, _task) = event_handler([r1.endpoint().node_id()]);
+    let (r2, store2, _) =
+        node_test_setup_with_events_fs(testdir.path().join("b"), Some(events_tx)).await?;
+    two_nodes_push_blobs(r1, &store1, r2, &store2, count_rx).await
+}
+
+#[tokio::test]
+async fn two_nodes_push_blobs_mem() -> TestResult<()> {
+    tracing_subscriber::fmt::try_init().ok();
+    let (r1, store1) = node_test_setup_mem().await?;
+    let (events_tx, count_rx, _task) = event_handler([r1.endpoint().node_id()]);
+    let (r2, store2) = node_test_setup_with_events_mem(Some(events_tx)).await?;
+    two_nodes_push_blobs(r1, &store1, r2, &store2, count_rx).await
+}
+
+pub async fn add_test_hash_seq(
+    blobs: &Store,
+    sizes: impl IntoIterator<Item = usize>,
+) -> TestResult<HashAndFormat> {
+    let batch = blobs.batch().await?;
+    let mut tts = Vec::new();
+    for size in sizes {
+        tts.push(batch.add_bytes(test_data(size)).await?);
+    }
+    let hash_seq = tts.iter().map(|tt| *tt.hash()).collect::<HashSeq>();
+    let root = batch
+        .add_bytes_with_opts((hash_seq, BlobFormat::HashSeq))
+        .with_named_tag("hs")
+        .await?;
+    Ok(root)
+}
+
+pub async fn add_test_hash_seq_incomplete(
+    blobs: &Store,
+    sizes: impl IntoIterator<Item = usize>,
+    present: impl Fn(usize) -> ChunkRanges,
+) -> TestResult<HashAndFormat> {
+    let batch = blobs.batch().await?;
+    let mut tts = Vec::new();
+    for (i, size) in sizes.into_iter().enumerate() {
+        let data = test_data(size);
+        // figure out the ranges to import, and manually create a temp tag.
+        let ranges = present(i + 1);
+        let (hash, bao) = create_n0_bao(&data, &ranges)?;
+        // why isn't import_bao_bytes returning a temp tag anyway?
+        tts.push(batch.temp_tag(hash).await?);
+        if !ranges.is_empty() {
+            blobs.import_bao_bytes(hash, ranges, bao).await?;
+        }
+    }
+    let hash_seq = tts.iter().map(|tt| *tt.hash()).collect::<HashSeq>();
+    let hash_seq_bytes = Bytes::from(hash_seq);
+    let ranges = present(0);
+    let (root, bao) = create_n0_bao(&hash_seq_bytes, &ranges)?;
+    let content = HashAndFormat::hash_seq(root);
+    blobs.tags().create(content).await?;
+    blobs.import_bao_bytes(root, ranges, bao).await?;
+    Ok(content)
+}
+
+async fn check_presence(store: &Store, sizes: &[usize]) -> TestResult<()> {
+    for size in sizes {
+        let expected = test_data(*size);
+        let hash = Hash::new(&expected);
+        let actual = store
+            .export_bao(hash, ChunkRanges::all())
+            .data_to_bytes()
+            .await?;
+        assert_eq!(actual, expected);
+    }
+    Ok(())
+}
+
+pub async fn node_test_setup_fs(db_path: PathBuf) -> TestResult<(Router, FsStore, PathBuf)> {
+    node_test_setup_with_events_fs(db_path, None).await
+}
+
+pub async fn node_test_setup_with_events_fs(
+    db_path: PathBuf,
+    events: Option<mpsc::Sender<Event>>,
+) -> TestResult<(Router, FsStore, PathBuf)> {
+    let store = crate::store::fs::FsStore::load(&db_path).await?;
+    let ep = Endpoint::builder().bind().await?;
+    let blobs = Blobs::new(&store, ep.clone(), events);
+    let router = Router::builder(ep).accept(crate::ALPN, blobs).spawn();
+    Ok((router, store, db_path))
+}
+
+pub async fn node_test_setup_mem() -> TestResult<(Router, MemStore)> {
+    node_test_setup_with_events_mem(None).await
+}
+
+pub async fn node_test_setup_with_events_mem(
+    events: Option<mpsc::Sender<Event>>,
+) -> TestResult<(Router, MemStore)> {
+    let store = MemStore::new();
+    let ep = Endpoint::builder().bind().await?;
+    let blobs = Blobs::new(&store, ep.clone(), events);
+    let router = Router::builder(ep).accept(crate::ALPN, blobs).spawn();
+    Ok((router, store))
+}
+
+/// Sets up two nodes with a router and a blob store each.
+///
+/// Note that this does not configure discovery, so nodes will only find each other
+/// with full node addresses, not just node ids!
+async fn two_node_test_setup_fs() -> TestResult<(
+    TempDir,
+    (Router, FsStore, PathBuf),
+    (Router, FsStore, PathBuf),
+)> {
+    let testdir = tempfile::tempdir().unwrap();
+    let db1_path = testdir.path().join("db1");
+    let db2_path = testdir.path().join("db2");
+    Ok((
+        testdir,
+        node_test_setup_fs(db1_path).await?,
+        node_test_setup_fs(db2_path).await?,
+    ))
+}
+
+/// Sets up two nodes with a router and a blob store each.
+///
+/// Note that this does not configure discovery, so nodes will only find each other
+/// with full node addresses, not just node ids!
+async fn two_node_test_setup_mem() -> TestResult<((Router, MemStore), (Router, MemStore))> {
+    Ok((node_test_setup_mem().await?, node_test_setup_mem().await?))
+}
+
+async fn two_nodes_hash_seq(
+    r1: Router,
+    store1: &Store,
+    r2: Router,
+    store2: &Store,
+) -> TestResult<()> {
+    let addr1 = r1.endpoint().node_addr().initialized().await?;
+    let sizes = INTERESTING_SIZES;
+    let root = add_test_hash_seq(store1, sizes).await?;
+    let conn = r2.endpoint().connect(addr1, crate::ALPN).await?;
+    store2.remote().fetch(conn, root).await?;
+    check_presence(store2, &sizes).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn two_nodes_hash_seq_fs() -> TestResult<()> {
+    tracing_subscriber::fmt::try_init().ok();
+    let (_testdir, (r1, store1, _), (r2, store2, _)) = two_node_test_setup_fs().await?;
+    two_nodes_hash_seq(r1, &store1, r2, &store2).await
+}
+
+#[tokio::test]
+async fn two_nodes_hash_seq_mem() -> TestResult<()> {
+    tracing_subscriber::fmt::try_init().ok();
+    let ((r1, store1), (r2, store2)) = two_node_test_setup_mem().await?;
+    two_nodes_hash_seq(r1, &store1, r2, &store2).await
+}
+
+#[tokio::test]
+async fn two_nodes_hash_seq_progress() -> TestResult<()> {
+    tracing_subscriber::fmt::try_init().ok();
+    let (_testdir, (r1, store1, _), (r2, store2, _)) = two_node_test_setup_fs().await?;
+    let addr1 = r1.endpoint().node_addr().initialized().await?;
+    let sizes = INTERESTING_SIZES;
+    let root = add_test_hash_seq(&store1, sizes).await?;
+    let conn = r2.endpoint().connect(addr1, crate::ALPN).await?;
+    let mut stream = store2.remote().fetch(conn, root).stream();
+    while let Some(item) = stream.next().await {
+        println!("{item:?}");
+    }
+    check_presence(&store2, &sizes).await?;
+    Ok(())
+}
+
+/// A node serves a hash sequence with all the interesting sizes.
+///
+/// The client requests the hash sequence and the children, but does not store the data.
+#[tokio::test]
+async fn node_serve_hash_seq() -> TestResult<()> {
+    tracing_subscriber::fmt::try_init().ok();
+    let testdir = tempfile::tempdir()?;
+    let db_path = testdir.path().join("db");
+    let store = crate::store::fs::FsStore::load(&db_path).await?;
+    let sizes = INTERESTING_SIZES;
+    let mut tts = Vec::new();
+    // add all the sizes
+    for size in sizes {
+        let tt = store.add_bytes(test_data(size)).await?;
+        tts.push(tt);
+    }
+    let hash_seq = tts.iter().map(|x| x.hash).collect::<HashSeq>();
+    let root_tt = store.add_bytes(hash_seq).await?;
+    let root = root_tt.hash;
+    let endpoint = Endpoint::builder().discovery_n0().bind().await?;
+    let blobs = crate::net_protocol::Blobs::new(&store, endpoint.clone(), None);
+    let r1 = Router::builder(endpoint)
+        .accept(crate::protocol::ALPN, blobs)
+        .spawn();
+    let addr1 = r1.endpoint().node_addr().initialized().await?;
+    info!("node addr: {addr1:?}");
+    let endpoint2 = Endpoint::builder().discovery_n0().bind().await?;
+    let conn = endpoint2.connect(addr1, crate::protocol::ALPN).await?;
+    let (hs, sizes) = get::request::get_hash_seq_and_sizes(&conn, &root, 1024, None).await?;
+    println!("hash seq: {hs:?}");
+    println!("sizes: {sizes:?}");
+    r1.shutdown().await?;
+    Ok(())
+}
+
+/// A node serves individual blobs with all the interesting sizes.
+///
+/// The client requests them all one by one, but does not store it.
+#[tokio::test]
+async fn node_serve_blobs() -> TestResult<()> {
+    tracing_subscriber::fmt::try_init().ok();
+    let testdir = tempfile::tempdir()?;
+    let db_path = testdir.path().join("db");
+    let store = crate::store::fs::FsStore::load(&db_path).await?;
+    let sizes = INTERESTING_SIZES;
+    // add all the sizes
+    let mut tts = Vec::new();
+    for size in sizes {
+        tts.push(store.add_bytes(test_data(size)).await?);
+    }
+    let endpoint = Endpoint::builder().discovery_n0().bind().await?;
+    let blobs = crate::net_protocol::Blobs::new(&store, endpoint.clone(), None);
+    let r1 = Router::builder(endpoint)
+        .accept(crate::protocol::ALPN, blobs)
+        .spawn();
+    let addr1 = r1.endpoint().node_addr().initialized().await?;
+    info!("node addr: {addr1:?}");
+    let endpoint2 = Endpoint::builder().discovery_n0().bind().await?;
+    let conn = endpoint2.connect(addr1, crate::protocol::ALPN).await?;
+    for size in sizes {
+        let expected = test_data(size);
+        let hash = Hash::new(&expected);
+        let mut stream = get::request::get_blob(conn.clone(), hash);
+        while let Some(item) = stream.next().await {
+            println!("{item:?}");
+        }
+        let actual = get::request::get_blob(conn.clone(), hash).await?;
+        assert_eq!(actual.len(), expected.len(), "size: {size}");
+    }
+    r1.shutdown().await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn node_smoke_fs() -> TestResult<()> {
+    tracing_subscriber::fmt::try_init().ok();
+    let testdir = tempfile::tempdir()?;
+    let db_path = testdir.path().join("db");
+    let store = crate::store::fs::FsStore::load(&db_path).await?;
+    node_smoke(&store).await
+}
+
+#[tokio::test]
+async fn node_smoke_mem() -> TestResult<()> {
+    tracing_subscriber::fmt::try_init().ok();
+    let store = crate::store::mem::MemStore::new();
+    node_smoke(&store).await
+}
+
+async fn node_smoke(store: &Store) -> TestResult<()> {
+    let tt = store.add_bytes(b"hello world".to_vec()).temp_tag().await?;
+    let hash = *tt.hash();
+    let endpoint = Endpoint::builder().discovery_n0().bind().await?;
+    let blobs = crate::net_protocol::Blobs::new(store, endpoint.clone(), None);
+    let r1 = Router::builder(endpoint)
+        .accept(crate::protocol::ALPN, blobs)
+        .spawn();
+    let addr1 = r1.endpoint().node_addr().initialized().await?;
+    info!("node addr: {addr1:?}");
+    let endpoint2 = Endpoint::builder().discovery_n0().bind().await?;
+    let conn = endpoint2.connect(addr1, crate::protocol::ALPN).await?;
+    let (size, stats) = get::request::get_unverified_size(&conn, &hash).await?;
+    info!("size: {} stats: {:?}", size, stats);
+    let data = get::request::get_blob(conn, hash).await?;
+    assert_eq!(data.as_ref(), b"hello world");
+    r1.shutdown().await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_export_chunk() -> TestResult {
+    tracing_subscriber::fmt::try_init().ok();
+    let testdir = tempfile::tempdir()?;
+    let db_path = testdir.path().join("db");
+    let store = crate::store::fs::FsStore::load(&db_path).await?;
+    let blobs = store.blobs();
+    for size in [1024 * 18 + 1] {
+        let data = vec![0u8; size];
+        let tt = store.add_slice(&data).temp_tag().await?;
+        let hash = *tt.hash();
+        let c = blobs.export_chunk(hash, 0).await;
+        println!("{c:?}");
+        let c = blobs.export_chunk(hash, 1000000).await;
+        println!("{c:?}");
+    }
+    Ok(())
+}
+
+async fn test_export_ranges(
+    store: &Store,
+    hash: Hash,
+    data: &[u8],
+    range: Range<u64>,
+) -> TestResult {
+    let actual = store
+        .export_ranges(hash, range.clone())
+        .concatenate()
+        .await?;
+    let start = (range.start as usize).min(data.len());
+    let end = (range.end as usize).min(data.len());
+    assert_eq!(&actual, &data[start..end]);
+    Ok(())
+}
+
+#[tokio::test]
+async fn export_ranges_smoke_fs() -> TestResult {
+    tracing_subscriber::fmt::try_init().ok();
+    let testdir = tempfile::tempdir()?;
+    let db_path = testdir.path().join("db");
+    let store = crate::store::fs::FsStore::load(&db_path).await?;
+    export_ranges_smoke(&store).await
+}
+
+#[tokio::test]
+async fn export_ranges_smoke_mem() -> TestResult {
+    tracing_subscriber::fmt::try_init().ok();
+    let store = MemStore::new();
+    export_ranges_smoke(&store).await
+}
+
+async fn export_ranges_smoke(store: &Store) -> TestResult {
+    let sizes = INTERESTING_SIZES;
+    for size in sizes {
+        let data = test_data(size);
+        let tt = store.add_bytes(data.clone()).await?;
+        let hash = tt.hash;
+        let size = size as u64;
+        test_export_ranges(store, hash, &data, 0..size).await?;
+        test_export_ranges(store, hash, &data, 0..(size / 2)).await?;
+        test_export_ranges(store, hash, &data, (size / 2)..size).await?;
+        test_export_ranges(store, hash, &data, (size / 2)..(size + size / 2)).await?;
+        test_export_ranges(store, hash, &data, size * 4..size * 5).await?;
+    }
+    Ok(())
+}
diff --git a/src/ticket.rs b/src/ticket.rs
index 69c512d14..6cbb5b24d 100644
--- a/src/ticket.rs
+++ b/src/ticket.rs
@@ -6,7 +6,7 @@ use iroh::{NodeAddr, NodeId, RelayUrl};
 use iroh_base::ticket::{self, Ticket};
 use serde::{Deserialize, Serialize};
 
-use crate::{BlobFormat, Hash};
+use crate::{BlobFormat, Hash, HashAndFormat};
 
 /// A token containing everything to get a file from the provider.
 ///
@@ -22,6 +22,15 @@ pub struct BlobTicket {
     hash: Hash,
 }
 
+impl From<BlobTicket> for HashAndFormat {
+    fn from(val: BlobTicket) -> Self {
+        HashAndFormat {
+            hash: val.hash,
+            format: val.format,
+        }
+    }
+}
+
 /// Wire format for [`BlobTicket`].
 ///
 /// In the future we might have multiple variants (not versions, since they
@@ -70,8 +79,8 @@ impl Ticket for BlobTicket {
         postcard::to_stdvec(&data).expect("postcard serialization failed")
     }
 
-    fn from_bytes(bytes: &[u8]) -> std::result::Result<Self, ticket::Error> {
-        let res: TicketWireFormat = postcard::from_bytes(bytes).map_err(ticket::Error::Postcard)?;
+    fn from_bytes(bytes: &[u8]) -> std::result::Result<Self, ticket::ParseError> {
+        let res: TicketWireFormat = postcard::from_bytes(bytes)?;
         let TicketWireFormat::Variant0(Variant0BlobTicket { node, format, hash }) = res;
         Ok(Self {
             node: NodeAddr {
@@ -86,7 +95,7 @@ impl Ticket for BlobTicket {
 }
 
 impl FromStr for BlobTicket {
-    type Err = ticket::Error;
+    type Err = ticket::ParseError;
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         Ticket::deserialize(s)
@@ -95,8 +104,8 @@ impl FromStr for BlobTicket {
 
 impl BlobTicket {
     /// Creates a new ticket.
-    pub fn new(node: NodeAddr, hash: Hash, format: BlobFormat) -> Result<Self> {
-        Ok(Self { hash, format, node })
+    pub fn new(node: NodeAddr, hash: Hash, format: BlobFormat) -> Self {
+        Self { hash, format, node }
     }
 
     /// The hash of the item this ticket can retrieve.
@@ -114,6 +123,13 @@ impl BlobTicket {
         self.format
     }
 
+    pub fn hash_and_format(&self) -> HashAndFormat {
+        HashAndFormat {
+            hash: self.hash,
+            format: self.format,
+        }
+    }
+
     /// True if the ticket is for a collection and should retrieve all blobs in it.
     pub fn recursive(&self) -> bool {
         self.format.is_hash_seq()
@@ -144,7 +160,7 @@ impl<'de> Deserialize<'de> for BlobTicket {
             Self::from_str(&s).map_err(serde::de::Error::custom)
         } else {
             let (peer, format, hash) = Deserialize::deserialize(deserializer)?;
-            Self::new(peer, hash, format).map_err(serde::de::Error::custom)
+            Ok(Self::new(peer, hash, format))
         }
     }
 }
@@ -154,9 +170,9 @@ mod tests {
     use std::net::SocketAddr;
 
     use iroh::{PublicKey, SecretKey};
+    use iroh_test::{assert_eq_hex, hexdump::parse_hexdump};
 
     use super::*;
-    use crate::{assert_eq_hex, util::hexdump::parse_hexdump};
 
     fn make_ticket() -> BlobTicket {
         let hash = Hash::new(b"hi there");
diff --git a/src/util.rs b/src/util.rs
index fcf3115bf..e110be083 100644
--- a/src/util.rs
+++ b/src/util.rs
@@ -1,418 +1,395 @@
-//! Utility functions and types.
-use std::{
-    borrow::Borrow,
-    fmt,
-    io::{BufReader, Read},
-    sync::{Arc, Weak},
-    time::SystemTime,
-};
-
-use bao_tree::{io::outboard::PreOrderOutboard, BaoTree, ChunkRanges};
-use bytes::Bytes;
-use derive_more::{Debug, Display, From, Into};
-use range_collections::range_set::RangeSetRange;
-use serde::{Deserialize, Serialize};
-
-use crate::{BlobFormat, Hash, HashAndFormat, IROH_BLOCK_SIZE};
-
-pub mod fs;
-pub mod io;
-mod mem_or_file;
-pub mod progress;
-pub use mem_or_file::MemOrFile;
-mod sparse_mem_file;
-pub use sparse_mem_file::SparseMemFile;
-pub mod local_pool;
-
-#[cfg(test)]
-pub(crate) mod hexdump;
-
-/// A tag
-#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, From, Into)]
-pub struct Tag(pub Bytes);
-
-#[cfg(feature = "redb")]
-mod redb_support {
-    use bytes::Bytes;
-    use redb::{Key as RedbKey, Value as RedbValue};
-
-    use super::Tag;
-
-    impl RedbValue for Tag {
-        type SelfType<'a> = Self;
-
-        type AsBytes<'a> = bytes::Bytes;
-
-        fn fixed_width() -> Option<usize> {
-            None
-        }
+use std::ops::{Bound, RangeBounds};
+
+use bao_tree::{io::round_up_to_chunks, ChunkNum, ChunkRanges};
+use range_collections::{range_set::RangeSetEntry, RangeSet2};
+
+pub mod channel;
+pub(crate) mod temp_tag;
+pub mod serde {
+    // Module that handles io::Error serialization/deserialization
+    pub mod io_error_serde {
+        use std::{fmt, io};
 
-        fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a>
+        use serde::{
+            de::{self, Visitor},
+            Deserializer, Serializer,
+        };
+
+        pub fn serialize<S>(error: &io::Error, serializer: S) -> Result<S::Ok, S::Error>
         where
-            Self: 'a,
+            S: Serializer,
         {
-            Self(Bytes::copy_from_slice(data))
+            // Serialize the error kind and message
+            serializer.serialize_str(&format!("{:?}:{}", error.kind(), error))
         }
 
-        fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a>
+        pub fn deserialize<'de, D>(deserializer: D) -> Result<io::Error, D::Error>
         where
-            Self: 'a,
-            Self: 'b,
+            D: Deserializer<'de>,
         {
-            value.0.clone()
-        }
+            struct IoErrorVisitor;
 
-        fn type_name() -> redb::TypeName {
-            redb::TypeName::new("Tag")
-        }
-    }
+            impl<'de> Visitor<'de> for IoErrorVisitor {
+                type Value = io::Error;
+
+                fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+                    formatter.write_str("an io::Error string representation")
+                }
+
+                fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+                where
+                    E: de::Error,
+                {
+                    // For simplicity, create a generic error
+                    // In a real app, you might want to parse the kind from the string
+                    Ok(io::Error::other(value))
+                }
+            }
 
-    impl RedbKey for Tag {
-        fn compare(data1: &[u8], data2: &[u8]) -> std::cmp::Ordering {
-            data1.cmp(data2)
+            deserializer.deserialize_str(IoErrorVisitor)
         }
     }
 }
 
-impl From<&[u8]> for Tag {
-    fn from(value: &[u8]) -> Self {
-        Self(Bytes::copy_from_slice(value))
-    }
+pub trait ChunkRangesExt {
+    fn last_chunk() -> Self;
+    fn chunk(offset: u64) -> Self;
+    fn bytes(ranges: impl RangeBounds<u64>) -> Self;
+    fn chunks(ranges: impl RangeBounds<u64>) -> Self;
+    fn offset(offset: u64) -> Self;
 }
 
-impl AsRef<[u8]> for Tag {
-    fn as_ref(&self) -> &[u8] {
-        self.0.as_ref()
+impl ChunkRangesExt for ChunkRanges {
+    fn last_chunk() -> Self {
+        ChunkRanges::from(ChunkNum(u64::MAX)..)
     }
-}
 
-impl Borrow<[u8]> for Tag {
-    fn borrow(&self) -> &[u8] {
-        self.0.as_ref()
+    /// Create a chunk range that contains a single chunk.
+    fn chunk(offset: u64) -> Self {
+        ChunkRanges::from(ChunkNum(offset)..ChunkNum(offset + 1))
     }
-}
 
-impl From<String> for Tag {
-    fn from(value: String) -> Self {
-        Self(Bytes::from(value))
+    /// Create a range of chunks that contains the given byte ranges.
+    /// The byte ranges are rounded up to the nearest chunk size.
+    fn bytes(ranges: impl RangeBounds<u64>) -> Self {
+        round_up_to_chunks(&bounds_from_range(ranges, |v| v))
     }
-}
 
-impl From<&str> for Tag {
-    fn from(value: &str) -> Self {
-        Self(Bytes::from(value.to_owned()))
+    /// Create a range of chunks from u64 chunk bounds.
+    ///
+    /// This is equivalent but more convenient than using the ChunkNum newtype.
+    fn chunks(ranges: impl RangeBounds<u64>) -> Self {
+        bounds_from_range(ranges, ChunkNum)
+    }
+
+    /// Create a chunk range that contains a single byte offset.
+    fn offset(offset: u64) -> Self {
+        Self::bytes(offset..offset + 1)
     }
 }
 
-impl Display for Tag {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let bytes = self.0.as_ref();
-        match std::str::from_utf8(bytes) {
-            Ok(s) => write!(f, "\"{}\"", s),
-            Err(_) => write!(f, "{}", hex::encode(bytes)),
+// todo: move to range_collections
+pub(crate) fn bounds_from_range<R, T, F>(range: R, f: F) -> RangeSet2<T>
+where
+    R: RangeBounds<u64>,
+    T: RangeSetEntry,
+    F: Fn(u64) -> T,
+{
+    let from = match range.start_bound() {
+        Bound::Included(start) => Some(*start),
+        Bound::Excluded(start) => {
+            let Some(start) = start.checked_add(1) else {
+                return RangeSet2::empty();
+            };
+            Some(start)
         }
+        Bound::Unbounded => None,
+    };
+    let to = match range.end_bound() {
+        Bound::Included(end) => end.checked_add(1),
+        Bound::Excluded(end) => Some(*end),
+        Bound::Unbounded => None,
+    };
+    match (from, to) {
+        (Some(from), Some(to)) => RangeSet2::from(f(from)..f(to)),
+        (Some(from), None) => RangeSet2::from(f(from)..),
+        (None, Some(to)) => RangeSet2::from(..f(to)),
+        (None, None) => RangeSet2::all(),
     }
 }
 
-struct DD<T: fmt::Display>(T);
+pub mod outboard_with_progress {
+    use std::io::{self, BufReader, Read};
+
+    use bao_tree::{
+        blake3,
+        io::{
+            outboard::PreOrderOutboard,
+            sync::{OutboardMut, WriteAt},
+        },
+        iter::BaoChunk,
+        BaoTree, ChunkNum,
+    };
+    use smallvec::SmallVec;
+
+    use super::sink::Sink;
+
+    fn hash_subtree(start_chunk: u64, data: &[u8], is_root: bool) -> blake3::Hash {
+        use blake3::hazmat::{ChainingValue, HasherExt};
+        if is_root {
+            debug_assert!(start_chunk == 0);
+            blake3::hash(data)
+        } else {
+            let mut hasher = blake3::Hasher::new();
+            hasher.set_input_offset(start_chunk * 1024);
+            hasher.update(data);
+            let non_root_hash: ChainingValue = hasher.finalize_non_root();
+            blake3::Hash::from(non_root_hash)
+        }
+    }
 
-impl<T: fmt::Display> fmt::Debug for DD<T> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        fmt::Display::fmt(&self.0, f)
+    fn parent_cv(
+        left_child: &blake3::Hash,
+        right_child: &blake3::Hash,
+        is_root: bool,
+    ) -> blake3::Hash {
+        use blake3::hazmat::{merge_subtrees_non_root, merge_subtrees_root, ChainingValue, Mode};
+        let left_child: ChainingValue = *left_child.as_bytes();
+        let right_child: ChainingValue = *right_child.as_bytes();
+        if is_root {
+            merge_subtrees_root(&left_child, &right_child, Mode::Hash)
+        } else {
+            blake3::Hash::from(merge_subtrees_non_root(
+                &left_child,
+                &right_child,
+                Mode::Hash,
+            ))
+        }
     }
-}
 
-impl Debug for Tag {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_tuple("Tag").field(&DD(self)).finish()
+    pub async fn init_outboard<R, W, P>(
+        data: R,
+        outboard: &mut PreOrderOutboard<W>,
+        progress: &mut P,
+    ) -> std::io::Result<std::result::Result<(), P::Error>>
+    where
+        W: WriteAt,
+        R: Read,
+        P: Sink<ChunkNum>,
+    {
+        // wrap the reader in a buffered reader, so we read in large chunks
+        // this reduces the number of io ops
+        let size = usize::try_from(outboard.tree.size()).unwrap_or(usize::MAX);
+        let read_buf_size = size.min(1024 * 1024);
+        let chunk_buf_size = size.min(outboard.tree.block_size().bytes());
+        let reader = BufReader::with_capacity(read_buf_size, data);
+        let mut buffer = SmallVec::<[u8; 128]>::from_elem(0u8, chunk_buf_size);
+        let res = init_impl(outboard.tree, reader, outboard, &mut buffer, progress).await?;
+        Ok(res)
     }
-}
 
-impl Tag {
-    /// Create a new tag that does not exist yet.
-    pub fn auto(time: SystemTime, exists: impl Fn(&[u8]) -> bool) -> Self {
-        let now = chrono::DateTime::<chrono::Utc>::from(time);
-        let mut i = 0;
-        loop {
-            let mut text = format!("auto-{}", now.format("%Y-%m-%dT%H:%M:%S%.3fZ"));
-            if i != 0 {
-                text.push_str(&format!("-{}", i));
-            }
-            if !exists(text.as_bytes()) {
-                return Self::from(text);
+    async fn init_impl<W, P>(
+        tree: BaoTree,
+        mut data: impl Read,
+        outboard: &mut PreOrderOutboard<W>,
+        buffer: &mut [u8],
+        progress: &mut P,
+    ) -> io::Result<std::result::Result<(), P::Error>>
+    where
+        W: WriteAt,
+        P: Sink<ChunkNum>,
+    {
+        // do not allocate for small trees
+        let mut stack = SmallVec::<[blake3::Hash; 10]>::new();
+        // debug_assert!(buffer.len() == tree.chunk_group_bytes());
+        for item in tree.post_order_chunks_iter() {
+            match item {
+                BaoChunk::Parent { is_root, node, .. } => {
+                    let right_hash = stack.pop().unwrap();
+                    let left_hash = stack.pop().unwrap();
+                    outboard.save(node, &(left_hash, right_hash))?;
+                    let parent = parent_cv(&left_hash, &right_hash, is_root);
+                    stack.push(parent);
+                }
+                BaoChunk::Leaf {
+                    size,
+                    is_root,
+                    start_chunk,
+                    ..
+                } => {
+                    if let Err(err) = progress.send(start_chunk).await {
+                        return Ok(Err(err));
+                    }
+                    let buf = &mut buffer[..size];
+                    data.read_exact(buf)?;
+                    let hash = hash_subtree(start_chunk.0, buf, is_root);
+                    stack.push(hash);
+                }
             }
-            i += 1;
         }
+        debug_assert_eq!(stack.len(), 1);
+        outboard.root = stack.pop().unwrap();
+        Ok(Ok(()))
     }
 
-    /// The successor of this tag in lexicographic order.
-    pub fn successor(&self) -> Self {
-        let mut bytes = self.0.to_vec();
-        // increment_vec(&mut bytes);
-        bytes.push(0);
-        Self(bytes.into())
-    }
-
-    /// If this is a prefix, get the next prefix.
-    ///
-    /// This is like successor, except that it will return None if the prefix is all 0xFF instead of appending a 0 byte.
-    pub fn next_prefix(&self) -> Option<Self> {
-        let mut bytes = self.0.to_vec();
-        if next_prefix(&mut bytes) {
-            Some(Self(bytes.into()))
-        } else {
-            None
+    #[cfg(test)]
+    mod tests {
+        use bao_tree::{
+            blake3,
+            io::{outboard::PreOrderOutboard, sync::CreateOutboard},
+            BaoTree,
+        };
+        use testresult::TestResult;
+
+        use crate::{
+            store::{fs::tests::test_data, IROH_BLOCK_SIZE},
+            util::{outboard_with_progress::init_outboard, sink::Drain},
+        };
+
+        #[tokio::test]
+        async fn init_outboard_with_progress() -> TestResult<()> {
+            for size in [1024 * 18 + 1] {
+                let data = test_data(size);
+                let mut o1 = PreOrderOutboard::<Vec<u8>> {
+                    tree: BaoTree::new(data.len() as u64, IROH_BLOCK_SIZE),
+                    ..Default::default()
+                };
+                let mut o2 = o1.clone();
+                o1.init_from(data.as_ref())?;
+                init_outboard(data.as_ref(), &mut o2, &mut Drain).await??;
+                assert_eq!(o1.root, blake3::hash(&data));
+                assert_eq!(o1.root, o2.root);
+                assert_eq!(o1.data, o2.data);
+            }
+            Ok(())
         }
     }
 }
 
-/// Option for commands that allow setting a tag
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
-pub enum SetTagOption {
-    /// A tag will be automatically generated
-    Auto,
-    /// The tag is explicitly named
-    Named(Tag),
-}
+pub mod sink {
+    use std::{future::Future, io};
 
-/// Trait used from temp tags to notify an abstract store that a temp tag is
-/// being dropped.
-pub trait TagDrop: std::fmt::Debug + Send + Sync + 'static {
-    /// Called on drop
-    fn on_drop(&self, inner: &HashAndFormat);
-}
+    use irpc::RpcMessage;
 
-/// A trait for things that can track liveness of blobs and collections.
-///
-/// This trait works together with [TempTag] to keep track of the liveness of a
-/// blob or collection.
-///
-/// It is important to include the format in the liveness tracking, since
-/// protecting a collection means protecting the blob and all its children,
-/// whereas protecting a raw blob only protects the blob itself.
-pub trait TagCounter: TagDrop + Sized {
-    /// Called on creation of a temp tag
-    fn on_create(&self, inner: &HashAndFormat);
-
-    /// Get this as a weak reference for use in temp tags
-    fn as_weak(self: &Arc<Self>) -> Weak<dyn TagDrop> {
-        let on_drop: Arc<dyn TagDrop> = self.clone();
-        Arc::downgrade(&on_drop)
-    }
+    /// Our version of a sink, that can be mapped etc.
+    pub trait Sink<Item> {
+        type Error;
+        fn send(
+            &mut self,
+            value: Item,
+        ) -> impl Future<Output = std::result::Result<(), Self::Error>>;
+
+        fn with_map_err<F, U>(self, f: F) -> WithMapErr<Self, F>
+        where
+            Self: Sized,
+            F: Fn(Self::Error) -> U + Send + 'static,
+        {
+            WithMapErr { inner: self, f }
+        }
 
-    /// Create a new temp tag for the given hash and format
-    fn temp_tag(self: &Arc<Self>, inner: HashAndFormat) -> TempTag {
-        self.on_create(&inner);
-        TempTag::new(inner, Some(self.as_weak()))
+        fn with_map<F, U>(self, f: F) -> WithMap<Self, F>
+        where
+            Self: Sized,
+            F: Fn(U) -> Item + Send + 'static,
+        {
+            WithMap { inner: self, f }
+        }
     }
-}
 
-/// A hash and format pair that is protected from garbage collection.
-///
-/// If format is raw, this will protect just the blob
-/// If format is collection, this will protect the collection and all blobs in it
-#[derive(Debug)]
-pub struct TempTag {
-    /// The hash and format we are pinning
-    inner: HashAndFormat,
-    /// optional callback to call on drop
-    on_drop: Option<Weak<dyn TagDrop>>,
-}
+    impl<I, T> Sink<T> for &mut I
+    where
+        I: Sink<T>,
+    {
+        type Error = I::Error;
 
-impl TempTag {
-    /// Create a new temp tag for the given hash and format
-    ///
-    /// This should only be used by store implementations.
-    ///
-    /// The caller is responsible for increasing the refcount on creation and to
-    /// make sure that temp tags that are created between a mark phase and a sweep
-    /// phase are protected.
-    pub fn new(inner: HashAndFormat, on_drop: Option<Weak<dyn TagDrop>>) -> Self {
-        Self { inner, on_drop }
+        async fn send(&mut self, value: T) -> std::result::Result<(), Self::Error> {
+            (*self).send(value).await
+        }
     }
 
-    /// The hash of the pinned item
-    pub fn inner(&self) -> &HashAndFormat {
-        &self.inner
-    }
+    pub struct IrpcSenderSink<T>(pub irpc::channel::mpsc::Sender<T>);
 
-    /// The hash of the pinned item
-    pub fn hash(&self) -> &Hash {
-        &self.inner.hash
-    }
+    impl<T> Sink<T> for IrpcSenderSink<T>
+    where
+        T: RpcMessage,
+    {
+        type Error = irpc::channel::SendError;
 
-    /// The format of the pinned item
-    pub fn format(&self) -> BlobFormat {
-        self.inner.format
+        async fn send(&mut self, value: T) -> std::result::Result<(), Self::Error> {
+            self.0.send(value).await
+        }
     }
 
-    /// The hash and format of the pinned item
-    pub fn hash_and_format(&self) -> HashAndFormat {
-        self.inner
-    }
+    pub struct IrpcSenderRefSink<'a, T>(pub &'a mut irpc::channel::mpsc::Sender<T>);
 
-    /// Keep the item alive until the end of the process
-    pub fn leak(mut self) {
-        // set the liveness tracker to None, so that the refcount is not decreased
-        // during drop. This means that the refcount will never reach 0 and the
-        // item will not be gced until the end of the process.
-        self.on_drop = None;
-    }
-}
+    impl<'a, T> Sink<T> for IrpcSenderRefSink<'a, T>
+    where
+        T: RpcMessage,
+    {
+        type Error = irpc::channel::SendError;
 
-impl Drop for TempTag {
-    fn drop(&mut self) {
-        if let Some(on_drop) = self.on_drop.take() {
-            if let Some(on_drop) = on_drop.upgrade() {
-                on_drop.on_drop(&self.inner);
-            }
+        async fn send(&mut self, value: T) -> std::result::Result<(), Self::Error> {
+            self.0.send(value).await
         }
     }
-}
 
-/// Get the number of bytes given a set of chunk ranges and the total size.
-///
-/// If some ranges are out of bounds, they will be clamped to the size.
-pub fn total_bytes(ranges: ChunkRanges, size: u64) -> u64 {
-    ranges
-        .iter()
-        .map(|range| {
-            let (start, end) = match range {
-                RangeSetRange::Range(r) => {
-                    (r.start.to_bytes().min(size), r.end.to_bytes().min(size))
-                }
-                RangeSetRange::RangeFrom(range) => (range.start.to_bytes().min(size), size),
-            };
-            end.saturating_sub(start)
-        })
-        .reduce(u64::saturating_add)
-        .unwrap_or_default()
-}
+    pub struct TokioMpscSenderSink<T>(pub tokio::sync::mpsc::Sender<T>);
 
-/// A non-sendable marker type
-#[derive(Debug)]
-pub(crate) struct NonSend {
-    _marker: std::marker::PhantomData<std::rc::Rc<()>>,
-}
+    impl<T> Sink<T> for TokioMpscSenderSink<T> {
+        type Error = tokio::sync::mpsc::error::SendError<T>;
 
-impl NonSend {
-    /// Create a new non-sendable marker.
-    #[allow(dead_code)]
-    pub const fn new() -> Self {
-        Self {
-            _marker: std::marker::PhantomData,
+        async fn send(&mut self, value: T) -> std::result::Result<(), Self::Error> {
+            self.0.send(value).await
         }
     }
-}
 
-/// copy a limited slice from a slice as a `Bytes`.
-pub(crate) fn copy_limited_slice(bytes: &[u8], offset: u64, len: usize) -> Bytes {
-    bytes[limited_range(offset, len, bytes.len())]
-        .to_vec()
-        .into()
-}
+    pub struct WithMapErr<P, F> {
+        inner: P,
+        f: F,
+    }
 
-pub(crate) fn limited_range(offset: u64, len: usize, buf_len: usize) -> std::ops::Range<usize> {
-    if offset < buf_len as u64 {
-        let start = offset as usize;
-        let end = start.saturating_add(len).min(buf_len);
-        start..end
-    } else {
-        0..0
+    impl<P, F, E, U> Sink<U> for WithMapErr<P, F>
+    where
+        P: Sink<U>,
+        F: Fn(P::Error) -> E + Send + 'static,
+    {
+        type Error = E;
+
+        async fn send(&mut self, value: U) -> std::result::Result<(), Self::Error> {
+            match self.inner.send(value).await {
+                Ok(()) => Ok(()),
+                Err(err) => {
+                    let err = (self.f)(err);
+                    Err(err)
+                }
+            }
+        }
     }
-}
 
-/// zero copy get a limited slice from a `Bytes` as a `Bytes`.
-#[allow(dead_code)]
-pub(crate) fn get_limited_slice(bytes: &Bytes, offset: u64, len: usize) -> Bytes {
-    bytes.slice(limited_range(offset, len, bytes.len()))
-}
+    pub struct WithMap<P, F> {
+        inner: P,
+        f: F,
+    }
 
-/// Compute raw outboard size, without the size header.
-#[allow(dead_code)]
-pub(crate) fn raw_outboard_size(size: u64) -> u64 {
-    BaoTree::new(size, IROH_BLOCK_SIZE).outboard_size()
-}
+    impl<P, F, T, U> Sink<T> for WithMap<P, F>
+    where
+        P: Sink<U>,
+        F: Fn(T) -> U + Send + 'static,
+    {
+        type Error = P::Error;
 
-/// Given a prefix, increment it lexographically.
-///
-/// If the prefix is all FF, this will return false because there is no
-/// higher prefix than that.
-#[allow(dead_code)]
-pub(crate) fn next_prefix(bytes: &mut [u8]) -> bool {
-    for byte in bytes.iter_mut().rev() {
-        if *byte < 255 {
-            *byte += 1;
-            return true;
+        async fn send(&mut self, value: T) -> std::result::Result<(), Self::Error> {
+            self.inner.send((self.f)(value)).await
         }
-        *byte = 0;
     }
-    false
-}
-
-/// Synchronously compute the outboard of a file, and return hash and outboard.
-///
-/// It is assumed that the file is not modified while this is running.
-///
-/// If it is modified while or after this is running, the outboard will be
-/// invalid, so any attempt to compute a slice from it will fail.
-///
-/// If the size of the file is changed while this is running, an error will be
-/// returned.
-///
-/// The computed outboard is without length prefix.
-pub(crate) fn compute_outboard(
-    read: impl Read,
-    size: u64,
-    progress: impl Fn(u64) -> std::io::Result<()> + Send + Sync + 'static,
-) -> std::io::Result<(Hash, Option<Vec<u8>>)> {
-    use bao_tree::io::sync::CreateOutboard;
-
-    // wrap the reader in a progress reader, so we can report progress.
-    let reader = ProgressReader::new(read, progress);
-    // wrap the reader in a buffered reader, so we read in large chunks
-    // this reduces the number of io ops and also the number of progress reports
-    let buf_size = usize::try_from(size).unwrap_or(usize::MAX).min(1024 * 1024);
-    let reader = BufReader::with_capacity(buf_size, reader);
-
-    let ob = PreOrderOutboard::<Vec<u8>>::create_sized(reader, size, IROH_BLOCK_SIZE)?;
-    let root = ob.root.into();
-    let data = ob.data;
-    tracing::trace!(%root, "done");
-    let data = if !data.is_empty() { Some(data) } else { None };
-    Ok((root, data))
-}
 
-/// Compute raw outboard, without the size header.
-#[cfg(test)]
-#[allow(dead_code)]
-pub(crate) fn raw_outboard(data: &[u8]) -> (Vec<u8>, Hash) {
-    let res = bao_tree::io::outboard::PreOrderMemOutboard::create(data, IROH_BLOCK_SIZE);
-    (res.data, res.root.into())
-}
+    pub struct Drain;
 
-/// A reader that calls a callback with the number of bytes read after each read.
-pub(crate) struct ProgressReader<R, F: Fn(u64) -> std::io::Result<()>> {
-    inner: R,
-    offset: u64,
-    cb: F,
-}
+    impl<T> Sink<T> for Drain {
+        type Error = io::Error;
 
-impl<R: std::io::Read, F: Fn(u64) -> std::io::Result<()>> ProgressReader<R, F> {
-    pub fn new(inner: R, cb: F) -> Self {
-        Self {
-            inner,
-            offset: 0,
-            cb,
+        async fn send(&mut self, _offset: T) -> std::result::Result<(), Self::Error> {
+            io::Result::Ok(())
         }
     }
 }
-
-impl<R: std::io::Read, F: Fn(u64) -> std::io::Result<()>> std::io::Read for ProgressReader<R, F> {
-    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
-        let read = self.inner.read(buf)?;
-        self.offset += read as u64;
-        (self.cb)(self.offset)?;
-        Ok(read)
-    }
-}
diff --git a/src/util/channel.rs b/src/util/channel.rs
new file mode 100644
index 000000000..248b0fb4f
--- /dev/null
+++ b/src/util/channel.rs
@@ -0,0 +1,54 @@
+pub mod oneshot {
+    use std::{
+        future::Future,
+        pin::Pin,
+        task::{Context, Poll},
+    };
+
+    pub fn channel<T>() -> (Sender<T>, Receiver<T>) {
+        let (tx, rx) = tokio::sync::oneshot::channel::<T>();
+        (Sender::Tokio(tx), Receiver::Tokio(rx))
+    }
+
+    #[derive(Debug)]
+    pub enum Sender<T> {
+        Tokio(tokio::sync::oneshot::Sender<T>),
+    }
+
+    impl<T> From<Sender<T>> for irpc::channel::oneshot::Sender<T> {
+        fn from(sender: Sender<T>) -> Self {
+            match sender {
+                Sender::Tokio(tx) => tx.into(),
+            }
+        }
+    }
+
+    impl<T> Sender<T> {
+        pub fn send(self, value: T) {
+            match self {
+                Self::Tokio(tx) => tx.send(value).ok(),
+            };
+        }
+    }
+
+    pub enum Receiver<T> {
+        Tokio(tokio::sync::oneshot::Receiver<T>),
+    }
+
+    impl<T> Future for Receiver<T> {
+        type Output = std::result::Result<T, tokio::sync::oneshot::error::RecvError>;
+
+        fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+            match self.as_mut().get_mut() {
+                Self::Tokio(rx) => {
+                    if rx.is_terminated() {
+                        // don't panic when polling a terminated receiver
+                        Poll::Pending
+                    } else {
+                        Future::poll(Pin::new(rx), cx)
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/src/util/fs.rs b/src/util/fs.rs
deleted file mode 100644
index dc868859d..000000000
--- a/src/util/fs.rs
+++ /dev/null
@@ -1,453 +0,0 @@
-//! Utilities for filesystem operations.
-use std::{
-    borrow::Cow,
-    fs::read_dir,
-    path::{Component, Path, PathBuf},
-};
-
-use anyhow::{bail, Context};
-use bytes::Bytes;
-/// A data source
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
-pub struct DataSource {
-    /// Custom name
-    name: String,
-    /// Path to the file
-    path: PathBuf,
-}
-
-impl DataSource {
-    /// Creates a new [`DataSource`] from a [`PathBuf`].
-    pub fn new(path: PathBuf) -> Self {
-        let name = path
-            .file_name()
-            .map(|s| s.to_string_lossy().to_string())
-            .unwrap_or_default();
-        DataSource { path, name }
-    }
-    /// Creates a new [`DataSource`] from a [`PathBuf`] and a custom name.
-    pub fn with_name(path: PathBuf, name: String) -> Self {
-        DataSource { path, name }
-    }
-
-    /// Returns blob name for this data source.
-    ///
-    /// If no name was provided when created it is derived from the path name.
-    pub fn name(&self) -> Cow<'_, str> {
-        Cow::Borrowed(&self.name)
-    }
-
-    /// Returns the path of this data source.
-    pub fn path(&self) -> &Path {
-        &self.path
-    }
-}
-
-impl From<PathBuf> for DataSource {
-    fn from(value: PathBuf) -> Self {
-        DataSource::new(value)
-    }
-}
-
-impl From<&std::path::Path> for DataSource {
-    fn from(value: &std::path::Path) -> Self {
-        DataSource::new(value.to_path_buf())
-    }
-}
-
-/// Create data sources from a path.
-#[cfg(feature = "rpc")]
-pub fn scan_path(
-    path: PathBuf,
-    wrap: crate::rpc::client::blobs::WrapOption,
-) -> anyhow::Result<Vec<DataSource>> {
-    use crate::rpc::client::blobs::WrapOption;
-    if path.is_dir() {
-        scan_dir(path, wrap)
-    } else {
-        let name = match wrap {
-            WrapOption::NoWrap => bail!("Cannot scan a file without wrapping"),
-            WrapOption::Wrap { name: None } => file_name(&path)?,
-            WrapOption::Wrap { name: Some(name) } => name,
-        };
-        Ok(vec![DataSource { name, path }])
-    }
-}
-
-#[cfg(feature = "rpc")]
-fn file_name(path: &Path) -> anyhow::Result<String> {
-    relative_canonicalized_path_to_string(path.file_name().context("path is invalid")?)
-}
-
-/// Create data sources from a directory.
-#[cfg(feature = "rpc")]
-pub fn scan_dir(
-    root: PathBuf,
-    wrap: crate::rpc::client::blobs::WrapOption,
-) -> anyhow::Result<Vec<DataSource>> {
-    use crate::rpc::client::blobs::WrapOption;
-    if !root.is_dir() {
-        bail!("Expected {} to be a file", root.to_string_lossy());
-    }
-    let prefix = match wrap {
-        WrapOption::NoWrap => None,
-        WrapOption::Wrap { name: None } => Some(file_name(&root)?),
-        WrapOption::Wrap { name: Some(name) } => Some(name),
-    };
-    let files = walkdir::WalkDir::new(&root).into_iter();
-    let data_sources = files
-        .map(|entry| {
-            let entry = entry?;
-            if !entry.file_type().is_file() {
-                // Skip symlinks. Directories are handled by WalkDir.
-                return Ok(None);
-            }
-            let path = entry.into_path();
-            let mut name = relative_canonicalized_path_to_string(path.strip_prefix(&root)?)?;
-            if let Some(prefix) = &prefix {
-                name = format!("{prefix}/{name}");
-            }
-            anyhow::Ok(Some(DataSource { name, path }))
-        })
-        .filter_map(Result::transpose);
-    let data_sources: Vec<anyhow::Result<DataSource>> = data_sources.collect::<Vec<_>>();
-    data_sources.into_iter().collect::<anyhow::Result<Vec<_>>>()
-}
-
-/// This function converts a canonicalized relative path to a string, returning
-/// an error if the path is not valid unicode.
-///
-/// This function will also fail if the path is non canonical, i.e. contains
-/// `..` or `.`, or if the path components contain any windows or unix path
-/// separators.
-pub fn relative_canonicalized_path_to_string(path: impl AsRef<Path>) -> anyhow::Result<String> {
-    canonicalized_path_to_string(path, true)
-}
-
-/// Loads a [`iroh::SecretKey`] from the provided file, or stores a newly generated one
-/// at the given location.
-#[cfg(feature = "rpc")]
-pub async fn load_secret_key(key_path: PathBuf) -> anyhow::Result<iroh::SecretKey> {
-    use iroh::SecretKey;
-    use tokio::io::AsyncWriteExt;
-
-    if key_path.exists() {
-        let keystr = tokio::fs::read(key_path).await?;
-
-        let ser_key = ssh_key::private::PrivateKey::from_openssh(keystr)?;
-        let ssh_key::private::KeypairData::Ed25519(kp) = ser_key.key_data() else {
-            bail!("invalid key format");
-        };
-        let secret_key = SecretKey::from_bytes(&kp.private.to_bytes());
-        Ok(secret_key)
-    } else {
-        let secret_key = SecretKey::generate(rand::rngs::OsRng);
-        let ckey = ssh_key::private::Ed25519Keypair {
-            public: secret_key.public().public().into(),
-            private: secret_key.secret().into(),
-        };
-        let ser_key =
-            ssh_key::private::PrivateKey::from(ckey).to_openssh(ssh_key::LineEnding::default())?;
-
-        // Try to canonicalize if possible
-        let key_path = key_path.canonicalize().unwrap_or(key_path);
-        let key_path_parent = key_path.parent().ok_or_else(|| {
-            anyhow::anyhow!("no parent directory found for '{}'", key_path.display())
-        })?;
-        tokio::fs::create_dir_all(&key_path_parent).await?;
-
-        // write to tempfile
-        let (file, temp_file_path) = tempfile::NamedTempFile::new_in(key_path_parent)
-            .context("unable to create tempfile")?
-            .into_parts();
-        let mut file = tokio::fs::File::from_std(file);
-        file.write_all(ser_key.as_bytes())
-            .await
-            .context("unable to write keyfile")?;
-        file.flush().await?;
-        drop(file);
-
-        // move file
-        tokio::fs::rename(temp_file_path, key_path)
-            .await
-            .context("failed to rename keyfile")?;
-
-        Ok(secret_key)
-    }
-}
-
-/// Information about the content on a path
-#[derive(Debug, Clone)]
-pub struct PathContent {
-    /// total size of all the files in the directory
-    pub size: u64,
-    /// total number of files in the directory
-    pub files: u64,
-}
-
-/// Walks the directory to get the total size and number of files in directory or file
-// TODO: possible combine with `scan_dir`
-pub fn path_content_info(path: impl AsRef<Path>) -> anyhow::Result<PathContent> {
-    path_content_info0(path)
-}
-
-fn path_content_info0(path: impl AsRef<Path>) -> anyhow::Result<PathContent> {
-    let mut files = 0;
-    let mut size = 0;
-    let path = path.as_ref();
-
-    if path.is_dir() {
-        for entry in read_dir(path)? {
-            let path0 = entry?.path();
-
-            match path_content_info0(path0) {
-                Ok(path_content) => {
-                    size += path_content.size;
-                    files += path_content.files;
-                }
-                Err(e) => bail!(e),
-            }
-        }
-    } else {
-        match path.try_exists() {
-            Ok(true) => {
-                size = path
-                    .metadata()
-                    .context(format!("Error reading metadata for {path:?}"))?
-                    .len();
-                files = 1;
-            }
-            Ok(false) => {
-                tracing::warn!("Not including broking symlink at {path:?}");
-            }
-            Err(e) => {
-                bail!(e);
-            }
-        }
-    }
-    Ok(PathContent { size, files })
-}
-
-/// Helper function that translates a key that was derived from the [`path_to_key`] function back
-/// into a path.
-///
-/// If `prefix` exists, it will be stripped before converting back to a path
-/// If `root` exists, will add the root as a parent to the created path
-/// Removes any null byte that has been appended to the key
-pub fn key_to_path(
-    key: impl AsRef<[u8]>,
-    prefix: Option<String>,
-    root: Option<PathBuf>,
-) -> anyhow::Result<PathBuf> {
-    let mut key = key.as_ref();
-    if key.is_empty() {
-        return Ok(PathBuf::new());
-    }
-    // if the last element is the null byte, remove it
-    if b'\0' == key[key.len() - 1] {
-        key = &key[..key.len() - 1]
-    }
-
-    let key = if let Some(prefix) = prefix {
-        let prefix = prefix.into_bytes();
-        if prefix[..] == key[..prefix.len()] {
-            &key[prefix.len()..]
-        } else {
-            anyhow::bail!("key {:?} does not begin with prefix {:?}", key, prefix);
-        }
-    } else {
-        key
-    };
-
-    let mut path = if key[0] == b'/' {
-        PathBuf::from("/")
-    } else {
-        PathBuf::new()
-    };
-    for component in key
-        .split(|c| c == &b'/')
-        .map(|c| String::from_utf8(c.into()).context("key contains invalid data"))
-    {
-        let component = component?;
-        path = path.join(component);
-    }
-
-    // add root if it exists
-    let path = if let Some(root) = root {
-        root.join(path)
-    } else {
-        path
-    };
-
-    Ok(path)
-}
-
-/// Helper function that creates a document key from a canonicalized path, removing the `root` and adding the `prefix`, if they exist
-///
-/// Appends the null byte to the end of the key.
-pub fn path_to_key(
-    path: impl AsRef<Path>,
-    prefix: Option<String>,
-    root: Option<PathBuf>,
-) -> anyhow::Result<Bytes> {
-    let path = path.as_ref();
-    let path = if let Some(root) = root {
-        path.strip_prefix(root)?
-    } else {
-        path
-    };
-    let suffix = canonicalized_path_to_string(path, false)?.into_bytes();
-    let mut key = if let Some(prefix) = prefix {
-        prefix.into_bytes().to_vec()
-    } else {
-        Vec::new()
-    };
-    key.extend(suffix);
-    key.push(b'\0');
-    Ok(key.into())
-}
-
-/// This function converts an already canonicalized path to a string.
-///
-/// If `must_be_relative` is true, the function will fail if any component of the path is
-/// `Component::RootDir`
-///
-/// This function will also fail if the path is non canonical, i.e. contains
-/// `..` or `.`, or if the path components contain any windows or unix path
-/// separators.
-pub fn canonicalized_path_to_string(
-    path: impl AsRef<Path>,
-    must_be_relative: bool,
-) -> anyhow::Result<String> {
-    let mut path_str = String::new();
-    let parts = path
-        .as_ref()
-        .components()
-        .filter_map(|c| match c {
-            Component::Normal(x) => {
-                let c = match x.to_str() {
-                    Some(c) => c,
-                    None => return Some(Err(anyhow::anyhow!("invalid character in path"))),
-                };
-
-                if !c.contains('/') && !c.contains('\\') {
-                    Some(Ok(c))
-                } else {
-                    Some(Err(anyhow::anyhow!("invalid path component {:?}", c)))
-                }
-            }
-            Component::RootDir => {
-                if must_be_relative {
-                    Some(Err(anyhow::anyhow!("invalid path component {:?}", c)))
-                } else {
-                    path_str.push('/');
-                    None
-                }
-            }
-            _ => Some(Err(anyhow::anyhow!("invalid path component {:?}", c))),
-        })
-        .collect::<anyhow::Result<Vec<_>>>()?;
-    let parts = parts.join("/");
-    path_str.push_str(&parts);
-    Ok(path_str)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_path_to_key_roundtrip() {
-        let path = PathBuf::from("/foo/bar");
-        let expect_path = PathBuf::from("/foo/bar");
-        let key = b"/foo/bar\0";
-        let expect_key = Bytes::from(&key[..]);
-
-        let got_key = path_to_key(path.clone(), None, None).unwrap();
-        let got_path = key_to_path(got_key.clone(), None, None).unwrap();
-
-        assert_eq!(expect_key, got_key);
-        assert_eq!(expect_path, got_path);
-
-        // including prefix
-        let prefix = String::from("prefix:");
-        let key = b"prefix:/foo/bar\0";
-        let expect_key = Bytes::from(&key[..]);
-        let got_key = path_to_key(path.clone(), Some(prefix.clone()), None).unwrap();
-        assert_eq!(expect_key, got_key);
-        let got_path = key_to_path(got_key, Some(prefix.clone()), None).unwrap();
-        assert_eq!(expect_path, got_path);
-
-        // including root
-        let root = PathBuf::from("/foo");
-        let key = b"prefix:bar\0";
-        let expect_key = Bytes::from(&key[..]);
-        let got_key = path_to_key(path, Some(prefix.clone()), Some(root.clone())).unwrap();
-        assert_eq!(expect_key, got_key);
-        let got_path = key_to_path(got_key, Some(prefix), Some(root)).unwrap();
-        assert_eq!(expect_path, got_path);
-    }
-
-    #[test]
-    fn test_canonicalized_path_to_string() {
-        assert_eq!(
-            canonicalized_path_to_string("foo/bar", true).unwrap(),
-            "foo/bar"
-        );
-        assert_eq!(canonicalized_path_to_string("", true).unwrap(), "");
-        assert_eq!(
-            canonicalized_path_to_string("foo bar/baz/bat", true).unwrap(),
-            "foo bar/baz/bat"
-        );
-        assert_eq!(
-            canonicalized_path_to_string("/foo/bar", true).map_err(|e| e.to_string()),
-            Err("invalid path component RootDir".to_string())
-        );
-
-        assert_eq!(
-            canonicalized_path_to_string("/foo/bar", false).unwrap(),
-            "/foo/bar"
-        );
-        let path = PathBuf::from("/").join("Ü").join("⁰€™■･�").join("東京");
-        assert_eq!(
-            canonicalized_path_to_string(path, false).unwrap(),
-            "/Ü/⁰€™■･�/東京"
-        )
-    }
-
-    #[test]
-    fn test_get_path_content() {
-        let dir = testdir::testdir!();
-        let PathContent { size, files } = path_content_info(&dir).unwrap();
-        assert_eq!(0, size);
-        assert_eq!(0, files);
-        let foo = b"hello_world";
-        let bar = b"ipsum lorem";
-        let bat = b"happy birthday";
-        let expect_size = foo.len() + bar.len() + bat.len();
-        std::fs::write(dir.join("foo.txt"), foo).unwrap();
-        std::fs::write(dir.join("bar.txt"), bar).unwrap();
-        std::fs::write(dir.join("bat.txt"), bat).unwrap();
-        let PathContent { size, files } = path_content_info(&dir).unwrap();
-        assert_eq!(expect_size as u64, size);
-        assert_eq!(3, files);
-
-        // create nested empty dirs
-        std::fs::create_dir(dir.join("1")).unwrap();
-        std::fs::create_dir(dir.join("2")).unwrap();
-        let dir3 = dir.join("3");
-        std::fs::create_dir(&dir3).unwrap();
-
-        // create a nested dir w/ content
-        let dir4 = dir3.join("4");
-        std::fs::create_dir(&dir4).unwrap();
-        std::fs::write(dir4.join("foo.txt"), foo).unwrap();
-        std::fs::write(dir4.join("bar.txt"), bar).unwrap();
-        std::fs::write(dir4.join("bat.txt"), bat).unwrap();
-
-        let expect_size = expect_size * 2;
-        let PathContent { size, files } = path_content_info(&dir).unwrap();
-        assert_eq!(expect_size as u64, size);
-        assert_eq!(6, files);
-    }
-}
diff --git a/src/util/hexdump.rs b/src/util/hexdump.rs
deleted file mode 100644
index 0c36b1bc8..000000000
--- a/src/util/hexdump.rs
+++ /dev/null
@@ -1,181 +0,0 @@
-use anyhow::{ensure, Context, Result};
-
-/// Parses a commented multi line hexdump into a vector of bytes.
-///
-/// This is useful to write wire level protocol tests.
-pub fn parse_hexdump(s: &str) -> Result<Vec<u8>> {
-    let mut result = Vec::new();
-
-    for (line_number, line) in s.lines().enumerate() {
-        let data_part = line.split('#').next().unwrap_or("");
-        let cleaned: String = data_part.chars().filter(|c| !c.is_whitespace()).collect();
-
-        ensure!(
-            cleaned.len() % 2 == 0,
-            "Non-even number of hex chars detected on line {}.",
-            line_number + 1
-        );
-
-        for i in (0..cleaned.len()).step_by(2) {
-            let byte_str = &cleaned[i..i + 2];
-            let byte = u8::from_str_radix(byte_str, 16)
-                .with_context(|| format!("Invalid hex data on line {}.", line_number + 1))?;
-
-            result.push(byte);
-        }
-    }
-
-    Ok(result)
-}
-
-/// Returns a hexdump of the given bytes in multiple lines as a String.
-pub fn print_hexdump(bytes: impl AsRef<[u8]>, line_lengths: impl AsRef<[usize]>) -> String {
-    let line_lengths = line_lengths.as_ref();
-    let mut bytes_iter = bytes.as_ref().iter();
-    let default_line_length = line_lengths
-        .last()
-        .filter(|x| **x != 0)
-        .copied()
-        .unwrap_or(16);
-    let mut line_lengths_iter = line_lengths.iter();
-    let mut output = String::new();
-
-    loop {
-        let line_length = line_lengths_iter
-            .next()
-            .copied()
-            .unwrap_or(default_line_length);
-        if line_length == 0 {
-            output.push('\n');
-        } else {
-            let line: Vec<_> = bytes_iter.by_ref().take(line_length).collect();
-
-            if line.is_empty() {
-                break;
-            }
-
-            for byte in &line {
-                output.push_str(&format!("{:02x} ", byte));
-            }
-            output.pop(); // Remove the trailing space
-            output.push('\n');
-        }
-    }
-
-    output
-}
-
-/// This is a macro to assert that two byte slices are equal.
-///
-/// It is like assert_eq!, but it will print a nicely formatted hexdump of the
-/// two slices if they are not equal. This makes it much easier to track down
-/// a difference in a large byte slice.
-#[macro_export]
-macro_rules! assert_eq_hex {
-    ($a:expr, $b:expr) => {
-        assert_eq_hex!($a, $b, [])
-    };
-    ($a:expr, $b:expr, $hint:expr) => {
-        let a = $a;
-        let b = $b;
-        let hint = $hint;
-        let ar: &[u8] = a.as_ref();
-        let br: &[u8] = b.as_ref();
-        let hintr: &[usize] = hint.as_ref();
-        if ar != br {
-            use $crate::util::hexdump::print_hexdump;
-            panic!(
-                "assertion failed: `(left == right)`\nleft:\n{}\nright:\n{}\n",
-                print_hexdump(ar, hintr),
-                print_hexdump(br, hintr),
-            )
-        }
-    };
-}
-
-#[cfg(test)]
-mod tests {
-    use super::{parse_hexdump, print_hexdump};
-
-    #[test]
-    fn test_basic() {
-        let input = r"
-            a1b2 # comment
-            3c4d
-        ";
-        let result = parse_hexdump(input).unwrap();
-        assert_eq!(result, vec![0xa1, 0xb2, 0x3c, 0x4d]);
-    }
-
-    #[test]
-    fn test_upper_case() {
-        let input = r"
-            A1B2 # comment
-            3C4D
-        ";
-        let result = parse_hexdump(input).unwrap();
-        assert_eq!(result, vec![0xa1, 0xb2, 0x3c, 0x4d]);
-    }
-
-    #[test]
-    fn test_mixed_case() {
-        let input = r"
-            a1B2 # comment
-            3C4d
-        ";
-        let result = parse_hexdump(input).unwrap();
-        assert_eq!(result, vec![0xa1, 0xb2, 0x3c, 0x4d]);
-    }
-
-    #[test]
-    fn test_odd_characters() {
-        let input = r"
-            a1b
-        ";
-        let result = parse_hexdump(input);
-        assert!(result.is_err());
-    }
-
-    #[test]
-    fn test_invalid_characters() {
-        let input = r"
-            a1g2 # 'g' is not valid in hex
-        ";
-        let result = parse_hexdump(input);
-        assert!(result.is_err());
-    }
-    #[test]
-    fn test_basic_hexdump() {
-        let data: &[u8] = &[0x1, 0x2, 0x3, 0x4, 0x5];
-        let output = print_hexdump(data, [1, 2]);
-        assert_eq!(output, "01\n02 03\n04 05\n");
-    }
-
-    #[test]
-    fn test_newline_insertion() {
-        let data: &[u8] = &[0x1, 0x2, 0x3, 0x4];
-        let output = print_hexdump(data, [1, 0, 2]);
-        assert_eq!(output, "01\n\n02 03\n04\n");
-    }
-
-    #[test]
-    fn test_indefinite_line_length() {
-        let data: &[u8] = &[0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8];
-        let output = print_hexdump(data, [2, 4]);
-        assert_eq!(output, "01 02\n03 04 05 06\n07 08\n");
-    }
-
-    #[test]
-    fn test_empty_data() {
-        let data: &[u8] = &[];
-        let output = print_hexdump(data, [1, 2]);
-        assert_eq!(output, "");
-    }
-
-    #[test]
-    fn test_zeros_then_default() {
-        let data: &[u8] = &[0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8];
-        let output = print_hexdump(data, [1, 0, 0, 2]);
-        assert_eq!(output, "01\n\n\n02 03\n04 05\n06 07\n08\n");
-    }
-}
diff --git a/src/util/io.rs b/src/util/io.rs
deleted file mode 100644
index fa75f4dab..000000000
--- a/src/util/io.rs
+++ /dev/null
@@ -1,102 +0,0 @@
-//! Utilities for working with tokio io
-
-use std::{io, pin::Pin, task::Poll};
-
-use iroh_io::AsyncStreamReader;
-use tokio::io::AsyncWrite;
-
-/// A reader that tracks the number of bytes read
-#[derive(Debug)]
-pub struct TrackingReader<R> {
-    inner: R,
-    read: u64,
-}
-
-impl<R> TrackingReader<R> {
-    /// Wrap a reader in a tracking reader
-    pub fn new(inner: R) -> Self {
-        Self { inner, read: 0 }
-    }
-
-    /// Get the number of bytes read
-    #[allow(dead_code)]
-    pub fn bytes_read(&self) -> u64 {
-        self.read
-    }
-
-    /// Get the inner reader
-    pub fn into_parts(self) -> (R, u64) {
-        (self.inner, self.read)
-    }
-}
-
-impl<R> AsyncStreamReader for TrackingReader<R>
-where
-    R: AsyncStreamReader,
-{
-    async fn read_bytes(&mut self, len: usize) -> io::Result<bytes::Bytes> {
-        let bytes = self.inner.read_bytes(len).await?;
-        self.read = self.read.saturating_add(bytes.len() as u64);
-        Ok(bytes)
-    }
-
-    async fn read<const L: usize>(&mut self) -> io::Result<[u8; L]> {
-        let res = self.inner.read::<L>().await?;
-        self.read = self.read.saturating_add(L as u64);
-        Ok(res)
-    }
-}
-
-/// A writer that tracks the number of bytes written
-#[derive(Debug)]
-pub struct TrackingWriter<W> {
-    inner: W,
-    written: u64,
-}
-
-impl<W> TrackingWriter<W> {
-    /// Wrap a writer in a tracking writer
-    pub fn new(inner: W) -> Self {
-        Self { inner, written: 0 }
-    }
-
-    /// Get the number of bytes written
-    #[allow(dead_code)]
-    pub fn bytes_written(&self) -> u64 {
-        self.written
-    }
-
-    /// Get the inner writer
-    pub fn into_parts(self) -> (W, u64) {
-        (self.inner, self.written)
-    }
-}
-
-impl<W: AsyncWrite + Unpin> AsyncWrite for TrackingWriter<W> {
-    fn poll_write(
-        mut self: Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-        buf: &[u8],
-    ) -> Poll<io::Result<usize>> {
-        let this = &mut *self;
-        let res = Pin::new(&mut this.inner).poll_write(cx, buf);
-        if let Poll::Ready(Ok(size)) = res {
-            this.written = this.written.saturating_add(size as u64);
-        }
-        res
-    }
-
-    fn poll_flush(
-        mut self: Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-    ) -> Poll<io::Result<()>> {
-        Pin::new(&mut self.inner).poll_flush(cx)
-    }
-
-    fn poll_shutdown(
-        mut self: Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-    ) -> Poll<io::Result<()>> {
-        Pin::new(&mut self.inner).poll_shutdown(cx)
-    }
-}
diff --git a/src/util/local_pool.rs b/src/util/local_pool.rs
deleted file mode 100644
index 5b0880dd9..000000000
--- a/src/util/local_pool.rs
+++ /dev/null
@@ -1,686 +0,0 @@
-//! A local task pool with proper shutdown
-use std::{
-    any::Any,
-    future::Future,
-    ops::Deref,
-    pin::Pin,
-    sync::{
-        atomic::{AtomicBool, Ordering},
-        Arc,
-    },
-};
-
-use futures_lite::FutureExt;
-use tokio::{
-    sync::{Notify, Semaphore},
-    task::{JoinError, JoinSet, LocalSet},
-};
-
-type BoxedFut<T = ()> = Pin<Box<dyn Future<Output = T>>>;
-type SpawnFn<T = ()> = Box<dyn FnOnce() -> BoxedFut<T> + Send + 'static>;
-
-enum Message {
-    /// Create a new task and execute it locally
-    Execute(SpawnFn),
-    /// Shutdown the thread after finishing all tasks
-    Finish,
-}
-
-/// A local task pool with proper shutdown
-///
-/// Unlike
-/// [`LocalPoolHandle`](https://docs.rs/tokio-util/latest/tokio_util/task/struct.LocalPoolHandle.html),
-/// this pool will join all its threads when dropped, ensuring that all Drop
-/// implementations are run to completion.
-///
-/// On drop, this pool will immediately cancel all *tasks* that are currently
-/// being executed, and will wait for all threads to finish executing their
-/// loops before returning. This means that all drop implementations will be
-/// able to run to completion before drop exits.
-///
-/// On [`LocalPool::finish`], this pool will notify all threads to shut down,
-/// and then wait for all threads to finish executing their loops before
-/// returning. This means that all currently executing tasks will be allowed to
-/// run to completion.
-///
-/// The pool will install the [`tracing::Subscriber`] which was set on the current thread of
-/// where it was created as the default subscriber in all spawned threads.
-#[derive(Debug)]
-pub struct LocalPool {
-    threads: Vec<std::thread::JoinHandle<()>>,
-    shutdown_sem: Arc<Semaphore>,
-    cancel_token: CancellationToken,
-    handle: LocalPoolHandle,
-}
-
-impl Deref for LocalPool {
-    type Target = LocalPoolHandle;
-
-    fn deref(&self) -> &Self::Target {
-        &self.handle
-    }
-}
-
-/// A handle to a [`LocalPool`]
-#[derive(Debug, Clone)]
-pub struct LocalPoolHandle {
-    /// The sender half of the channel used to send tasks to the pool
-    send: async_channel::Sender<Message>,
-}
-
-/// What to do when a panic occurs in a pool thread
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum PanicMode {
-    /// Log the panic and continue
-    ///
-    /// The panic will be re-thrown when the pool is dropped.
-    LogAndContinue,
-    /// Log the panic and immediately shut down the pool.
-    ///
-    /// The panic will be re-thrown when the pool is dropped.
-    Shutdown,
-}
-
-/// Local task pool configuration
-#[derive(Clone, Debug)]
-pub struct Config {
-    /// Number of threads in the pool
-    pub threads: usize,
-    /// Prefix for thread names
-    pub thread_name_prefix: &'static str,
-    /// Ignore panics in pool threads
-    pub panic_mode: PanicMode,
-}
-
-impl Default for Config {
-    fn default() -> Self {
-        Self {
-            threads: num_cpus::get(),
-            thread_name_prefix: "local-pool",
-            panic_mode: PanicMode::Shutdown,
-        }
-    }
-}
-
-impl Default for LocalPool {
-    fn default() -> Self {
-        Self::new(Default::default())
-    }
-}
-
-impl LocalPool {
-    /// Create a new local pool with a single std thread.
-    pub fn single() -> Self {
-        Self::new(Config {
-            threads: 1,
-            ..Default::default()
-        })
-    }
-
-    /// Create a new local pool with the given config.
-    ///
-    /// This will use the current tokio runtime handle, so it must be called
-    /// from within a tokio runtime.
-    pub fn new(config: Config) -> Self {
-        let Config {
-            threads,
-            thread_name_prefix,
-            panic_mode,
-        } = config;
-        let cancel_token = CancellationToken::new();
-        let (send, recv) = async_channel::unbounded::<Message>();
-        let shutdown_sem = Arc::new(Semaphore::new(0));
-        let handle = tokio::runtime::Handle::current();
-        let handles = (0..threads)
-            .map(|i| {
-                Self::spawn_pool_thread(
-                    format!("{thread_name_prefix}-{i}"),
-                    recv.clone(),
-                    cancel_token.clone(),
-                    panic_mode,
-                    shutdown_sem.clone(),
-                    handle.clone(),
-                )
-            })
-            .collect::<std::io::Result<Vec<_>>>()
-            .expect("invalid thread name");
-        Self {
-            threads: handles,
-            handle: LocalPoolHandle { send },
-            cancel_token,
-            shutdown_sem,
-        }
-    }
-
-    /// Get a cheaply cloneable handle to the pool
-    ///
-    /// This is not strictly necessary since we implement deref for
-    /// LocalPoolHandle, but makes getting a handle more explicit.
-    pub fn handle(&self) -> &LocalPoolHandle {
-        &self.handle
-    }
-
-    /// Spawn a new pool thread.
-    fn spawn_pool_thread(
-        thread_name: String,
-        recv: async_channel::Receiver<Message>,
-        cancel_token: CancellationToken,
-        panic_mode: PanicMode,
-        shutdown_sem: Arc<Semaphore>,
-        handle: tokio::runtime::Handle,
-    ) -> std::io::Result<std::thread::JoinHandle<()>> {
-        let tracing_dispatcher = tracing::dispatcher::get_default(|dispatcher| dispatcher.clone());
-        std::thread::Builder::new()
-            .name(thread_name)
-            .spawn(move || {
-                let _tracing_guard = tracing::dispatcher::set_default(&tracing_dispatcher);
-                let mut s = JoinSet::new();
-                let mut last_panic = None;
-                let mut handle_join = |res: Option<std::result::Result<(), JoinError>>| -> bool {
-                    if let Some(Err(e)) = res {
-                        if let Ok(panic) = e.try_into_panic() {
-                            let panic_info = get_panic_info(&panic);
-                            let thread_name = get_thread_name();
-                            tracing::error!(
-                                "Panic in local pool thread: {}\n{}",
-                                thread_name,
-                                panic_info
-                            );
-                            last_panic = Some(panic);
-                        }
-                    }
-                    panic_mode == PanicMode::LogAndContinue || last_panic.is_none()
-                };
-                let ls = LocalSet::new();
-                let shutdown_mode = handle.block_on(ls.run_until(async {
-                    loop {
-                        tokio::select! {
-                            // poll the set of futures
-                            res = s.join_next(), if !s.is_empty() => {
-                                if !handle_join(res) {
-                                    break ShutdownMode::Stop;
-                                }
-                            },
-                            // if the cancel token is cancelled, break the loop immediately
-                            _ = cancel_token.cancelled() => break ShutdownMode::Stop,
-                            // if we receive a message, execute it
-                            msg = recv.recv() => {
-                                match msg {
-                                    // just push into the join set
-                                    Ok(Message::Execute(f)) => {
-                                        s.spawn_local((f)());
-                                    }
-                                    // break with optional semaphore
-                                    Ok(Message::Finish) => break ShutdownMode::Finish,
-                                    // if the sender is dropped, break the loop immediately
-                                    Err(async_channel::RecvError) => break ShutdownMode::Stop,
-                                }
-                            },
-                        }
-                    }
-                }));
-                // soft shutdown mode is just like normal running, except that
-                // we don't add any more tasks and stop when there are no more
-                // tasks to run.
-                if shutdown_mode == ShutdownMode::Finish {
-                    // somebody is asking for a clean shutdown, wait for all tasks to finish
-                    handle.block_on(ls.run_until(async {
-                        loop {
-                            tokio::select! {
-                                res = s.join_next() => {
-                                    if res.is_none() || !handle_join(res) {
-                                        break;
-                                    }
-                                }
-                                _ = cancel_token.cancelled() => break,
-                            }
-                        }
-                    }));
-                }
-                // Always add the permit. If nobody is waiting for it, it does
-                // no harm.
-                shutdown_sem.add_permits(1);
-                if let Some(_panic) = last_panic {
-                    // std::panic::resume_unwind(panic);
-                }
-            })
-    }
-
-    /// A future that resolves when the pool is cancelled
-    pub async fn cancelled(&self) {
-        self.cancel_token.cancelled().await
-    }
-
-    /// Immediately stop polling all tasks and wait for all threads to finish.
-    ///
-    /// This is like drop, but waits for thread completion asynchronously.
-    ///
-    /// If there was a panic on any of the threads, it will be re-thrown here.
-    pub async fn shutdown(self) {
-        self.cancel_token.cancel();
-        self.await_thread_completion().await;
-        // just make it explicit that this is where drop runs
-        drop(self);
-    }
-
-    /// Gently shut down the pool
-    ///
-    /// Notifies all the pool threads to shut down and waits for them to finish.
-    ///
-    /// If you just want to drop the pool without giving the threads a chance to
-    /// process their remaining tasks, just use [`Self::shutdown`].
-    ///
-    /// If you want to wait for only a limited time for the tasks to finish,
-    /// you can race this function with a timeout.
-    pub async fn finish(self) {
-        // we assume that there are exactly as many threads as there are handles.
-        // also, we assume that the threads are still running.
-        for _ in 0..self.threads_u32() {
-            // send the shutdown message
-            // sending will fail if all threads are already finished, but
-            // in that case we don't need to do anything.
-            //
-            // Threads will add a permit in any case, so await_thread_completion
-            // will then immediately return.
-            self.send.send(Message::Finish).await.ok();
-        }
-        self.await_thread_completion().await;
-    }
-
-    fn threads_u32(&self) -> u32 {
-        self.threads
-            .len()
-            .try_into()
-            .expect("invalid number of threads")
-    }
-
-    async fn await_thread_completion(&self) {
-        // wait for all threads to finish.
-        // Each thread will add a permit to the semaphore.
-        let wait_for_semaphore = async move {
-            let _ = self
-                .shutdown_sem
-                .acquire_many(self.threads_u32())
-                .await
-                .expect("semaphore closed");
-        };
-        // race the semaphore wait with the cancel token in case somebody
-        // cancels the pool while we are waiting.
-        tokio::select! {
-            _ = wait_for_semaphore => {}
-            _ = self.cancel_token.cancelled() => {}
-        }
-    }
-}
-
-impl Drop for LocalPool {
-    fn drop(&mut self) {
-        self.cancel_token.cancel();
-        let current_thread_id = std::thread::current().id();
-        for handle in self.threads.drain(..) {
-            // we have no control over from where Drop is called, especially
-            // if the pool ends up in an Arc. So we need to check if we are
-            // dropping from within a pool thread and skip it in that case.
-            if handle.thread().id() == current_thread_id {
-                tracing::error!("Dropping LocalPool from within a pool thread.");
-                continue;
-            }
-            // Log any panics and resume them
-            if let Err(panic) = handle.join() {
-                let panic_info = get_panic_info(&panic);
-                let thread_name = get_thread_name();
-                tracing::error!("Error joining thread: {}\n{}", thread_name, panic_info);
-                // std::panic::resume_unwind(panic);
-            }
-        }
-    }
-}
-
-/// Errors for spawn failures
-#[derive(thiserror::Error, Debug)]
-pub enum SpawnError {
-    /// Task was dropped, either due to a panic or because the pool was shut down.
-    #[error("cancelled")]
-    Cancelled,
-}
-
-type SpawnResult<T> = std::result::Result<T, SpawnError>;
-
-/// Future returned by [`LocalPoolHandle::spawn`] and [`LocalPoolHandle::try_spawn`].
-///
-/// Dropping this future will immediately cancel the task. The task can fail if
-/// the pool is shut down or if the task panics. In both cases the future will
-/// resolve to [`SpawnError::Cancelled`].
-#[repr(transparent)]
-#[derive(Debug)]
-pub struct Run<T>(tokio::sync::oneshot::Receiver<T>);
-
-impl<T> Run<T> {
-    /// Abort the task
-    ///
-    /// Dropping the future will also abort the task.
-    pub fn abort(&mut self) {
-        self.0.close();
-    }
-}
-
-impl<T> Future for Run<T> {
-    type Output = std::result::Result<T, SpawnError>;
-
-    fn poll(
-        mut self: Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-    ) -> std::task::Poll<Self::Output> {
-        // map a RecvError (other side was dropped) to a SpawnError::Shutdown
-        //
-        // The only way the receiver can be dropped is if the pool is shut down.
-        self.0.poll(cx).map_err(|_| SpawnError::Cancelled)
-    }
-}
-
-impl From<SpawnError> for std::io::Error {
-    fn from(e: SpawnError) -> Self {
-        std::io::Error::new(std::io::ErrorKind::Other, e)
-    }
-}
-
-impl LocalPoolHandle {
-    /// Get the number of tasks in the queue
-    ///
-    /// This is *not* the number of tasks being executed, but the number of
-    /// tasks waiting to be scheduled for execution. If this number is high,
-    /// it indicates that the pool is very busy.
-    ///
-    /// You might want to use this to throttle or reject requests.
-    pub fn waiting_tasks(&self) -> usize {
-        self.send.len()
-    }
-
-    /// Spawn a task in the pool and return a future that resolves when the task
-    /// is done.
-    ///
-    /// If you don't care about the result, prefer [`LocalPoolHandle::spawn_detached`]
-    /// since it is more efficient.
-    pub fn try_spawn<T, F, Fut>(&self, gen: F) -> SpawnResult<Run<T>>
-    where
-        F: FnOnce() -> Fut + Send + 'static,
-        Fut: Future<Output = T> + 'static,
-        T: Send + 'static,
-    {
-        let (mut send_res, recv_res) = tokio::sync::oneshot::channel();
-        let item = move || async move {
-            let fut = (gen)();
-            tokio::select! {
-                // send the result to the receiver
-                res = fut => { send_res.send(res).ok(); }
-                // immediately stop the task if the receiver is dropped
-                _ = send_res.closed() => {}
-            }
-        };
-        self.try_spawn_detached(item)?;
-        Ok(Run(recv_res))
-    }
-
-    /// Spawn a task in the pool.
-    ///
-    /// The task will run to completion unless the pool is shut down or the task
-    /// panics. In case of panic, the pool will either log the panic and continue
-    /// or immediately shut down, depending on the [`PanicMode`].
-    pub fn try_spawn_detached<F, Fut>(&self, gen: F) -> SpawnResult<()>
-    where
-        F: FnOnce() -> Fut + Send + 'static,
-        Fut: Future<Output = ()> + 'static,
-    {
-        let gen: SpawnFn = Box::new(move || Box::pin(gen()));
-        self.try_spawn_detached_boxed(gen)
-    }
-
-    /// Spawn a task in the pool and await the result.
-    ///
-    /// Like [`LocalPoolHandle::try_spawn`], but panics if the pool is shut down.
-    pub fn spawn<T, F, Fut>(&self, gen: F) -> Run<T>
-    where
-        F: FnOnce() -> Fut + Send + 'static,
-        Fut: Future<Output = T> + 'static,
-        T: Send + 'static,
-    {
-        self.try_spawn(gen).expect("pool is shut down")
-    }
-
-    /// Spawn a task in the pool.
-    ///
-    /// Like [`LocalPoolHandle::try_spawn_detached`], but panics if the pool is shut down.
-    pub fn spawn_detached<F, Fut>(&self, gen: F)
-    where
-        F: FnOnce() -> Fut + Send + 'static,
-        Fut: Future<Output = ()> + 'static,
-    {
-        self.try_spawn_detached(gen).expect("pool is shut down")
-    }
-
-    /// Spawn a task in the pool.
-    ///
-    /// This is like [`LocalPoolHandle::try_spawn_detached`], but assuming that the
-    /// generator function is already boxed. This is the lowest overhead way to
-    /// spawn a task in the pool.
-    pub fn try_spawn_detached_boxed(&self, gen: SpawnFn) -> SpawnResult<()> {
-        self.send
-            .send_blocking(Message::Execute(gen))
-            .map_err(|_| SpawnError::Cancelled)
-    }
-}
-
-/// Thread shutdown mode
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-enum ShutdownMode {
-    /// Finish all tasks and then stop
-    Finish,
-    /// Stop immediately
-    Stop,
-}
-
-fn get_panic_info(panic: &Box<dyn Any + Send>) -> String {
-    if let Some(s) = panic.downcast_ref::<&str>() {
-        s.to_string()
-    } else if let Some(s) = panic.downcast_ref::<String>() {
-        s.clone()
-    } else {
-        "Panic info unavailable".to_string()
-    }
-}
-
-fn get_thread_name() -> String {
-    std::thread::current()
-        .name()
-        .unwrap_or("unnamed")
-        .to_string()
-}
-
-/// A lightweight cancellation token
-#[derive(Debug, Clone)]
-struct CancellationToken {
-    inner: Arc<CancellationTokenInner>,
-}
-
-#[derive(Debug)]
-struct CancellationTokenInner {
-    is_cancelled: AtomicBool,
-    notify: Notify,
-}
-
-impl CancellationToken {
-    fn new() -> Self {
-        Self {
-            inner: Arc::new(CancellationTokenInner {
-                is_cancelled: AtomicBool::new(false),
-                notify: Notify::new(),
-            }),
-        }
-    }
-
-    fn cancel(&self) {
-        if !self.inner.is_cancelled.swap(true, Ordering::SeqCst) {
-            self.inner.notify.notify_waiters();
-        }
-    }
-
-    async fn cancelled(&self) {
-        if self.is_cancelled() {
-            return;
-        }
-
-        // Wait for notification if not cancelled
-        self.inner.notify.notified().await;
-    }
-
-    fn is_cancelled(&self) -> bool {
-        self.inner.is_cancelled.load(Ordering::SeqCst)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::{sync::atomic::AtomicU64, time::Duration};
-
-    use tracing::info;
-    use tracing_test::traced_test;
-
-    use super::*;
-
-    /// A struct that simulates a long running drop operation
-    #[derive(Debug)]
-    struct TestDrop(Option<Arc<AtomicU64>>);
-
-    impl Drop for TestDrop {
-        fn drop(&mut self) {
-            // delay to make sure the drop is executed completely
-            std::thread::sleep(Duration::from_millis(100));
-            // increment the drop counter
-            if let Some(counter) = self.0.take() {
-                counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
-            }
-        }
-    }
-
-    impl TestDrop {
-        fn new(counter: Arc<AtomicU64>) -> Self {
-            Self(Some(counter))
-        }
-
-        fn forget(mut self) {
-            self.0.take();
-        }
-    }
-
-    /// Create a non-send test future that captures a TestDrop instance
-    async fn delay_then_drop(x: TestDrop) {
-        tokio::time::sleep(Duration::from_millis(100)).await;
-        // drop x at the end. we will never get here when the future is
-        // no longer polled, but drop should still be called
-        drop(x);
-    }
-
-    /// Use a TestDrop instance to test cancellation
-    async fn delay_then_forget(x: TestDrop, delay: Duration) {
-        tokio::time::sleep(delay).await;
-        x.forget();
-    }
-
-    #[tokio::test]
-    #[traced_test]
-    async fn test_tracing() {
-        // This test wants to make sure that logging inside the pool propagates to the
-        // tracing subscriber that was set for the current thread at the time the pool was
-        // created.
-        //
-        // Look, there should be a custom tracing subscriber here that allows us to inspect
-        // the messages sent to it so we can verify it received all the messages.  But have
-        // you ever tried to implement a tracing subscriber?  In the mean time this test will
-        // just always pass, to really see the test run it with:
-        //
-        // cargo nextest run -p iroh-blobs local_pool::tests::test_tracing --success-output final
-        //
-        // and eyeball the output.  yolo
-        info!("hello from the test");
-        let pool = LocalPool::single();
-        pool.spawn(|| async move {
-            info!("hello from the pool");
-        })
-        .await
-        .unwrap();
-    }
-
-    #[tokio::test]
-    async fn test_drop() {
-        let _ = tracing_subscriber::fmt::try_init();
-        let pool = LocalPool::new(Config::default());
-        let counter = Arc::new(AtomicU64::new(0));
-        let n = 4;
-        for _ in 0..n {
-            let td = TestDrop::new(counter.clone());
-            pool.spawn_detached(move || delay_then_drop(td));
-        }
-        drop(pool);
-        assert_eq!(counter.load(std::sync::atomic::Ordering::SeqCst), n);
-    }
-
-    #[tokio::test]
-    async fn test_finish() {
-        let _ = tracing_subscriber::fmt::try_init();
-        let pool = LocalPool::new(Config::default());
-        let counter = Arc::new(AtomicU64::new(0));
-        let n = 4;
-        for _ in 0..n {
-            let td = TestDrop::new(counter.clone());
-            pool.spawn_detached(move || delay_then_drop(td));
-        }
-        pool.finish().await;
-        assert_eq!(counter.load(std::sync::atomic::Ordering::SeqCst), n);
-    }
-
-    #[tokio::test]
-    async fn test_cancel() {
-        let _ = tracing_subscriber::fmt::try_init();
-        let pool = LocalPool::new(Config {
-            threads: 2,
-            ..Config::default()
-        });
-        let c1 = Arc::new(AtomicU64::new(0));
-        let td1 = TestDrop::new(c1.clone());
-        let handle = pool.spawn(move || {
-            // this one will be aborted anyway, so use a long delay to make sure
-            // that it does not accidentally run to completion
-            delay_then_forget(td1, Duration::from_secs(10))
-        });
-        drop(handle);
-        let c2 = Arc::new(AtomicU64::new(0));
-        let td2 = TestDrop::new(c2.clone());
-        let _handle = pool.spawn(move || {
-            // this one will not be aborted, so use a short delay so the test
-            // does not take too long
-            delay_then_forget(td2, Duration::from_millis(100))
-        });
-        pool.finish().await;
-        // c1 will be aborted, so drop will run before forget, so the counter will be increased
-        assert_eq!(c1.load(std::sync::atomic::Ordering::SeqCst), 1);
-        // c2 will not be aborted, so drop will run after forget, so the counter will not be increased
-        assert_eq!(c2.load(std::sync::atomic::Ordering::SeqCst), 0);
-    }
-
-    // #[tokio::test]
-    // #[should_panic]
-    // #[ignore = "todo"]
-    // async fn test_panic() {
-    //     let _ = tracing_subscriber::fmt::try_init();
-    //     let pool = LocalPool::new(Config {
-    //         threads: 2,
-    //         ..Config::default()
-    //     });
-    //     pool.spawn_detached(|| async {
-    //         panic!("test panic");
-    //     });
-    //     // we can't use shutdown here, because we need to allow time for the
-    //     // panic to happen.
-    //     pool.finish().await;
-    // }
-}
diff --git a/src/util/mem_or_file.rs b/src/util/mem_or_file.rs
deleted file mode 100644
index d929a19c9..000000000
--- a/src/util/mem_or_file.rs
+++ /dev/null
@@ -1,104 +0,0 @@
-use std::{fs::File, io};
-
-use bao_tree::io::sync::{ReadAt, Size};
-use bytes::Bytes;
-
-/// This is a general purpose Either, just like Result, except that the two cases
-/// are Mem for something that is in memory, and File for something that is somewhere
-/// external and only available via io.
-#[derive(Debug)]
-pub enum MemOrFile<M, F> {
-    /// We got it all in memory
-    Mem(M),
-    /// A file
-    File(F),
-}
-
-/// Helper methods for a common way to use MemOrFile, where the memory part is something
-/// like a slice, and the file part is a tuple consisiting of path or file and size.
-impl<M, F> MemOrFile<M, (F, u64)>
-where
-    M: AsRef<[u8]>,
-{
-    /// Get the size of the MemOrFile
-    pub fn size(&self) -> u64 {
-        match self {
-            MemOrFile::Mem(mem) => mem.as_ref().len() as u64,
-            MemOrFile::File((_, size)) => *size,
-        }
-    }
-}
-
-impl ReadAt for MemOrFile<Bytes, File> {
-    fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<usize> {
-        match self {
-            MemOrFile::Mem(mem) => mem.as_ref().read_at(offset, buf),
-            MemOrFile::File(file) => file.read_at(offset, buf),
-        }
-    }
-}
-
-impl Size for MemOrFile<Bytes, File> {
-    fn size(&self) -> io::Result<Option<u64>> {
-        match self {
-            MemOrFile::Mem(mem) => Ok(Some(mem.len() as u64)),
-            MemOrFile::File(file) => file.size(),
-        }
-    }
-}
-
-impl<M: Default, F> Default for MemOrFile<M, F> {
-    fn default() -> Self {
-        MemOrFile::Mem(Default::default())
-    }
-}
-
-impl<M, F> MemOrFile<M, F> {
-    /// Turn a reference to a MemOrFile into a MemOrFile of references
-    pub fn as_ref(&self) -> MemOrFile<&M, &F> {
-        match self {
-            MemOrFile::Mem(mem) => MemOrFile::Mem(mem),
-            MemOrFile::File(file) => MemOrFile::File(file),
-        }
-    }
-
-    /// True if this is a Mem
-    pub fn is_mem(&self) -> bool {
-        matches!(self, MemOrFile::Mem(_))
-    }
-
-    /// Get the mem part
-    pub fn mem(&self) -> Option<&M> {
-        match self {
-            MemOrFile::Mem(mem) => Some(mem),
-            MemOrFile::File(_) => None,
-        }
-    }
-
-    /// Map the file part of this MemOrFile
-    pub fn map_file<F2>(self, f: impl FnOnce(F) -> F2) -> MemOrFile<M, F2> {
-        match self {
-            MemOrFile::Mem(mem) => MemOrFile::Mem(mem),
-            MemOrFile::File(file) => MemOrFile::File(f(file)),
-        }
-    }
-
-    /// Try to map the file part of this MemOrFile
-    pub fn try_map_file<F2, E>(
-        self,
-        f: impl FnOnce(F) -> Result<F2, E>,
-    ) -> Result<MemOrFile<M, F2>, E> {
-        match self {
-            MemOrFile::Mem(mem) => Ok(MemOrFile::Mem(mem)),
-            MemOrFile::File(file) => f(file).map(MemOrFile::File),
-        }
-    }
-
-    /// Map the memory part of this MemOrFile
-    pub fn map_mem<M2>(self, f: impl FnOnce(M) -> M2) -> MemOrFile<M2, F> {
-        match self {
-            MemOrFile::Mem(mem) => MemOrFile::Mem(f(mem)),
-            MemOrFile::File(file) => MemOrFile::File(file),
-        }
-    }
-}
diff --git a/src/util/progress.rs b/src/util/progress.rs
deleted file mode 100644
index a84d46bc7..000000000
--- a/src/util/progress.rs
+++ /dev/null
@@ -1,687 +0,0 @@
-//! Utilities for reporting progress.
-//!
-//! The main entry point is the [ProgressSender] trait.
-use std::{future::Future, io, marker::PhantomData, ops::Deref, sync::Arc};
-
-use bytes::Bytes;
-use iroh_io::AsyncSliceWriter;
-
-/// A general purpose progress sender. This should be usable for reporting progress
-/// from both blocking and non-blocking contexts.
-///
-/// # Id generation
-///
-/// Any good progress protocol will refer to entities by means of a unique id.
-/// E.g. if you want to report progress about some file operation, including details
-/// such as the full path of the file would be very wasteful. It is better to
-/// introduce a unique id for the file and then report progress using that id.
-///
-/// The [IdGenerator] trait provides a method to generate such ids, [IdGenerator::new_id].
-///
-/// # Sending important messages
-///
-/// Some messages are important for the receiver to receive. E.g. start and end
-/// messages for some operation. If the receiver would miss one of these messages,
-/// it would lose the ability to make sense of the progress message stream.
-///
-/// This trait provides a method to send such important messages, in both blocking
-/// contexts where you have to block until the message is sent [ProgressSender::blocking_send],
-/// and non-blocking contexts where you have to yield until the message is sent [ProgressSender::send].
-///
-/// # Sending unimportant messages
-///
-/// Some messages are self-contained and not important for the receiver to receive.
-/// E.g. if you send millions of progress messages for copying a file that each
-/// contain an id and the number of bytes copied so far, it is not important for
-/// the receiver to receive every single one of these messages. In fact it is
-/// useful to drop some of these messages because waiting for the progress events
-/// to be sent can slow down the actual operation.
-///
-/// This trait provides a method to send such unimportant messages that can be
-/// used in both blocking and non-blocking contexts, [ProgressSender::try_send].
-///
-/// # Errors
-///
-/// When the receiver is dropped, sending a message will fail. This provides a way
-/// for the receiver to signal that the operation should be stopped.
-///
-/// E.g. for a blocking copy operation that reports frequent progress messages,
-/// as soon as the receiver is dropped, this is a signal to stop the copy operation.
-///
-/// The error type is [ProgressSendError], which can be converted to an [std::io::Error]
-/// for convenience.
-///
-/// # Transforming the message type
-///
-/// Sometimes you have a progress sender that sends a message of type `A` but an
-/// operation that reports progress of type `B`. If you have a transformation for
-/// every `B` to an `A`, you can use the [ProgressSender::with_map] method to transform the message.
-///
-/// This is similar to the `futures::SinkExt::with` method.
-///
-/// # Filtering the message type
-///
-/// Sometimes you have a progress sender that sends a message of enum `A` but an
-/// operation that reports progress of type `B`. You are interested only in some
-/// enum cases of `A` that can be transformed to `B`. You can use the [ProgressSender::with_filter_map]
-/// method to filter and transform the message.
-///
-/// # No-op progress sender
-///
-/// If you don't want to report progress, you can use the [IgnoreProgressSender] type.
-///
-/// # Async channel progress sender
-///
-/// If you want to use an async channel, you can use the [AsyncChannelProgressSender] type.
-///
-/// # Implementing your own progress sender
-///
-/// Progress senders will frequently be used in a multi-threaded context.
-///
-/// They must be **cheap** to clone and send between threads.
-/// They must also be thread safe, which is ensured by the [Send] and [Sync] bounds.
-/// They must also be unencumbered by lifetimes, which is ensured by the `'static` bound.
-///
-/// A typical implementation will wrap the sender part of a channel and an id generator.
-pub trait ProgressSender: std::fmt::Debug + Clone + Send + Sync + 'static {
-    /// The message being sent.
-    type Msg: Send + Sync + 'static;
-
-    /// Send a message and wait if the receiver is full.
-    ///
-    /// Use this to send important progress messages where delivery must be guaranteed.
-    #[must_use]
-    fn send(&self, msg: Self::Msg) -> impl Future<Output = ProgressSendResult<()>> + Send;
-
-    /// Try to send a message and drop it if the receiver is full.
-    ///
-    /// Use this to send progress messages where delivery is not important, e.g. a self contained progress message.
-    fn try_send(&self, msg: Self::Msg) -> ProgressSendResult<()>;
-
-    /// Send a message and block if the receiver is full.
-    ///
-    /// Use this to send important progress messages where delivery must be guaranteed.
-    fn blocking_send(&self, msg: Self::Msg) -> ProgressSendResult<()>;
-
-    /// Transform the message type by mapping to the type of this sender.
-    fn with_map<U: Send + Sync + 'static, F: Fn(U) -> Self::Msg + Send + Sync + Clone + 'static>(
-        self,
-        f: F,
-    ) -> WithMap<Self, U, F> {
-        WithMap(self, f, PhantomData)
-    }
-
-    /// Transform the message type by filter-mapping to the type of this sender.
-    fn with_filter_map<
-        U: Send + Sync + 'static,
-        F: Fn(U) -> Option<Self::Msg> + Send + Sync + Clone + 'static,
-    >(
-        self,
-        f: F,
-    ) -> WithFilterMap<Self, U, F> {
-        WithFilterMap(self, f, PhantomData)
-    }
-
-    /// Create a boxed progress sender to get rid of the concrete type.
-    fn boxed(self) -> BoxedProgressSender<Self::Msg>
-    where
-        Self: IdGenerator,
-    {
-        BoxedProgressSender(Arc::new(BoxableProgressSenderWrapper(self)))
-    }
-}
-
-/// A boxed progress sender
-pub struct BoxedProgressSender<T>(Arc<dyn BoxableProgressSender<T>>);
-
-impl<T> Clone for BoxedProgressSender<T> {
-    fn clone(&self) -> Self {
-        Self(self.0.clone())
-    }
-}
-
-impl<T> std::fmt::Debug for BoxedProgressSender<T> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_tuple("BoxedProgressSender").field(&self.0).finish()
-    }
-}
-
-type BoxFuture<'a, T> = std::pin::Pin<Box<dyn Future<Output = T> + Send + 'a>>;
-
-/// Boxable progress sender
-trait BoxableProgressSender<T>: IdGenerator + std::fmt::Debug + Send + Sync + 'static {
-    /// Send a message and wait if the receiver is full.
-    ///
-    /// Use this to send important progress messages where delivery must be guaranteed.
-    #[must_use]
-    fn send(&self, msg: T) -> BoxFuture<'_, ProgressSendResult<()>>;
-
-    /// Try to send a message and drop it if the receiver is full.
-    ///
-    /// Use this to send progress messages where delivery is not important, e.g. a self contained progress message.
-    fn try_send(&self, msg: T) -> ProgressSendResult<()>;
-
-    /// Send a message and block if the receiver is full.
-    ///
-    /// Use this to send important progress messages where delivery must be guaranteed.
-    fn blocking_send(&self, msg: T) -> ProgressSendResult<()>;
-}
-
-impl<I: ProgressSender + IdGenerator> BoxableProgressSender<I::Msg>
-    for BoxableProgressSenderWrapper<I>
-{
-    fn send(&self, msg: I::Msg) -> BoxFuture<'_, ProgressSendResult<()>> {
-        Box::pin(self.0.send(msg))
-    }
-
-    fn try_send(&self, msg: I::Msg) -> ProgressSendResult<()> {
-        self.0.try_send(msg)
-    }
-
-    fn blocking_send(&self, msg: I::Msg) -> ProgressSendResult<()> {
-        self.0.blocking_send(msg)
-    }
-}
-
-/// Boxable progress sender wrapper, used internally.
-#[derive(Debug)]
-#[repr(transparent)]
-struct BoxableProgressSenderWrapper<I>(I);
-
-impl<I: ProgressSender + IdGenerator> IdGenerator for BoxableProgressSenderWrapper<I> {
-    fn new_id(&self) -> u64 {
-        self.0.new_id()
-    }
-}
-
-impl<T: Send + Sync + 'static> IdGenerator for Arc<dyn BoxableProgressSender<T>> {
-    fn new_id(&self) -> u64 {
-        self.deref().new_id()
-    }
-}
-
-impl<T: Send + Sync + 'static> ProgressSender for Arc<dyn BoxableProgressSender<T>> {
-    type Msg = T;
-
-    fn send(&self, msg: T) -> impl Future<Output = ProgressSendResult<()>> + Send {
-        self.deref().send(msg)
-    }
-
-    fn try_send(&self, msg: T) -> ProgressSendResult<()> {
-        self.deref().try_send(msg)
-    }
-
-    fn blocking_send(&self, msg: T) -> ProgressSendResult<()> {
-        self.deref().blocking_send(msg)
-    }
-}
-
-impl<T: Send + Sync + 'static> IdGenerator for BoxedProgressSender<T> {
-    fn new_id(&self) -> u64 {
-        self.0.new_id()
-    }
-}
-
-impl<T: Send + Sync + 'static> ProgressSender for BoxedProgressSender<T> {
-    type Msg = T;
-
-    async fn send(&self, msg: T) -> ProgressSendResult<()> {
-        self.0.send(msg).await
-    }
-
-    fn try_send(&self, msg: T) -> ProgressSendResult<()> {
-        self.0.try_send(msg)
-    }
-
-    fn blocking_send(&self, msg: T) -> ProgressSendResult<()> {
-        self.0.blocking_send(msg)
-    }
-}
-
-impl<T: ProgressSender> ProgressSender for Option<T> {
-    type Msg = T::Msg;
-
-    async fn send(&self, msg: Self::Msg) -> ProgressSendResult<()> {
-        if let Some(inner) = self {
-            inner.send(msg).await
-        } else {
-            Ok(())
-        }
-    }
-
-    fn try_send(&self, msg: Self::Msg) -> ProgressSendResult<()> {
-        if let Some(inner) = self {
-            inner.try_send(msg)
-        } else {
-            Ok(())
-        }
-    }
-
-    fn blocking_send(&self, msg: Self::Msg) -> ProgressSendResult<()> {
-        if let Some(inner) = self {
-            inner.blocking_send(msg)
-        } else {
-            Ok(())
-        }
-    }
-}
-
-/// An id generator, to be combined with a progress sender.
-pub trait IdGenerator {
-    /// Get a new unique id
-    fn new_id(&self) -> u64;
-}
-
-/// A no-op progress sender.
-pub struct IgnoreProgressSender<T>(PhantomData<T>);
-
-impl<T> Default for IgnoreProgressSender<T> {
-    fn default() -> Self {
-        Self(PhantomData)
-    }
-}
-
-impl<T> Clone for IgnoreProgressSender<T> {
-    fn clone(&self) -> Self {
-        Self(PhantomData)
-    }
-}
-
-impl<T> std::fmt::Debug for IgnoreProgressSender<T> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("IgnoreProgressSender").finish()
-    }
-}
-
-impl<T: Send + Sync + 'static> ProgressSender for IgnoreProgressSender<T> {
-    type Msg = T;
-
-    async fn send(&self, _msg: T) -> std::result::Result<(), ProgressSendError> {
-        Ok(())
-    }
-
-    fn try_send(&self, _msg: T) -> std::result::Result<(), ProgressSendError> {
-        Ok(())
-    }
-
-    fn blocking_send(&self, _msg: T) -> std::result::Result<(), ProgressSendError> {
-        Ok(())
-    }
-}
-
-impl<T> IdGenerator for IgnoreProgressSender<T> {
-    fn new_id(&self) -> u64 {
-        0
-    }
-}
-
-/// Transform the message type by mapping to the type of this sender.
-///
-/// See [ProgressSender::with_map].
-pub struct WithMap<
-    I: ProgressSender,
-    U: Send + Sync + 'static,
-    F: Fn(U) -> I::Msg + Clone + Send + Sync + 'static,
->(I, F, PhantomData<U>);
-
-impl<
-        I: ProgressSender,
-        U: Send + Sync + 'static,
-        F: Fn(U) -> I::Msg + Clone + Send + Sync + 'static,
-    > std::fmt::Debug for WithMap<I, U, F>
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_tuple("With").field(&self.0).finish()
-    }
-}
-
-impl<
-        I: ProgressSender,
-        U: Send + Sync + 'static,
-        F: Fn(U) -> I::Msg + Clone + Send + Sync + 'static,
-    > Clone for WithMap<I, U, F>
-{
-    fn clone(&self) -> Self {
-        Self(self.0.clone(), self.1.clone(), PhantomData)
-    }
-}
-
-impl<
-        I: ProgressSender,
-        U: Send + Sync + 'static,
-        F: Fn(U) -> I::Msg + Clone + Send + Sync + 'static,
-    > ProgressSender for WithMap<I, U, F>
-{
-    type Msg = U;
-
-    async fn send(&self, msg: U) -> std::result::Result<(), ProgressSendError> {
-        let msg = (self.1)(msg);
-        self.0.send(msg).await
-    }
-
-    fn try_send(&self, msg: U) -> std::result::Result<(), ProgressSendError> {
-        let msg = (self.1)(msg);
-        self.0.try_send(msg)
-    }
-
-    fn blocking_send(&self, msg: U) -> std::result::Result<(), ProgressSendError> {
-        let msg = (self.1)(msg);
-        self.0.blocking_send(msg)
-    }
-}
-
-/// Transform the message type by filter-mapping to the type of this sender.
-///
-/// See [ProgressSender::with_filter_map].
-pub struct WithFilterMap<I, U, F>(I, F, PhantomData<U>);
-
-impl<
-        I: ProgressSender,
-        U: Send + Sync + 'static,
-        F: Fn(U) -> Option<I::Msg> + Clone + Send + Sync + 'static,
-    > std::fmt::Debug for WithFilterMap<I, U, F>
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_tuple("FilterWith").field(&self.0).finish()
-    }
-}
-
-impl<
-        I: ProgressSender,
-        U: Send + Sync + 'static,
-        F: Fn(U) -> Option<I::Msg> + Clone + Send + Sync + 'static,
-    > Clone for WithFilterMap<I, U, F>
-{
-    fn clone(&self) -> Self {
-        Self(self.0.clone(), self.1.clone(), PhantomData)
-    }
-}
-
-impl<I: IdGenerator, U, F> IdGenerator for WithFilterMap<I, U, F> {
-    fn new_id(&self) -> u64 {
-        self.0.new_id()
-    }
-}
-
-impl<
-        I: IdGenerator + ProgressSender,
-        U: Send + Sync + 'static,
-        F: Fn(U) -> I::Msg + Clone + Send + Sync + 'static,
-    > IdGenerator for WithMap<I, U, F>
-{
-    fn new_id(&self) -> u64 {
-        self.0.new_id()
-    }
-}
-
-impl<
-        I: ProgressSender,
-        U: Send + Sync + 'static,
-        F: Fn(U) -> Option<I::Msg> + Clone + Send + Sync + 'static,
-    > ProgressSender for WithFilterMap<I, U, F>
-{
-    type Msg = U;
-
-    async fn send(&self, msg: U) -> std::result::Result<(), ProgressSendError> {
-        if let Some(msg) = (self.1)(msg) {
-            self.0.send(msg).await
-        } else {
-            Ok(())
-        }
-    }
-
-    fn try_send(&self, msg: U) -> std::result::Result<(), ProgressSendError> {
-        if let Some(msg) = (self.1)(msg) {
-            self.0.try_send(msg)
-        } else {
-            Ok(())
-        }
-    }
-
-    fn blocking_send(&self, msg: U) -> std::result::Result<(), ProgressSendError> {
-        if let Some(msg) = (self.1)(msg) {
-            self.0.blocking_send(msg)
-        } else {
-            Ok(())
-        }
-    }
-}
-
-/// A progress sender that uses an async channel.
-pub struct AsyncChannelProgressSender<T> {
-    sender: async_channel::Sender<T>,
-    id: std::sync::Arc<std::sync::atomic::AtomicU64>,
-}
-
-impl<T> std::fmt::Debug for AsyncChannelProgressSender<T> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("AsyncChannelProgressSender")
-            .field("id", &self.id)
-            .field("sender", &self.sender)
-            .finish()
-    }
-}
-
-impl<T> Clone for AsyncChannelProgressSender<T> {
-    fn clone(&self) -> Self {
-        Self {
-            sender: self.sender.clone(),
-            id: self.id.clone(),
-        }
-    }
-}
-
-impl<T> AsyncChannelProgressSender<T> {
-    /// Create a new progress sender from an async channel sender.
-    pub fn new(sender: async_channel::Sender<T>) -> Self {
-        Self {
-            sender,
-            id: std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)),
-        }
-    }
-
-    /// Returns true if `other` sends on the same `async_channel` channel as `self`.
-    pub fn same_channel(&self, other: &AsyncChannelProgressSender<T>) -> bool {
-        same_channel(&self.sender, &other.sender)
-    }
-}
-
-/// Given a value that is aligned and sized like a pointer, return the value of
-/// the pointer as a usize.
-fn get_as_ptr<T>(value: &T) -> Option<usize> {
-    use std::mem;
-    if mem::size_of::<T>() == std::mem::size_of::<usize>()
-        && mem::align_of::<T>() == mem::align_of::<usize>()
-    {
-        // SAFETY: size and alignment requirements are checked and met
-        unsafe { Some(mem::transmute_copy(value)) }
-    } else {
-        None
-    }
-}
-
-fn same_channel<T>(a: &async_channel::Sender<T>, b: &async_channel::Sender<T>) -> bool {
-    // This relies on async_channel::Sender being just a newtype wrapper around
-    // an Arc<Channel<T>>, so if two senders point to the same channel, the
-    // pointers will be the same.
-    get_as_ptr(a).unwrap() == get_as_ptr(b).unwrap()
-}
-
-impl<T> IdGenerator for AsyncChannelProgressSender<T> {
-    fn new_id(&self) -> u64 {
-        self.id.fetch_add(1, std::sync::atomic::Ordering::SeqCst)
-    }
-}
-
-impl<T: Send + Sync + 'static> ProgressSender for AsyncChannelProgressSender<T> {
-    type Msg = T;
-
-    async fn send(&self, msg: Self::Msg) -> std::result::Result<(), ProgressSendError> {
-        self.sender
-            .send(msg)
-            .await
-            .map_err(|_| ProgressSendError::ReceiverDropped)
-    }
-
-    fn try_send(&self, msg: Self::Msg) -> std::result::Result<(), ProgressSendError> {
-        match self.sender.try_send(msg) {
-            Ok(_) => Ok(()),
-            Err(async_channel::TrySendError::Full(_)) => Ok(()),
-            Err(async_channel::TrySendError::Closed(_)) => Err(ProgressSendError::ReceiverDropped),
-        }
-    }
-
-    fn blocking_send(&self, msg: Self::Msg) -> std::result::Result<(), ProgressSendError> {
-        match self.sender.send_blocking(msg) {
-            Ok(_) => Ok(()),
-            Err(_) => Err(ProgressSendError::ReceiverDropped),
-        }
-    }
-}
-
-/// An error that can occur when sending progress messages.
-///
-/// Really the only error that can occur is if the receiver is dropped.
-#[derive(Debug, Clone, thiserror::Error)]
-pub enum ProgressSendError {
-    /// The receiver was dropped.
-    #[error("receiver dropped")]
-    ReceiverDropped,
-}
-
-/// A result type for progress sending.
-pub type ProgressSendResult<T> = std::result::Result<T, ProgressSendError>;
-
-impl From<ProgressSendError> for std::io::Error {
-    fn from(e: ProgressSendError) -> Self {
-        std::io::Error::new(std::io::ErrorKind::BrokenPipe, e)
-    }
-}
-
-/// A slice writer that adds a synchronous progress callback.
-///
-/// This wraps any `AsyncSliceWriter`, passes through all operations to the inner writer, and
-/// calls the passed `on_write` callback whenever data is written.
-#[derive(Debug)]
-pub struct ProgressSliceWriter<W, F>(W, F);
-
-impl<W: AsyncSliceWriter, F: FnMut(u64)> ProgressSliceWriter<W, F> {
-    /// Create a new `ProgressSliceWriter` from an inner writer and a progress callback
-    ///
-    /// The `on_write` function is called for each write, with the `offset` as the first and the
-    /// length of the data as the second param.
-    pub fn new(inner: W, on_write: F) -> Self {
-        Self(inner, on_write)
-    }
-
-    /// Return the inner writer
-    pub fn into_inner(self) -> W {
-        self.0
-    }
-}
-
-impl<W: AsyncSliceWriter + 'static, F: FnMut(u64, usize) + 'static> AsyncSliceWriter
-    for ProgressSliceWriter<W, F>
-{
-    async fn write_bytes_at(&mut self, offset: u64, data: Bytes) -> io::Result<()> {
-        (self.1)(offset, data.len());
-        self.0.write_bytes_at(offset, data).await
-    }
-
-    async fn write_at(&mut self, offset: u64, data: &[u8]) -> io::Result<()> {
-        (self.1)(offset, data.len());
-        self.0.write_at(offset, data).await
-    }
-
-    async fn sync(&mut self) -> io::Result<()> {
-        self.0.sync().await
-    }
-
-    async fn set_len(&mut self, size: u64) -> io::Result<()> {
-        self.0.set_len(size).await
-    }
-}
-
-/// A slice writer that adds a fallible progress callback.
-///
-/// This wraps any `AsyncSliceWriter`, passes through all operations to the inner writer, and
-/// calls the passed `on_write` callback whenever data is written. `on_write` must return an
-/// `io::Result`, and can abort the download by returning an error.
-#[derive(Debug)]
-pub struct FallibleProgressSliceWriter<W, F>(W, F);
-
-impl<W: AsyncSliceWriter, F: Fn(u64, usize) -> io::Result<()> + 'static>
-    FallibleProgressSliceWriter<W, F>
-{
-    /// Create a new `ProgressSliceWriter` from an inner writer and a progress callback
-    ///
-    /// The `on_write` function is called for each write, with the `offset` as the first and the
-    /// length of the data as the second param. `on_write` must return a future which resolves to
-    /// an `io::Result`. If `on_write` returns an error, the download is aborted.
-    pub fn new(inner: W, on_write: F) -> Self {
-        Self(inner, on_write)
-    }
-
-    /// Return the inner writer.
-    pub fn into_inner(self) -> W {
-        self.0
-    }
-}
-
-impl<W: AsyncSliceWriter + 'static, F: Fn(u64, usize) -> io::Result<()> + 'static> AsyncSliceWriter
-    for FallibleProgressSliceWriter<W, F>
-{
-    async fn write_bytes_at(&mut self, offset: u64, data: Bytes) -> io::Result<()> {
-        (self.1)(offset, data.len())?;
-        self.0.write_bytes_at(offset, data).await
-    }
-
-    async fn write_at(&mut self, offset: u64, data: &[u8]) -> io::Result<()> {
-        (self.1)(offset, data.len())?;
-        self.0.write_at(offset, data).await
-    }
-
-    async fn sync(&mut self) -> io::Result<()> {
-        self.0.sync().await
-    }
-
-    async fn set_len(&mut self, size: u64) -> io::Result<()> {
-        self.0.set_len(size).await
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::sync::Arc;
-
-    use super::*;
-
-    #[test]
-    fn get_as_ptr_works() {
-        struct Wrapper(Arc<u64>);
-        let x = Wrapper(Arc::new(1u64));
-        assert_eq!(
-            get_as_ptr(&x).unwrap(),
-            Arc::as_ptr(&x.0) as usize - 2 * std::mem::size_of::<usize>()
-        );
-    }
-
-    #[test]
-    fn get_as_ptr_wrong_use() {
-        struct Wrapper(#[allow(dead_code)] u8);
-        let x = Wrapper(1);
-        assert!(get_as_ptr(&x).is_none());
-    }
-
-    #[test]
-    fn test_sender_is_ptr() {
-        assert_eq!(
-            std::mem::size_of::<usize>(),
-            std::mem::size_of::<async_channel::Sender<u8>>()
-        );
-        assert_eq!(
-            std::mem::align_of::<usize>(),
-            std::mem::align_of::<async_channel::Sender<u8>>()
-        );
-    }
-}
diff --git a/src/util/temp_tag.rs b/src/util/temp_tag.rs
new file mode 100644
index 000000000..feb333bba
--- /dev/null
+++ b/src/util/temp_tag.rs
@@ -0,0 +1,280 @@
+#![allow(dead_code)]
+use std::{
+    collections::HashMap,
+    sync::{Arc, Mutex, Weak},
+};
+
+use serde::{Deserialize, Serialize};
+use tracing::{trace, warn};
+
+use crate::{api::proto::Scope, BlobFormat, Hash, HashAndFormat};
+
+/// An ephemeral, in-memory tag that protects content while the process is running.
+///
+/// If format is raw, this will protect just the blob
+/// If format is collection, this will protect the collection and all blobs in it
+#[derive(Debug, Serialize, Deserialize)]
+#[must_use = "TempTag is a temporary tag that should be used to protect content while the process is running. \
+       If you want to keep the content alive, use TempTag::leak()"]
+pub struct TempTag {
+    /// The hash and format we are pinning
+    inner: HashAndFormat,
+    /// optional callback to call on drop
+    #[serde(skip)]
+    on_drop: Option<Weak<dyn TagDrop>>,
+}
+
+impl AsRef<Hash> for TempTag {
+    fn as_ref(&self) -> &Hash {
+        &self.inner.hash
+    }
+}
+
+/// A trait for things that can track liveness of blobs and collections.
+///
+/// This trait works together with [TempTag] to keep track of the liveness of a
+/// blob or collection.
+///
+/// It is important to include the format in the liveness tracking, since
+/// protecting a collection means protecting the blob and all its children,
+/// whereas protecting a raw blob only protects the blob itself.
+pub trait TagCounter: TagDrop + Sized {
+    /// Called on creation of a temp tag
+    fn on_create(&self, inner: &HashAndFormat);
+
+    /// Get this as a weak reference for use in temp tags
+    fn as_weak(self: &Arc<Self>) -> Weak<dyn TagDrop> {
+        let on_drop: Arc<dyn TagDrop> = self.clone();
+        Arc::downgrade(&on_drop)
+    }
+
+    /// Create a new temp tag for the given hash and format
+    fn temp_tag(self: &Arc<Self>, inner: HashAndFormat) -> TempTag {
+        self.on_create(&inner);
+        TempTag::new(inner, Some(self.as_weak()))
+    }
+}
+
+/// Trait used from temp tags to notify an abstract store that a temp tag is
+/// being dropped.
+pub trait TagDrop: std::fmt::Debug + Send + Sync + 'static {
+    /// Called on drop
+    fn on_drop(&self, inner: &HashAndFormat);
+}
+
+impl From<&TempTag> for HashAndFormat {
+    fn from(val: &TempTag) -> Self {
+        val.inner
+    }
+}
+
+impl From<TempTag> for HashAndFormat {
+    fn from(val: TempTag) -> Self {
+        val.inner
+    }
+}
+
+impl TempTag {
+    /// Create a new temp tag for the given hash and format
+    ///
+    /// This should only be used by store implementations.
+    ///
+    /// The caller is responsible for increasing the refcount on creation and to
+    /// make sure that temp tags that are created between a mark phase and a sweep
+    /// phase are protected.
+    pub fn new(inner: HashAndFormat, on_drop: Option<Weak<dyn TagDrop>>) -> Self {
+        Self { inner, on_drop }
+    }
+
+    /// The empty temp tag. We don't track the empty blob since we always have it.
+    pub fn leaking_empty(format: BlobFormat) -> Self {
+        Self {
+            inner: HashAndFormat {
+                hash: Hash::EMPTY,
+                format,
+            },
+            on_drop: None,
+        }
+    }
+
+    /// The hash of the pinned item
+    pub fn inner(&self) -> &HashAndFormat {
+        &self.inner
+    }
+
+    /// The hash of the pinned item
+    pub fn hash(&self) -> &Hash {
+        &self.inner.hash
+    }
+
+    /// The format of the pinned item
+    pub fn format(&self) -> BlobFormat {
+        self.inner.format
+    }
+
+    /// The hash and format of the pinned item
+    pub fn hash_and_format(&self) -> &HashAndFormat {
+        &self.inner
+    }
+
+    /// Keep the item alive until the end of the process
+    pub fn leak(&mut self) {
+        // set the liveness tracker to None, so that the refcount is not decreased
+        // during drop. This means that the refcount will never reach 0 and the
+        // item will not be gced until the end of the process.
+        self.on_drop = None;
+    }
+}
+
+impl Drop for TempTag {
+    fn drop(&mut self) {
+        if let Some(on_drop) = self.on_drop.take() {
+            if let Some(on_drop) = on_drop.upgrade() {
+                on_drop.on_drop(&self.inner);
+            }
+        }
+    }
+}
+
+#[derive(Debug, Default, Clone)]
+struct TempCounters {
+    /// number of raw temp tags for a hash
+    raw: u64,
+    /// number of hash seq temp tags for a hash
+    hash_seq: u64,
+}
+
+impl TempCounters {
+    fn counter(&self, format: BlobFormat) -> u64 {
+        match format {
+            BlobFormat::Raw => self.raw,
+            BlobFormat::HashSeq => self.hash_seq,
+        }
+    }
+
+    fn counter_mut(&mut self, format: BlobFormat) -> &mut u64 {
+        match format {
+            BlobFormat::Raw => &mut self.raw,
+            BlobFormat::HashSeq => &mut self.hash_seq,
+        }
+    }
+
+    fn inc(&mut self, format: BlobFormat) {
+        let counter = self.counter_mut(format);
+        *counter = counter.checked_add(1).unwrap();
+    }
+
+    fn dec(&mut self, format: BlobFormat) {
+        let counter = self.counter_mut(format);
+        *counter = counter.saturating_sub(1);
+    }
+
+    fn is_empty(&self) -> bool {
+        self.raw == 0 && self.hash_seq == 0
+    }
+}
+
+#[derive(Debug, Default)]
+pub(crate) struct TempTags {
+    scopes: HashMap<Scope, Arc<TempTagScope>>,
+    next_scope: u64,
+}
+
+impl TempTags {
+    pub fn create_scope(&mut self) -> (Scope, Arc<TempTagScope>) {
+        self.next_scope += 1;
+        let id = Scope(self.next_scope);
+        let scope = self.scopes.entry(id).or_default();
+        (id, scope.clone())
+    }
+
+    pub fn end_scope(&mut self, scope: Scope) {
+        self.scopes.remove(&scope);
+    }
+
+    pub fn list(&self) -> Vec<HashAndFormat> {
+        self.scopes
+            .values()
+            .flat_map(|scope| scope.list())
+            .collect()
+    }
+
+    pub fn create(&mut self, scope: Scope, content: HashAndFormat) -> TempTag {
+        let scope = self.scopes.entry(scope).or_default();
+
+        scope.temp_tag(content)
+    }
+
+    pub fn contains(&self, hash: Hash) -> bool {
+        self.scopes
+            .values()
+            .any(|scope| scope.0.lock().unwrap().contains(&HashAndFormat::raw(hash)))
+    }
+}
+
+#[derive(Debug, Default)]
+pub(crate) struct TempTagScope(Mutex<TempCounterMap>);
+
+impl TempTagScope {
+    pub fn list(&self) -> impl Iterator<Item = HashAndFormat> + 'static {
+        let guard = self.0.lock().unwrap();
+        let res = guard.keys();
+        drop(guard);
+        res.into_iter()
+    }
+}
+
+impl TagDrop for TempTagScope {
+    fn on_drop(&self, inner: &HashAndFormat) {
+        trace!("Dropping temp tag {:?}", inner);
+        self.0.lock().unwrap().dec(inner);
+    }
+}
+
+impl TagCounter for TempTagScope {
+    fn on_create(&self, inner: &HashAndFormat) {
+        trace!("Creating temp tag {:?}", inner);
+        self.0.lock().unwrap().inc(*inner);
+    }
+}
+
+#[derive(Debug, Clone, Default)]
+pub(crate) struct TempCounterMap(HashMap<Hash, TempCounters>);
+
+impl TempCounterMap {
+    pub fn inc(&mut self, value: HashAndFormat) {
+        self.0.entry(value.hash).or_default().inc(value.format)
+    }
+
+    pub fn dec(&mut self, value: &HashAndFormat) {
+        let HashAndFormat { hash, format } = value;
+        let Some(counters) = self.0.get_mut(hash) else {
+            warn!("Decrementing non-existent temp tag");
+            return;
+        };
+        counters.dec(*format);
+        if counters.is_empty() {
+            self.0.remove(hash);
+        }
+    }
+
+    pub fn contains(&self, haf: &HashAndFormat) -> bool {
+        let Some(entry) = self.0.get(&haf.hash) else {
+            return false;
+        };
+        entry.counter(haf.format) > 0
+    }
+
+    pub fn keys(&self) -> Vec<HashAndFormat> {
+        let mut res = Vec::new();
+        for (k, v) in self.0.iter() {
+            if v.raw > 0 {
+                res.push(HashAndFormat::raw(*k));
+            }
+            if v.hash_seq > 0 {
+                res.push(HashAndFormat::hash_seq(*k));
+            }
+        }
+        res
+    }
+}
diff --git a/tests/blobs.rs b/tests/blobs.rs
index 6779ebe9a..dcb8118dc 100644
--- a/tests/blobs.rs
+++ b/tests/blobs.rs
@@ -1,65 +1,116 @@
-#![cfg(all(feature = "net_protocol", feature = "rpc"))]
 use std::{
-    sync::{Arc, Mutex},
-    time::Duration,
+    net::{Ipv4Addr, SocketAddr, SocketAddrV4},
+    ops::Deref,
+    path::Path,
 };
 
-use iroh::Endpoint;
-use iroh_blobs::{net_protocol::Blobs, store::GcConfig};
+use iroh_blobs::{
+    api::{
+        blobs::{AddProgressItem, Blobs},
+        Store,
+    },
+    store::{fs::FsStore, mem::MemStore},
+    Hash,
+};
+use n0_future::StreamExt;
 use testresult::TestResult;
 
+/// Interesting sizes for testing.
+pub const INTERESTING_SIZES: [usize; 8] = [
+    0,               // annoying corner case - always present, handled by the api
+    1,               // less than 1 chunk, data inline, outboard not needed
+    1024,            // exactly 1 chunk, data inline, outboard not needed
+    1024 * 16 - 1,   // less than 1 chunk group, data inline, outboard not needed
+    1024 * 16,       // exactly 1 chunk group, data inline, outboard not needed
+    1024 * 16 + 1,   // data file, outboard inline (just 1 hash pair)
+    1024 * 1024,     // data file, outboard inline (many hash pairs)
+    1024 * 1024 * 8, // data file, outboard file
+];
+
+async fn blobs_smoke(path: &Path, blobs: &Blobs) -> TestResult<()> {
+    // test importing and exporting bytes
+    {
+        let expected = b"hello".to_vec();
+        let expected_hash = Hash::new(&expected);
+        let tt = blobs.add_bytes(expected.clone()).await?;
+        let hash = tt.hash;
+        assert_eq!(hash, expected_hash);
+        let actual = blobs.get_bytes(hash).await?;
+        assert_eq!(actual, expected);
+    }
+
+    // test importing and exporting a file
+    {
+        let expected = b"somestuffinafile".to_vec();
+        let temp1 = path.join("test1");
+        std::fs::write(&temp1, &expected)?;
+        let tt = blobs.add_path(temp1).await?;
+        let hash = tt.hash;
+        let expected_hash = Hash::new(&expected);
+        assert_eq!(hash, expected_hash);
+
+        let temp2 = path.join("test2");
+        blobs.export(hash, &temp2).await?;
+        let actual = std::fs::read(&temp2)?;
+        assert_eq!(actual, expected);
+    }
+
+    // test importing a large file with progress
+    {
+        let expected = vec![0u8; 1024 * 1024];
+        let temp1 = path.join("test3");
+        std::fs::write(&temp1, &expected)?;
+        let mut stream = blobs.add_path(temp1).stream().await;
+        let mut res = None;
+        while let Some(item) = stream.next().await {
+            if let AddProgressItem::Done(tt) = item {
+                res = Some(tt);
+                break;
+            }
+        }
+        let actual_hash = res.as_ref().map(|x| *x.hash());
+        let expected_hash = Hash::new(&expected);
+        assert_eq!(actual_hash, Some(expected_hash));
+    }
+
+    {
+        let hashes = blobs.list().hashes().await?;
+        assert_eq!(hashes.len(), 3);
+    }
+    Ok(())
+}
+
 #[tokio::test]
-async fn blobs_gc_smoke() -> TestResult<()> {
-    let endpoint = Endpoint::builder().bind().await?;
-    let blobs = Blobs::memory().build(&endpoint);
-    let client = blobs.client();
-    blobs.start_gc(GcConfig {
-        period: Duration::from_millis(1),
-        done_callback: None,
-    })?;
-    let h1 = client.add_bytes(b"test".to_vec()).await?;
-    tokio::time::sleep(Duration::from_millis(100)).await;
-    assert!(client.has(h1.hash).await?);
-    client.tags().delete(h1.tag).await?;
-    tokio::time::sleep(Duration::from_millis(100)).await;
-    assert!(!client.has(h1.hash).await?);
+async fn blobs_smoke_fs() -> TestResult {
+    tracing_subscriber::fmt::try_init().ok();
+    let td = tempfile::tempdir()?;
+    let store = FsStore::load(td.path().join("a")).await?;
+    blobs_smoke(td.path(), store.blobs()).await?;
+    store.shutdown().await?;
     Ok(())
 }
 
 #[tokio::test]
-async fn blobs_gc_protected() -> TestResult<()> {
-    let endpoint = Endpoint::builder().bind().await?;
-    let blobs = Blobs::memory().build(&endpoint);
-    let client = blobs.client();
-    let h1 = client.add_bytes(b"test".to_vec()).await?;
-    let protected = Arc::new(Mutex::new(Vec::new()));
-    blobs.add_protected(Box::new({
-        let protected = protected.clone();
-        move |x| {
-            let protected = protected.clone();
-            Box::pin(async move {
-                let protected = protected.lock().unwrap();
-                for h in protected.as_slice() {
-                    x.insert(*h);
-                }
-            })
-        }
-    }))?;
-    blobs.start_gc(GcConfig {
-        period: Duration::from_millis(1),
-        done_callback: None,
-    })?;
-    tokio::time::sleep(Duration::from_millis(100)).await;
-    // protected from gc due to tag
-    assert!(client.has(h1.hash).await?);
-    protected.lock().unwrap().push(h1.hash);
-    client.tags().delete(h1.tag).await?;
-    tokio::time::sleep(Duration::from_millis(100)).await;
-    // protected from gc due to being in protected set
-    assert!(client.has(h1.hash).await?);
-    protected.lock().unwrap().clear();
-    tokio::time::sleep(Duration::from_millis(100)).await;
-    // not protected, must be gone
-    assert!(!client.has(h1.hash).await?);
+async fn blobs_smoke_mem() -> TestResult {
+    tracing_subscriber::fmt::try_init().ok();
+    let td = tempfile::tempdir()?;
+    let store = MemStore::new();
+    blobs_smoke(td.path(), store.blobs()).await?;
+    store.shutdown().await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn blobs_smoke_fs_rpc() -> TestResult {
+    tracing_subscriber::fmt::try_init().ok();
+    let unspecified = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, 0));
+    let (server, cert) = irpc::util::make_server_endpoint(unspecified)?;
+    let client = irpc::util::make_client_endpoint(unspecified, &[cert.as_ref()])?;
+    let td = tempfile::tempdir()?;
+    let store = FsStore::load(td.path().join("a")).await?;
+    tokio::spawn(store.deref().clone().listen(server.clone()));
+    let api = Store::connect(client, server.local_addr()?);
+    blobs_smoke(td.path(), api.blobs()).await?;
+    api.shutdown().await?;
     Ok(())
 }
diff --git a/tests/gc.rs b/tests/gc.rs
deleted file mode 100644
index 24404ad18..000000000
--- a/tests/gc.rs
+++ /dev/null
@@ -1,526 +0,0 @@
-#![cfg(feature = "rpc")]
-use std::{
-    io,
-    io::{Cursor, Write},
-    path::PathBuf,
-    time::Duration,
-};
-
-use anyhow::Result;
-use bao_tree::{
-    blake3,
-    io::{
-        fsm::{BaoContentItem, ResponseDecoderNext},
-        sync::Outboard,
-    },
-    BaoTree, ChunkRanges,
-};
-use bytes::Bytes;
-use iroh::{protocol::Router, Endpoint, NodeAddr, NodeId};
-use iroh_blobs::{
-    hashseq::HashSeq,
-    net_protocol::Blobs,
-    rpc::client::{blobs, tags},
-    store::{
-        bao_tree, BaoBatchWriter, ConsistencyCheckProgress, EntryStatus, GcConfig, MapEntryMut,
-        MapMut, ReportLevel, Store,
-    },
-    util::{
-        progress::{AsyncChannelProgressSender, ProgressSender as _},
-        Tag,
-    },
-    BlobFormat, HashAndFormat, TempTag, IROH_BLOCK_SIZE,
-};
-use rand::RngCore;
-use testdir::testdir;
-use tokio::io::AsyncReadExt;
-
-/// An iroh node that just has the blobs transport
-#[derive(Debug)]
-pub struct Node<S> {
-    pub router: iroh::protocol::Router,
-    pub blobs: Blobs<S>,
-    pub store: S,
-}
-
-impl<S: Store> Node<S> {
-    pub fn blob_store(&self) -> &S {
-        &self.store
-    }
-
-    /// Returns the node id
-    pub fn node_id(&self) -> NodeId {
-        self.router.endpoint().node_id()
-    }
-
-    /// Returns the node address
-    pub async fn node_addr(&self) -> anyhow::Result<NodeAddr> {
-        self.router.endpoint().node_addr().await
-    }
-
-    /// Shuts down the node
-    pub async fn shutdown(self) -> anyhow::Result<()> {
-        self.router.shutdown().await
-    }
-
-    /// Returns an in-memory blobs client
-    pub fn blobs(&self) -> &blobs::MemClient {
-        self.blobs.client()
-    }
-
-    /// Returns an in-memory tags client
-    pub fn tags(&self) -> tags::MemClient {
-        self.blobs().tags()
-    }
-}
-
-pub fn create_test_data(size: usize) -> Bytes {
-    let mut rand = rand::thread_rng();
-    let mut res = vec![0u8; size];
-    rand.fill_bytes(&mut res);
-    res.into()
-}
-
-/// Take some data and encode it
-pub fn simulate_remote(data: &[u8]) -> (blake3::Hash, Cursor<Bytes>) {
-    let outboard = bao_tree::io::outboard::PostOrderMemOutboard::create(data, IROH_BLOCK_SIZE);
-    let mut encoded = Vec::new();
-    encoded
-        .write_all(outboard.tree.size().to_le_bytes().as_ref())
-        .unwrap();
-    bao_tree::io::sync::encode_ranges_validated(data, &outboard, &ChunkRanges::all(), &mut encoded)
-        .unwrap();
-    let hash = outboard.root();
-    (hash, Cursor::new(encoded.into()))
-}
-
-/// Wrap a bao store in a node that has gc enabled.
-async fn node<S: Store>(store: S, gc_period: Duration) -> (Node<S>, async_channel::Receiver<()>) {
-    let (gc_send, gc_recv) = async_channel::unbounded();
-    let endpoint = Endpoint::builder().discovery_n0().bind().await.unwrap();
-    let blobs = Blobs::builder(store.clone()).build(&endpoint);
-    let router = Router::builder(endpoint)
-        .accept(iroh_blobs::ALPN, blobs.clone())
-        .spawn();
-    blobs
-        .start_gc(GcConfig {
-            period: gc_period,
-            done_callback: Some(Box::new(move || {
-                gc_send.send_blocking(()).ok();
-            })),
-        })
-        .unwrap();
-    (
-        Node {
-            store,
-            router,
-            blobs,
-        },
-        gc_recv,
-    )
-}
-
-/// Wrap a bao store in a node that has gc enabled.
-async fn persistent_node(
-    path: PathBuf,
-    gc_period: Duration,
-) -> (
-    Node<iroh_blobs::store::fs::Store>,
-    async_channel::Receiver<()>,
-) {
-    let store = iroh_blobs::store::fs::Store::load(path).await.unwrap();
-    node(store, gc_period).await
-}
-
-async fn mem_node(
-    gc_period: Duration,
-) -> (
-    Node<iroh_blobs::store::mem::Store>,
-    async_channel::Receiver<()>,
-) {
-    let store = iroh_blobs::store::mem::Store::new();
-    node(store, gc_period).await
-}
-
-async fn gc_test_node() -> (
-    Node<iroh_blobs::store::mem::Store>,
-    iroh_blobs::store::mem::Store,
-    async_channel::Receiver<()>,
-) {
-    let (node, gc_recv) = mem_node(Duration::from_millis(500)).await;
-    let store = node.blob_store().clone();
-    (node, store, gc_recv)
-}
-
-async fn step(evs: &async_channel::Receiver<()>) {
-    // drain the event queue, we want a new GC
-    while evs.try_recv().is_ok() {}
-    // wait for several GC cycles
-    for _ in 0..3 {
-        evs.recv().await.unwrap();
-    }
-}
-
-/// Test the absolute basics of gc, temp tags and tags for blobs.
-#[tokio::test]
-async fn gc_basics() -> Result<()> {
-    let _ = tracing_subscriber::fmt::try_init();
-    let (node, bao_store, evs) = gc_test_node().await;
-    let data1 = create_test_data(1234);
-    let tt1 = bao_store.import_bytes(data1, BlobFormat::Raw).await?;
-    let data2 = create_test_data(5678);
-    let tt2 = bao_store.import_bytes(data2, BlobFormat::Raw).await?;
-    let h1 = *tt1.hash();
-    let h2 = *tt2.hash();
-    // temp tags are still there, so the entries should be there
-    step(&evs).await;
-    assert_eq!(bao_store.entry_status(&h1).await?, EntryStatus::Complete);
-    assert_eq!(bao_store.entry_status(&h2).await?, EntryStatus::Complete);
-
-    // drop the first tag, the entry should be gone after some time
-    drop(tt1);
-    step(&evs).await;
-    assert_eq!(bao_store.entry_status(&h1).await?, EntryStatus::NotFound);
-    assert_eq!(bao_store.entry_status(&h2).await?, EntryStatus::Complete);
-
-    // create an explicit tag for h1 (as raw) and then delete the temp tag. Entry should still be there.
-    let tag = Tag::from("test");
-    bao_store
-        .set_tag(tag.clone(), HashAndFormat::raw(h2))
-        .await?;
-    drop(tt2);
-    tracing::info!("dropped tt2");
-    step(&evs).await;
-    assert_eq!(bao_store.entry_status(&h2).await?, EntryStatus::Complete);
-
-    // delete the explicit tag, entry should be gone
-    bao_store.delete_tag(tag).await?;
-    step(&evs).await;
-    assert_eq!(bao_store.entry_status(&h2).await?, EntryStatus::NotFound);
-
-    node.shutdown().await?;
-    Ok(())
-}
-
-/// Test gc for sequences of hashes that protect their children from deletion.
-#[tokio::test]
-async fn gc_hashseq_impl() -> Result<()> {
-    let _ = tracing_subscriber::fmt::try_init();
-    let (node, bao_store, evs) = gc_test_node().await;
-    let data1 = create_test_data(1234);
-    let tt1 = bao_store.import_bytes(data1, BlobFormat::Raw).await?;
-    let data2 = create_test_data(5678);
-    let tt2 = bao_store.import_bytes(data2, BlobFormat::Raw).await?;
-    let seq = vec![*tt1.hash(), *tt2.hash()]
-        .into_iter()
-        .collect::<HashSeq>();
-    let ttr = bao_store
-        .import_bytes(seq.into_inner(), BlobFormat::HashSeq)
-        .await?;
-    let h1 = *tt1.hash();
-    let h2 = *tt2.hash();
-    let hr = *ttr.hash();
-    drop(tt1);
-    drop(tt2);
-
-    // there is a temp tag for the link seq, so it and its entries should be there
-    step(&evs).await;
-    assert_eq!(bao_store.entry_status(&h1).await?, EntryStatus::Complete);
-    assert_eq!(bao_store.entry_status(&h2).await?, EntryStatus::Complete);
-    assert_eq!(bao_store.entry_status(&hr).await?, EntryStatus::Complete);
-
-    // make a permanent tag for the link seq, then delete the temp tag. Entries should still be there.
-    let tag = Tag::from("test");
-    bao_store
-        .set_tag(tag.clone(), HashAndFormat::hash_seq(hr))
-        .await?;
-    drop(ttr);
-    step(&evs).await;
-    assert_eq!(bao_store.entry_status(&h1).await?, EntryStatus::Complete);
-    assert_eq!(bao_store.entry_status(&h2).await?, EntryStatus::Complete);
-    assert_eq!(bao_store.entry_status(&hr).await?, EntryStatus::Complete);
-
-    // change the permanent tag to be just for the linkseq itself as a blob. Only the linkseq should be there, not the entries.
-    bao_store
-        .set_tag(tag.clone(), HashAndFormat::raw(hr))
-        .await?;
-    step(&evs).await;
-    assert_eq!(bao_store.entry_status(&h1).await?, EntryStatus::NotFound);
-    assert_eq!(bao_store.entry_status(&h2).await?, EntryStatus::NotFound);
-    assert_eq!(bao_store.entry_status(&hr).await?, EntryStatus::Complete);
-
-    // delete the permanent tag, everything should be gone
-    bao_store.delete_tag(tag).await?;
-    step(&evs).await;
-    assert_eq!(bao_store.entry_status(&h1).await?, EntryStatus::NotFound);
-    assert_eq!(bao_store.entry_status(&h2).await?, EntryStatus::NotFound);
-    assert_eq!(bao_store.entry_status(&hr).await?, EntryStatus::NotFound);
-
-    node.shutdown().await?;
-    Ok(())
-}
-
-fn path(root: PathBuf, suffix: &'static str) -> impl Fn(&iroh_blobs::Hash) -> PathBuf {
-    move |hash| root.join(format!("{}.{}", hash.to_hex(), suffix))
-}
-
-fn data_path(root: PathBuf) -> impl Fn(&iroh_blobs::Hash) -> PathBuf {
-    // this assumes knowledge of the internal directory structure of the flat store
-    path(root.join("data"), "data")
-}
-
-fn outboard_path(root: PathBuf) -> impl Fn(&iroh_blobs::Hash) -> PathBuf {
-    // this assumes knowledge of the internal directory structure of the flat store
-    path(root.join("data"), "obao4")
-}
-
-async fn check_consistency(store: &impl Store) -> anyhow::Result<ReportLevel> {
-    let mut max_level = ReportLevel::Trace;
-    let (tx, rx) = async_channel::bounded(1);
-    let task = tokio::task::spawn(async move {
-        while let Ok(ev) = rx.recv().await {
-            if let ConsistencyCheckProgress::Update { level, .. } = &ev {
-                max_level = max_level.max(*level);
-            }
-        }
-    });
-    store
-        .consistency_check(false, AsyncChannelProgressSender::new(tx).boxed())
-        .await?;
-    task.await?;
-    Ok(max_level)
-}
-
-/// Test gc for sequences of hashes that protect their children from deletion.
-#[tokio::test]
-async fn gc_file_basics() -> Result<()> {
-    let _ = tracing_subscriber::fmt::try_init();
-    let dir = testdir!();
-    let path = data_path(dir.clone());
-    let outboard_path = outboard_path(dir.clone());
-    let (node, evs) = persistent_node(dir.clone(), Duration::from_millis(100)).await;
-    let bao_store = node.blob_store().clone();
-    let data1 = create_test_data(10000000);
-    let tt1 = bao_store
-        .import_bytes(data1.clone(), BlobFormat::Raw)
-        .await?;
-    let data2 = create_test_data(1000000);
-    let tt2 = bao_store
-        .import_bytes(data2.clone(), BlobFormat::Raw)
-        .await?;
-    let seq = vec![*tt1.hash(), *tt2.hash()]
-        .into_iter()
-        .collect::<HashSeq>();
-    let ttr = bao_store
-        .import_bytes(seq.into_inner(), BlobFormat::HashSeq)
-        .await?;
-
-    let h1 = *tt1.hash();
-    let h2 = *tt2.hash();
-    let hr = *ttr.hash();
-
-    // data is protected by the temp tag
-    step(&evs).await;
-    bao_store.sync().await?;
-    assert!(check_consistency(&bao_store).await? <= ReportLevel::Info);
-    // h1 is for a giant file, so we will have both data and outboard files
-    assert!(path(&h1).exists());
-    assert!(outboard_path(&h1).exists());
-    // h2 is for a mid sized file, so we will have just the data file
-    assert!(path(&h2).exists());
-    assert!(!outboard_path(&h2).exists());
-    // hr so small that data will be inlined and outboard will not exist at all
-    assert!(!path(&hr).exists());
-    assert!(!outboard_path(&hr).exists());
-
-    drop(tt1);
-    drop(tt2);
-    let tag = Tag::from("test");
-    bao_store
-        .set_tag(tag.clone(), HashAndFormat::hash_seq(*ttr.hash()))
-        .await?;
-    drop(ttr);
-
-    // data is now protected by a normal tag, nothing should be gone
-    step(&evs).await;
-    bao_store.sync().await?;
-    assert!(check_consistency(&bao_store).await? <= ReportLevel::Info);
-    // h1 is for a giant file, so we will have both data and outboard files
-    assert!(path(&h1).exists());
-    assert!(outboard_path(&h1).exists());
-    // h2 is for a mid sized file, so we will have just the data file
-    assert!(path(&h2).exists());
-    assert!(!outboard_path(&h2).exists());
-    // hr so small that data will be inlined and outboard will not exist at all
-    assert!(!path(&hr).exists());
-    assert!(!outboard_path(&hr).exists());
-
-    tracing::info!("changing tag from hashseq to raw, this should orphan the children");
-    bao_store
-        .set_tag(tag.clone(), HashAndFormat::raw(hr))
-        .await?;
-
-    // now only hr itself should be protected, but not its children
-    step(&evs).await;
-    bao_store.sync().await?;
-    assert!(check_consistency(&bao_store).await? <= ReportLevel::Info);
-    // h1 should be gone
-    assert!(!path(&h1).exists());
-    assert!(!outboard_path(&h1).exists());
-    // h2 should still not be there
-    assert!(!path(&h2).exists());
-    assert!(!outboard_path(&h2).exists());
-    // hr should still not be there
-    assert!(!path(&hr).exists());
-    assert!(!outboard_path(&hr).exists());
-
-    bao_store.delete_tag(tag).await?;
-    step(&evs).await;
-    bao_store.sync().await?;
-    assert!(check_consistency(&bao_store).await? <= ReportLevel::Info);
-    // h1 should be gone
-    assert!(!path(&h1).exists());
-    assert!(!outboard_path(&h1).exists());
-    // h2 should still not be there
-    assert!(!path(&h2).exists());
-    assert!(!outboard_path(&h2).exists());
-    // hr should still not be there
-    assert!(!path(&hr).exists());
-    assert!(!outboard_path(&hr).exists());
-
-    node.shutdown().await?;
-
-    Ok(())
-}
-
-/// Add a file to the store in the same way a download works.
-///
-/// we know the hash in advance, create a partial entry, write the data to it and
-/// the outboard file, then commit it to a complete entry.
-///
-/// During this time, the partial entry is protected by a temp tag.
-async fn simulate_download_partial<S: iroh_blobs::store::Store>(
-    bao_store: &S,
-    data: Bytes,
-) -> io::Result<(S::EntryMut, TempTag)> {
-    // simulate the remote side.
-    let (hash, mut response) = simulate_remote(data.as_ref());
-    // simulate the local side.
-    // we got a hash and a response from the remote side.
-    let tt = bao_store.temp_tag(HashAndFormat::raw(hash.into()));
-    // get the size
-    let size = response.read_u64_le().await?;
-    // start reading the response
-    let mut reading = bao_tree::io::fsm::ResponseDecoder::new(
-        hash,
-        ChunkRanges::all(),
-        BaoTree::new(size, IROH_BLOCK_SIZE),
-        response,
-    );
-    // create the partial entry
-    let entry = bao_store.get_or_create(hash.into(), size).await?;
-    // create the
-    let mut bw = entry.batch_writer().await?;
-    let mut buf = Vec::new();
-    while let ResponseDecoderNext::More((next, res)) = reading.next().await {
-        let item = res?;
-        match &item {
-            BaoContentItem::Parent(_) => {
-                buf.push(item);
-            }
-            BaoContentItem::Leaf(_) => {
-                buf.push(item);
-                let batch = std::mem::take(&mut buf);
-                bw.write_batch(size, batch).await?;
-            }
-        }
-        reading = next;
-    }
-    bw.sync().await?;
-    drop(bw);
-    Ok((entry, tt))
-}
-
-async fn simulate_download_complete<S: iroh_blobs::store::Store>(
-    bao_store: &S,
-    data: Bytes,
-) -> io::Result<TempTag> {
-    let (entry, tt) = simulate_download_partial(bao_store, data).await?;
-    // commit the entry
-    bao_store.insert_complete(entry).await?;
-    Ok(tt)
-}
-
-/// Test that partial files are deleted.
-#[tokio::test]
-async fn gc_file_partial() -> Result<()> {
-    let _ = tracing_subscriber::fmt::try_init();
-    let dir = testdir!();
-    let path = data_path(dir.clone());
-    let outboard_path = outboard_path(dir.clone());
-
-    let (node, evs) = persistent_node(dir.clone(), Duration::from_millis(10)).await;
-    let bao_store = node.blob_store().clone();
-
-    let data1: Bytes = create_test_data(10000000);
-    let (_entry, tt1) = simulate_download_partial(&bao_store, data1.clone()).await?;
-    drop(_entry);
-    let h1 = *tt1.hash();
-    // partial data and outboard files should be there
-    step(&evs).await;
-    bao_store.sync().await?;
-    assert!(check_consistency(&bao_store).await? <= ReportLevel::Info);
-    assert!(path(&h1).exists());
-    assert!(outboard_path(&h1).exists());
-
-    drop(tt1);
-    // partial data and outboard files should be gone
-    step(&evs).await;
-    bao_store.sync().await?;
-    assert!(check_consistency(&bao_store).await? <= ReportLevel::Info);
-    assert!(!path(&h1).exists());
-    assert!(!outboard_path(&h1).exists());
-
-    node.shutdown().await?;
-    Ok(())
-}
-
-#[tokio::test]
-async fn gc_file_stress() -> Result<()> {
-    let _ = tracing_subscriber::fmt::try_init();
-    let dir = testdir!();
-
-    let (node, evs) = persistent_node(dir.clone(), Duration::from_secs(1)).await;
-    let bao_store = node.blob_store().clone();
-
-    let mut deleted = Vec::new();
-    let mut live = Vec::new();
-    // download
-    for i in 0..100 {
-        let data: Bytes = create_test_data(16 * 1024 * 3 + 1);
-        let tt = simulate_download_complete(&bao_store, data).await.unwrap();
-        if i % 100 == 0 {
-            let tag = Tag::from(format!("test{}", i));
-            bao_store
-                .set_tag(tag.clone(), HashAndFormat::raw(*tt.hash()))
-                .await?;
-            live.push(*tt.hash());
-        } else {
-            deleted.push(*tt.hash());
-        }
-    }
-    step(&evs).await;
-
-    for h in deleted.iter() {
-        assert_eq!(bao_store.entry_status(h).await?, EntryStatus::NotFound);
-        assert!(!dir.join(format!("data/{}.data", h.to_hex())).exists());
-    }
-
-    for h in live.iter() {
-        assert_eq!(bao_store.entry_status(h).await?, EntryStatus::Complete);
-        assert!(dir.join(format!("data/{}.data", h.to_hex())).exists());
-    }
-
-    node.shutdown().await?;
-    Ok(())
-}
diff --git a/tests/rpc.rs b/tests/rpc.rs
deleted file mode 100644
index d8addc010..000000000
--- a/tests/rpc.rs
+++ /dev/null
@@ -1,111 +0,0 @@
-#![cfg(feature = "test")]
-use std::{net::SocketAddr, path::PathBuf, vec};
-
-use iroh_blobs::{downloader, net_protocol::Blobs};
-use quic_rpc::client::QuinnConnector;
-use tempfile::TempDir;
-use testresult::TestResult;
-use tokio_util::task::AbortOnDropHandle;
-
-type QC = QuinnConnector<iroh_blobs::rpc::proto::RpcService>;
-type BlobsClient = iroh_blobs::rpc::client::blobs::Client<QC>;
-
-/// An iroh node that just has the blobs transport
-#[derive(Debug)]
-pub struct Node {
-    pub router: iroh::protocol::Router,
-    pub blobs: Blobs<iroh_blobs::store::fs::Store>,
-    pub rpc_task: AbortOnDropHandle<()>,
-}
-
-impl Node {
-    pub async fn new(path: PathBuf) -> anyhow::Result<(Self, SocketAddr, Vec<u8>)> {
-        let store = iroh_blobs::store::fs::Store::load(path).await?;
-        let endpoint = iroh::Endpoint::builder().bind().await?;
-        let blobs = Blobs::builder(store).build(&endpoint);
-        let router = iroh::protocol::Router::builder(endpoint)
-            .accept(iroh_blobs::ALPN, blobs.clone())
-            .spawn();
-        let (config, key) = quic_rpc::transport::quinn::configure_server()?;
-        let endpoint = quinn::Endpoint::server(config, "127.0.0.1:0".parse().unwrap())?;
-        let local_addr = endpoint.local_addr()?;
-        let rpc_server = quic_rpc::transport::quinn::QuinnListener::new(endpoint)?;
-        let rpc_server =
-            quic_rpc::RpcServer::<iroh_blobs::rpc::proto::RpcService, _>::new(rpc_server);
-        let blobs2 = blobs.clone();
-        let rpc_task = rpc_server
-            .spawn_accept_loop(move |msg, chan| blobs2.clone().handle_rpc_request(msg, chan));
-        let node = Self {
-            router,
-            blobs,
-            rpc_task,
-        };
-        Ok((node, local_addr, key))
-    }
-}
-
-async fn node_and_client() -> TestResult<(Node, BlobsClient, TempDir)> {
-    let testdir = tempfile::tempdir()?;
-    let (node, addr, key) = Node::new(testdir.path().join("blobs")).await?;
-    let client = quic_rpc::transport::quinn::make_client_endpoint(
-        "127.0.0.1:0".parse().unwrap(),
-        &[key.as_slice()],
-    )?;
-    let client = QuinnConnector::<iroh_blobs::rpc::proto::RpcService>::new(
-        client,
-        addr,
-        "localhost".to_string(),
-    );
-    let client = quic_rpc::RpcClient::<iroh_blobs::rpc::proto::RpcService, _>::new(client);
-    let client = iroh_blobs::rpc::client::blobs::Client::new(client);
-    Ok((node, client, testdir))
-}
-
-#[tokio::test]
-async fn quinn_rpc_smoke() -> TestResult<()> {
-    let _ = tracing_subscriber::fmt::try_init();
-    let (_node, client, _testdir) = node_and_client().await?;
-    let data = b"hello";
-    let hash = client.add_bytes(data.to_vec()).await?.hash;
-    assert_eq!(hash, iroh_blobs::Hash::new(data));
-    let data2 = client.read_to_bytes(hash).await?;
-    assert_eq!(data, &data2[..]);
-    Ok(())
-}
-
-#[tokio::test]
-async fn quinn_rpc_large() -> TestResult<()> {
-    let _ = tracing_subscriber::fmt::try_init();
-    let (_node, client, _testdir) = node_and_client().await?;
-    let data = vec![0; 1024 * 1024 * 16];
-    let hash = client.add_bytes(data.clone()).await?.hash;
-    assert_eq!(hash, iroh_blobs::Hash::new(&data));
-    let data2 = client.read_to_bytes(hash).await?;
-    assert_eq!(data, &data2[..]);
-    Ok(())
-}
-
-#[tokio::test]
-async fn downloader_config() -> TestResult<()> {
-    let _ = tracing_subscriber::fmt::try_init();
-    let endpoint = iroh::Endpoint::builder().bind().await?;
-    let store = iroh_blobs::store::mem::Store::default();
-    let expected = downloader::Config {
-        concurrency: downloader::ConcurrencyLimits {
-            max_concurrent_requests: usize::MAX,
-            max_concurrent_requests_per_node: usize::MAX,
-            max_open_connections: usize::MAX,
-            max_concurrent_dials_per_hash: usize::MAX,
-        },
-        retry: downloader::RetryConfig {
-            max_retries_per_node: u32::MAX,
-            initial_retry_delay: std::time::Duration::from_secs(1),
-        },
-    };
-    let blobs = Blobs::builder(store)
-        .downloader_config(expected)
-        .build(&endpoint);
-    let actual = blobs.downloader().config();
-    assert_eq!(&expected, actual);
-    Ok(())
-}
diff --git a/tests/tags.rs b/tests/tags.rs
index 8a4af2d54..3864bc545 100644
--- a/tests/tags.rs
+++ b/tests/tags.rs
@@ -1,20 +1,23 @@
-#![cfg(all(feature = "net_protocol", feature = "rpc"))]
-use futures_lite::StreamExt;
-use futures_util::Stream;
-use iroh::Endpoint;
+use std::{
+    net::{Ipv4Addr, SocketAddr, SocketAddrV4},
+    ops::Deref,
+};
+
 use iroh_blobs::{
-    net_protocol::Blobs,
-    rpc::{
-        client::tags::{self, TagInfo},
-        proto::RpcService,
+    api::{
+        self,
+        tags::{TagInfo, Tags},
+        Store,
     },
-    Hash, HashAndFormat,
+    store::{fs::FsStore, mem::MemStore},
+    BlobFormat, Hash, HashAndFormat,
 };
+use n0_future::{Stream, StreamExt};
 use testresult::TestResult;
 
-async fn to_vec<T>(stream: impl Stream<Item = anyhow::Result<T>>) -> anyhow::Result<Vec<T>> {
+async fn to_vec<T>(stream: impl Stream<Item = api::Result<T>>) -> api::Result<Vec<T>> {
     let res = stream.collect::<Vec<_>>().await;
-    res.into_iter().collect::<anyhow::Result<Vec<_>>>()
+    res.into_iter().collect::<api::Result<Vec<_>>>()
 }
 
 fn expected(tags: impl IntoIterator<Item = &'static str>) -> Vec<TagInfo> {
@@ -23,18 +26,15 @@ fn expected(tags: impl IntoIterator<Item = &'static str>) -> Vec<TagInfo> {
         .collect()
 }
 
-async fn set<C: quic_rpc::Connector<RpcService>>(
-    tags: &tags::Client<C>,
-    names: impl IntoIterator<Item = &str>,
-) -> TestResult<()> {
+async fn set(tags: &Tags, names: impl IntoIterator<Item = &str>) -> TestResult<()> {
     for name in names {
         tags.set(name, Hash::new(name)).await?;
     }
     Ok(())
 }
 
-async fn tags_smoke<C: quic_rpc::Connector<RpcService>>(tags: tags::Client<C>) -> TestResult<()> {
-    set(&tags, ["a", "b", "c", "d", "e"]).await?;
+async fn tags_smoke(tags: &Tags) -> TestResult<()> {
+    set(tags, ["a", "b", "c", "d", "e"]).await?;
     let stream = tags.list().await?;
     let res = to_vec(stream).await?;
     assert_eq!(res, expected(["a", "b", "c", "d", "e"]));
@@ -65,7 +65,7 @@ async fn tags_smoke<C: quic_rpc::Connector<RpcService>>(tags: tags::Client<C>) -
     let res = to_vec(stream).await?;
     assert_eq!(res, expected([]));
 
-    set(&tags, ["a", "aa", "aaa", "aab", "b"]).await?;
+    set(tags, ["a", "aa", "aaa", "aab", "b"]).await?;
 
     let stream = tags.list_prefix("aa").await?;
     let res = to_vec(stream).await?;
@@ -81,7 +81,7 @@ async fn tags_smoke<C: quic_rpc::Connector<RpcService>>(tags: tags::Client<C>) -
     let res = to_vec(stream).await?;
     assert_eq!(res, expected([]));
 
-    set(&tags, ["a", "b", "c"]).await?;
+    set(tags, ["a", "b", "c"]).await?;
 
     assert_eq!(
         tags.get("b").await?,
@@ -107,12 +107,12 @@ async fn tags_smoke<C: quic_rpc::Connector<RpcService>>(tags: tags::Client<C>) -
         vec![TagInfo {
             name: "a".into(),
             hash: Hash::new("a"),
-            format: iroh_blobs::BlobFormat::HashSeq,
+            format: BlobFormat::HashSeq,
         }]
     );
 
     tags.delete_all().await?;
-    set(&tags, ["c"]).await?;
+    set(tags, ["c"]).await?;
     tags.rename("c", "f").await?;
     let stream = tags.list().await?;
     let res = to_vec(stream).await?;
@@ -121,7 +121,7 @@ async fn tags_smoke<C: quic_rpc::Connector<RpcService>>(tags: tags::Client<C>) -
         vec![TagInfo {
             name: "f".into(),
             hash: Hash::new("c"),
-            format: iroh_blobs::BlobFormat::Raw,
+            format: BlobFormat::Raw,
         }]
     );
 
@@ -132,19 +132,30 @@ async fn tags_smoke<C: quic_rpc::Connector<RpcService>>(tags: tags::Client<C>) -
 
 #[tokio::test]
 async fn tags_smoke_mem() -> TestResult<()> {
-    let endpoint = Endpoint::builder().bind().await?;
-    let blobs = Blobs::memory().build(&endpoint);
-    let client = blobs.client();
-    tags_smoke(client.tags()).await
+    tracing_subscriber::fmt::try_init().ok();
+    let store = MemStore::new();
+    tags_smoke(store.tags()).await
 }
 
 #[tokio::test]
 async fn tags_smoke_fs() -> TestResult<()> {
+    tracing_subscriber::fmt::try_init().ok();
     let td = tempfile::tempdir()?;
-    let endpoint = Endpoint::builder().bind().await?;
-    let blobs = Blobs::persistent(td.path().join("blobs.db"))
-        .await?
-        .build(&endpoint);
-    let client = blobs.client();
-    tags_smoke(client.tags()).await
+    let store = FsStore::load(td.path().join("a")).await?;
+    tags_smoke(store.tags()).await
+}
+
+#[tokio::test]
+async fn tags_smoke_fs_rpc() -> TestResult<()> {
+    tracing_subscriber::fmt::try_init().ok();
+    let unspecified = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, 0));
+    let (server, cert) = irpc::util::make_server_endpoint(unspecified)?;
+    let client = irpc::util::make_client_endpoint(unspecified, &[cert.as_ref()])?;
+    let td = tempfile::tempdir()?;
+    let store = FsStore::load(td.path().join("a")).await?;
+    tokio::spawn(store.deref().clone().listen(server.clone()));
+    let api = Store::connect(client, server.local_addr()?);
+    tags_smoke(api.tags()).await?;
+    api.shutdown().await?;
+    Ok(())
 }