Skip to content

Commit a3bef55

Browse files
committed
WIP
1 parent ccacf9e commit a3bef55

File tree

18 files changed

+129
-148
lines changed

18 files changed

+129
-148
lines changed

Cargo.toml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ futures-util = "0.3.30"
9595
testdir = "0.9.1"
9696

9797
[features]
98-
default = ["fs-store", "net_protocol"]
98+
default = ["fs-store", "net_protocol", "formats-collection"]
9999
downloader = ["dep:parking_lot", "tokio-util/time", "dep:hashlink"]
100100
net_protocol = ["downloader", "dep:futures-util"]
101101
fs-store = ["dep:reflink-copy", "redb", "dep:tempfile"]
@@ -112,6 +112,8 @@ rpc = [
112112
"dep:walkdir",
113113
"downloader",
114114
]
115+
formats = []
116+
formats-collection = ["formats"]
115117

116118
example-iroh = [
117119
"dep:clap",

examples/local-swarm-discovery.rs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,10 @@ mod progress {
140140
ProgressStyle,
141141
};
142142
use iroh_blobs::{
143-
fetch::{db::DownloadProgress, progress::BlobProgress, Stats},
143+
fetch::{
144+
progress::{BlobProgress, DownloadProgress},
145+
Stats,
146+
},
144147
Hash,
145148
};
146149

src/cli.rs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,10 @@ use iroh::{NodeAddr, PublicKey, RelayUrl};
1919
use tokio::io::AsyncWriteExt;
2020

2121
use crate::{
22-
fetch::{db::DownloadProgress, progress::BlobProgress, Stats},
22+
fetch::{
23+
progress::{BlobProgress, DownloadProgress},
24+
Stats,
25+
},
2326
net_protocol::DownloadMode,
2427
provider::AddProgress,
2528
rpc::client::blobs::{

src/downloader.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ use tokio_util::{either::Either, sync::CancellationToken, time::delay_queue};
5555
use tracing::{debug, error, error_span, trace, warn, Instrument};
5656

5757
use crate::{
58-
fetch::{db::DownloadProgress, Stats},
58+
fetch::{progress::DownloadProgress, Stats},
5959
metrics::Metrics,
6060
store::Store,
6161
util::{local_pool::LocalPoolHandle, progress::ProgressSender},

src/downloader/get.rs

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ use iroh::endpoint;
77

88
use super::{progress::BroadcastProgressSender, DownloadKind, FailureAction, GetStartFut, Getter};
99
use crate::{
10-
fetch::{db::fetch_to_db_in_steps, Error},
11-
store::Store,
10+
fetch::Error,
11+
store::{fetch_to_db_in_steps, FetchState, FetchStateNeedsConn, Store},
1212
};
1313

1414
impl From<Error> for FailureAction {
@@ -34,7 +34,7 @@ pub(crate) struct IoGetter<S: Store> {
3434

3535
impl<S: Store> Getter for IoGetter<S> {
3636
type Connection = endpoint::Connection;
37-
type NeedsConn = crate::fetch::db::FetchStateNeedsConn;
37+
type NeedsConn = FetchStateNeedsConn;
3838

3939
fn get(
4040
&mut self,
@@ -45,10 +45,8 @@ impl<S: Store> Getter for IoGetter<S> {
4545
async move {
4646
match fetch_to_db_in_steps(store, kind.hash_and_format(), progress_sender).await {
4747
Err(err) => Err(err.into()),
48-
Ok(crate::fetch::db::FetchState::Complete(stats)) => {
49-
Ok(super::GetOutput::Complete(stats))
50-
}
51-
Ok(crate::fetch::db::FetchState::NeedsConn(needs_conn)) => {
48+
Ok(FetchState::Complete(stats)) => Ok(super::GetOutput::Complete(stats)),
49+
Ok(FetchState::NeedsConn(needs_conn)) => {
5250
Ok(super::GetOutput::NeedsConn(needs_conn))
5351
}
5452
}
@@ -57,7 +55,7 @@ impl<S: Store> Getter for IoGetter<S> {
5755
}
5856
}
5957

60-
impl super::NeedsConn<endpoint::Connection> for crate::fetch::db::FetchStateNeedsConn {
58+
impl super::NeedsConn<endpoint::Connection> for FetchStateNeedsConn {
6159
fn proceed(self, conn: endpoint::Connection) -> super::GetProceedFut {
6260
async move {
6361
let res = self.proceed(conn).await;

src/downloader/progress.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ use parking_lot::Mutex;
1111

1212
use super::DownloadKind;
1313
use crate::{
14-
fetch::{db::DownloadProgress, progress::TransferState},
14+
fetch::progress::{DownloadProgress, TransferState},
1515
util::progress::{AsyncChannelProgressSender, IdGenerator, ProgressSendError, ProgressSender},
1616
};
1717

src/downloader/test.rs

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,7 @@ use iroh::SecretKey;
1010

1111
use super::*;
1212
use crate::{
13-
fetch::{
14-
db::BlobId,
15-
progress::{BlobProgress, TransferState},
16-
},
13+
fetch::progress::{BlobId, BlobProgress, TransferState},
1714
util::{
1815
local_pool::LocalPool,
1916
progress::{AsyncChannelProgressSender, IdGenerator},

src/fetch.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ use crate::{
3131

3232
mod error;
3333
pub use error::Error;
34-
pub mod db;
3534
pub mod progress;
3635
pub mod request;
3736

src/fetch/progress.rs

Lines changed: 88 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use std::{collections::HashMap, num::NonZeroU64};
55
use serde::{Deserialize, Serialize};
66
use tracing::warn;
77

8-
use super::db::{BlobId, DownloadProgress};
8+
use super::Stats;
99
use crate::{protocol::RangeSpec, store::BaoBlobSize, Hash};
1010

1111
/// The identifier for progress events.
@@ -183,3 +183,90 @@ impl TransferState {
183183
}
184184
}
185185
}
186+
187+
/// Progress updates for the get operation.
188+
#[derive(Debug, Clone, Serialize, Deserialize)]
189+
pub enum DownloadProgress {
190+
/// Initial state if subscribing to a running or queued transfer.
191+
InitialState(TransferState),
192+
/// Data was found locally.
193+
FoundLocal {
194+
/// child offset
195+
child: BlobId,
196+
/// The hash of the entry.
197+
hash: Hash,
198+
/// The size of the entry in bytes.
199+
size: BaoBlobSize,
200+
/// The ranges that are available locally.
201+
valid_ranges: RangeSpec,
202+
},
203+
/// A new connection was established.
204+
Connected,
205+
/// An item was found with hash `hash`, from now on referred to via `id`.
206+
Found {
207+
/// A new unique progress id for this entry.
208+
id: u64,
209+
/// Identifier for this blob within this download.
210+
///
211+
/// Will always be [`BlobId::Root`] unless a hashseq is downloaded, in which case this
212+
/// allows to identify the children by their offset in the hashseq.
213+
child: BlobId,
214+
/// The hash of the entry.
215+
hash: Hash,
216+
/// The size of the entry in bytes.
217+
size: u64,
218+
},
219+
/// An item was found with hash `hash`, from now on referred to via `id`.
220+
FoundHashSeq {
221+
/// The name of the entry.
222+
hash: Hash,
223+
/// Number of children in the collection, if known.
224+
children: u64,
225+
},
226+
/// We got progress ingesting item `id`.
227+
Progress {
228+
/// The unique id of the entry.
229+
id: u64,
230+
/// The offset of the progress, in bytes.
231+
offset: u64,
232+
},
233+
/// We are done with `id`.
234+
Done {
235+
/// The unique id of the entry.
236+
id: u64,
237+
},
238+
/// All operations finished.
239+
///
240+
/// This will be the last message in the stream.
241+
AllDone(Stats),
242+
/// We got an error and need to abort.
243+
///
244+
/// This will be the last message in the stream.
245+
Abort(serde_error::Error),
246+
}
247+
248+
/// The id of a blob in a transfer
249+
#[derive(
250+
Debug, Copy, Clone, Ord, PartialOrd, Eq, PartialEq, std::hash::Hash, Serialize, Deserialize,
251+
)]
252+
pub enum BlobId {
253+
/// The root blob (child id 0)
254+
Root,
255+
/// A child blob (child id > 0)
256+
Child(NonZeroU64),
257+
}
258+
259+
impl BlobId {
260+
pub(crate) fn from_offset(id: u64) -> Self {
261+
NonZeroU64::new(id).map(Self::Child).unwrap_or(Self::Root)
262+
}
263+
}
264+
265+
impl From<BlobId> for u64 {
266+
fn from(value: BlobId) -> Self {
267+
match value {
268+
BlobId::Root => 0,
269+
BlobId::Child(id) => id.into(),
270+
}
271+
}
272+
}

src/format.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,5 @@
1313
//! n-1 items, where n is the number of blobs in the HashSeq.
1414
//!
1515
//! [postcard]: https://docs.rs/postcard/latest/postcard/
16+
#[cfg(feature = "formats-collection")]
1617
pub mod collection;

0 commit comments

Comments
 (0)