Merge branch 'main' into hide-test-and-util

Author: rklaehn

README.md

@@ -22,6 +22,7 @@ This crate is used together with [iroh](https://crates.io/crates/iroh). Connecti
 
 - **Requester:** The side that asks for data. It is initiating requests to one or many providers.
 
+A node can be a provider and a requester at the same time.
 
 ## Getting started
 
@@ -31,33 +32,33 @@ Iroh provides a [`Router`](https://docs.rs/iroh/latest/iroh/protocol/struct.Rout
 
 Here is a basic example of how to set up `iroh-blobs` with `iroh`:
 
-```rust
+```rust,no_run
 use iroh::{protocol::Router, Endpoint};
-use iroh_blobs::{store::Store, net_protocol::Blobs};
+use iroh_blobs::{store::mem::MemStore, BlobsProtocol};
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
     // create an iroh endpoint that includes the standard discovery mechanisms
     // we've built at number0
     let endpoint = Endpoint::builder().discovery_n0().bind().await?;
 
-    // create an in-memory blob store
-    // use `iroh_blobs::net_protocol::Blobs::persistent` to load or create a
-    // persistent blob store from a path
-    let blobs = Blobs::memory().build(&endpoint);
-
-    // turn on the "rpc" feature if you need to create blobs and tags clients
-    let blobs_client = blobs.client();
-    let tags_client = blobs_client.tags();
+    // create a protocol handler using an in-memory blob store.
+    let store = MemStore::new();
+    let blobs = BlobsProtocol::new(&store, endpoint.clone(), None);
 
     // build the router
     let router = Router::builder(endpoint)
         .accept(iroh_blobs::ALPN, blobs.clone())
         .spawn();
 
-    // do fun stuff with the blobs protocol!
+    let tag = blobs.add_slice(b"Hello world").await?;
+    println!("We are now serving {}", blobs.ticket(tag).await?);
+
+    // wait for control-c
+    tokio::signal::ctrl_c().await;
+
+    // clean shutdown of router and store
     router.shutdown().await?;
-    drop(tags_client);
     Ok(())
 }
 ```
@@ -81,4 +82,4 @@ at your option.
 
 Unless you explicitly state otherwise, any contribution intentionally submitted
 for inclusion in this project by you, as defined in the Apache-2.0 license,
-shall be dual licensed as above, without any additional terms or conditions.
+shall be dual licensed as above, without any additional terms or conditions.

src/api/blobs.rs

@@ -127,7 +127,7 @@ impl Blobs {
         .await
     }
 
-    pub fn add_slice(&self, data: impl AsRef<[u8]>) -> AddProgress {
+    pub fn add_slice(&self, data: impl AsRef<[u8]>) -> AddProgress<'_> {
         let options = ImportBytesRequest {
             data: Bytes::copy_from_slice(data.as_ref()),
             format: crate::BlobFormat::Raw,
@@ -136,7 +136,7 @@ impl Blobs {
         self.add_bytes_impl(options)
     }
 
-    pub fn add_bytes(&self, data: impl Into<bytes::Bytes>) -> AddProgress {
+    pub fn add_bytes(&self, data: impl Into<bytes::Bytes>) -> AddProgress<'_> {
         let options = ImportBytesRequest {
             data: data.into(),
             format: crate::BlobFormat::Raw,
@@ -145,7 +145,7 @@ impl Blobs {
         self.add_bytes_impl(options)
     }
 
-    pub fn add_bytes_with_opts(&self, options: impl Into<AddBytesOptions>) -> AddProgress {
+    pub fn add_bytes_with_opts(&self, options: impl Into<AddBytesOptions>) -> AddProgress<'_> {
         let options = options.into();
         let request = ImportBytesRequest {
             data: options.data,
@@ -155,7 +155,7 @@ impl Blobs {
         self.add_bytes_impl(request)
     }
 
-    fn add_bytes_impl(&self, options: ImportBytesRequest) -> AddProgress {
+    fn add_bytes_impl(&self, options: ImportBytesRequest) -> AddProgress<'_> {
         trace!("{options:?}");
         let this = self.clone();
         let stream = Gen::new(|co| async move {
@@ -180,7 +180,7 @@ impl Blobs {
         AddProgress::new(self, stream)
     }
 
-    pub fn add_path_with_opts(&self, options: impl Into<AddPathOptions>) -> AddProgress {
+    pub fn add_path_with_opts(&self, options: impl Into<AddPathOptions>) -> AddProgress<'_> {
         let options = options.into();
         self.add_path_with_opts_impl(ImportPathRequest {
             path: options.path,
@@ -190,7 +190,7 @@ impl Blobs {
         })
     }
 
-    fn add_path_with_opts_impl(&self, options: ImportPathRequest) -> AddProgress {
+    fn add_path_with_opts_impl(&self, options: ImportPathRequest) -> AddProgress<'_> {
         trace!("{:?}", options);
         let client = self.client.clone();
         let stream = Gen::new(|co| async move {
@@ -215,7 +215,7 @@ impl Blobs {
         AddProgress::new(self, stream)
     }
 
-    pub fn add_path(&self, path: impl AsRef<Path>) -> AddProgress {
+    pub fn add_path(&self, path: impl AsRef<Path>) -> AddProgress<'_> {
         self.add_path_with_opts(AddPathOptions {
             path: path.as_ref().to_owned(),
             mode: ImportMode::Copy,
@@ -226,7 +226,7 @@ impl Blobs {
     pub async fn add_stream(
         &self,
         data: impl Stream<Item = io::Result<Bytes>> + Send + Sync + 'static,
-    ) -> AddProgress {
+    ) -> AddProgress<'_> {
         let inner = ImportByteStreamRequest {
             format: crate::BlobFormat::Raw,
             scope: Scope::default(),
@@ -521,7 +521,7 @@ pub struct Batch<'a> {
 }
 
 impl<'a> Batch<'a> {
-    pub fn add_bytes(&self, data: impl Into<Bytes>) -> BatchAddProgress {
+    pub fn add_bytes(&self, data: impl Into<Bytes>) -> BatchAddProgress<'_> {
         let options = ImportBytesRequest {
             data: data.into(),
             format: crate::BlobFormat::Raw,
@@ -530,7 +530,7 @@ impl<'a> Batch<'a> {
         BatchAddProgress(self.blobs.add_bytes_impl(options))
     }
 
-    pub fn add_bytes_with_opts(&self, options: impl Into<AddBytesOptions>) -> BatchAddProgress {
+    pub fn add_bytes_with_opts(&self, options: impl Into<AddBytesOptions>) -> BatchAddProgress<'_> {
         let options = options.into();
         BatchAddProgress(self.blobs.add_bytes_impl(ImportBytesRequest {
             data: options.data,
@@ -539,7 +539,7 @@ impl<'a> Batch<'a> {
         }))
     }
 
-    pub fn add_slice(&self, data: impl AsRef<[u8]>) -> BatchAddProgress {
+    pub fn add_slice(&self, data: impl AsRef<[u8]>) -> BatchAddProgress<'_> {
         let options = ImportBytesRequest {
             data: Bytes::copy_from_slice(data.as_ref()),
             format: crate::BlobFormat::Raw,
@@ -548,7 +548,7 @@ impl<'a> Batch<'a> {
         BatchAddProgress(self.blobs.add_bytes_impl(options))
     }
 
-    pub fn add_path_with_opts(&self, options: impl Into<AddPathOptions>) -> BatchAddProgress {
+    pub fn add_path_with_opts(&self, options: impl Into<AddPathOptions>) -> BatchAddProgress<'_> {
         let options = options.into();
         BatchAddProgress(self.blobs.add_path_with_opts_impl(ImportPathRequest {
             path: options.path,

src/lib.rs

@@ -1,3 +1,31 @@
+#![doc = include_str!("../README.md")]
+//! # Module docs
+//!
+//! The crate is designed to be used from the [iroh] crate.
+//!
+//! It implements a [protocol] for streaming content-addressed data transfer using
+//! [BLAKE3] verified streaming.
+//!
+//! It also provides a [store] module for storage of blobs and outboards,
+//! as well as a [persistent](crate::store::fs) and a [memory](crate::store::mem)
+//! store implementation.
+//!
+//! To implement a server, the [provider] module provides helpers for handling
+//! connections and individual requests given a store.
+//!
+//! To perform get requests, the [get] module provides utilities to perform
+//! requests and store the result in a store, as well as a low level state
+//! machine for executing requests.
+//!
+//! The client API is available in the [api] module. You can get a client
+//! either from one of the [store] implementations, or from the [BlobsProtocol]
+//! via a
+//!
+//! The [downloader](api::downloader) module provides a component to download blobs from
+//! multiple sources and store them in a store.
+//!
+//! [BLAKE3]: https://github.com/BLAKE3-team/BLAKE3-specs/blob/master/blake3.pdf
+//! [iroh]: https://docs.rs/iroh
 mod hash;
 pub mod store;
 pub use hash::{BlobFormat, Hash, HashAndFormat};

src/net_protocol.rs

@@ -36,7 +36,7 @@
 //! # }
 //! ```
 
-use std::{fmt::Debug, future::Future, sync::Arc};
+use std::{fmt::Debug, future::Future, ops::Deref, sync::Arc};
 
 use iroh::{
     endpoint::Connection,
@@ -66,6 +66,14 @@ pub struct BlobsProtocol {
     pub(crate) inner: Arc<BlobsInner>,
 }
 
+impl Deref for BlobsProtocol {
+    type Target = Store;
+
+    fn deref(&self) -> &Self::Target {
+        &self.inner.store
+    }
+}
+
 impl BlobsProtocol {
     pub fn new(store: &Store, endpoint: Endpoint, events: Option<mpsc::Sender<Event>>) -> Self {
         Self {