From 66acba999527440145a906e7de7243c8dab372c0 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Mon, 11 Apr 2022 11:26:12 +0530 Subject: [PATCH 01/79] Initial addition for example with Smart pointers --- src/lib.rs | 1 + src/lib/cli.rs | 3 +++ src/lib/smart_pointers.rs | 45 +++++++++++++++++++++++++++++++++++++++ src/main.rs | 11 +++++++++- src/smart_pointers.rs | 0 5 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 src/lib/smart_pointers.rs create mode 100644 src/smart_pointers.rs diff --git a/src/lib.rs b/src/lib.rs index a85656b..4268ad0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,7 @@ pub(crate) mod builder; pub(crate) mod cli; pub(crate) mod dispatch; pub(crate) mod oop_pattern; +pub(crate) mod smart_pointers; //mod generics_1; //mod iterators_1; //mod trait_objects_1; diff --git a/src/lib/cli.rs b/src/lib/cli.rs index 360d514..0ad9940 100644 --- a/src/lib/cli.rs +++ b/src/lib/cli.rs @@ -19,6 +19,9 @@ pub(crate) enum Commands { /// Implementing an Object-Oriented Design Pattern with Type state TypeState, + + /// Smart pointers + SmartPointers, } pub(crate) fn runner(mut mk: impl FnMut() -> T) -> T { diff --git a/src/lib/smart_pointers.rs b/src/lib/smart_pointers.rs new file mode 100644 index 0000000..640f88e --- /dev/null +++ b/src/lib/smart_pointers.rs @@ -0,0 +1,45 @@ +pub(crate) struct Message { + content: String, + bytes: Vec, +} + +impl Message { + pub(crate) fn update(mut self, content: &str) -> Self { + let bytes: Vec = content.to_string().as_bytes().to_vec(); + self.bytes = bytes; + + self + } + + pub(crate) fn content(&self) -> &String { + &self.content + } + + pub(crate) fn bytes(&self) -> &Vec { + &self.bytes + } +} + +pub(crate) struct MessageBuilder { + content: String, +} + +impl MessageBuilder { + pub(crate) fn new() -> Self { + Self { + content: String::default(), + } + } + + pub(crate) fn content(mut self, content: &str) -> Self { + self.content = content.to_string(); + self + } + + pub(crate) fn build(&self) -> Message { + Message { + content: self.content.to_string(), + bytes: vec![0], + } + } +} diff --git a/src/main.rs b/src/main.rs index fd725d9..15ce63e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,7 +7,7 @@ use clap::Parser; use lib::cli::{runner, Args, Commands}; -use lib::{builder::TaskManagerBuilder, dispatch::*, oop_pattern::*}; +use lib::{builder::TaskManagerBuilder, dispatch::*, oop_pattern::*, smart_pointers::*}; use log::{debug, info}; mod lib; @@ -69,6 +69,15 @@ fn main() { assert_eq!("I ate fish at lunch", post.content()); }) } + Some(Commands::SmartPointers) => runner(|| { + info!("Tutorial: Smart pointers\n"); + + let new_message = MessageBuilder::new().content("hello").build(); + let new_message = new_message.update("foo"); + + let byte_zero: u8 = 0; + assert_ne!(new_message.bytes(), &vec![byte_zero]); + }), _ => info!("Command not found"), }; } diff --git a/src/smart_pointers.rs b/src/smart_pointers.rs new file mode 100644 index 0000000..e69de29 From 3b3a14100679f5fd76d72c4178d77ebdf4e6453d Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Mon, 11 Apr 2022 13:34:14 +0530 Subject: [PATCH 02/79] Add example to convert String to Vec and back --- src/lib/smart_pointers.rs | 4 ++++ src/main.rs | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/src/lib/smart_pointers.rs b/src/lib/smart_pointers.rs index 640f88e..3f2503b 100644 --- a/src/lib/smart_pointers.rs +++ b/src/lib/smart_pointers.rs @@ -11,6 +11,10 @@ impl Message { self } + pub(crate) fn content_from_bytes(&self) -> Option { + Some(String::from_utf8(self.bytes.clone()).ok()?) + } + pub(crate) fn content(&self) -> &String { &self.content } diff --git a/src/main.rs b/src/main.rs index 15ce63e..88156de 100644 --- a/src/main.rs +++ b/src/main.rs @@ -77,6 +77,12 @@ fn main() { let byte_zero: u8 = 0; assert_ne!(new_message.bytes(), &vec![byte_zero]); + + let message = new_message.update("Häagen-Dazs"); + assert_eq!( + message.content_from_bytes().unwrap(), + "Häagen-Dazs".to_string() + ); }), _ => info!("Command not found"), }; From 464d3688896822fb65314845e16b7943551b465a Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Mon, 11 Apr 2022 16:02:28 +0530 Subject: [PATCH 03/79] Add example with UnsafeCell --- src/lib/smart_pointers.rs | 30 ++++++++++++++++++++++++++++++ src/main.rs | 27 ++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/src/lib/smart_pointers.rs b/src/lib/smart_pointers.rs index 3f2503b..65a69fc 100644 --- a/src/lib/smart_pointers.rs +++ b/src/lib/smart_pointers.rs @@ -1,3 +1,33 @@ +use std::cell::UnsafeCell; + +pub(crate) struct Cell { + value: UnsafeCell, +} + +impl Cell { + pub(crate) fn new(value: T) -> Self { + Cell { + value: UnsafeCell::new(value), + } + } + + pub(crate) fn set(&self, value: T) { + // SAFETY: we know no-one else is concurrently mutating self (because !Sync) + // SAFETY: we're not invalidating any references as we are not sharing any. + unsafe { *self.value.get() = value }; + } + + pub(crate) fn get(&self) -> T + where + T: Copy, + { + // SAFETY: we know no-one else is concurrently mutating this value, since only this thread + // can mutate (because !Sync) and it is executing this function + unsafe { *self.value.get() } + } +} + +/// Contrived example storing a String as Vec pub(crate) struct Message { content: String, bytes: Vec, diff --git a/src/main.rs b/src/main.rs index 88156de..61ecc72 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ //! This is the main application -#![forbid(unsafe_code)] +// Disabled for use of UnsafeCell +//#![forbid(unsafe_code)] #![allow(unused_imports)] #![deny(unreachable_pub, private_in_public, unstable_features)] #![warn(rust_2018_idioms, future_incompatible, nonstandard_style)] @@ -9,6 +10,8 @@ use clap::Parser; use lib::cli::{runner, Args, Commands}; use lib::{builder::TaskManagerBuilder, dispatch::*, oop_pattern::*, smart_pointers::*}; use log::{debug, info}; +use std::io::Read; +use std::sync::Arc; mod lib; @@ -83,6 +86,28 @@ fn main() { message.content_from_bytes().unwrap(), "Häagen-Dazs".to_string() ); + + struct BytesToString { + value: String, + } + + impl BytesToString { + pub(crate) fn new(value: &Vec) -> Self { + Self { + value: String::from_utf8(value.clone()).unwrap_or_default(), + } + } + } + + impl Into for BytesToString { + fn into(self) -> String { + self.value + } + } + + let x = Cell::new(message.bytes()); + let contents: String = BytesToString::new(x.get()).into(); + assert_eq!(contents, "Häagen-Dazs".to_string()); }), _ => info!("Command not found"), }; From 070d7c6707e7ed886a5a1513ad6bdbd00695394f Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Mon, 11 Apr 2022 18:46:06 +0530 Subject: [PATCH 04/79] Add comments --- src/main.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main.rs b/src/main.rs index 61ecc72..288a4b6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -87,6 +87,7 @@ fn main() { "Häagen-Dazs".to_string() ); + // Example of using a new-type to implement the Into trait struct BytesToString { value: String, } @@ -106,6 +107,8 @@ fn main() { } let x = Cell::new(message.bytes()); + + // BytesToString is used as a new-type to convert Vec to a String let contents: String = BytesToString::new(x.get()).into(); assert_eq!(contents, "Häagen-Dazs".to_string()); }), From 70f36bd5794f501dafdcfd8ef8d69b94a4486f5d Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Mon, 11 Apr 2022 18:50:18 +0530 Subject: [PATCH 05/79] Add note --- src/lib/smart_pointers.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lib/smart_pointers.rs b/src/lib/smart_pointers.rs index 65a69fc..59eee16 100644 --- a/src/lib/smart_pointers.rs +++ b/src/lib/smart_pointers.rs @@ -4,6 +4,10 @@ pub(crate) struct Cell { value: UnsafeCell, } +// implied by UnsafeCell +// impl !Sync for Cell +// https://doc.rust-lang.org/std/cell/struct.UnsafeCell.html#impl-Sync + impl Cell { pub(crate) fn new(value: T) -> Self { Cell { From a769b9aba43a5f872e2e6e4030de234a7b96afed Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Tue, 12 Apr 2022 09:59:16 +0530 Subject: [PATCH 06/79] Add notes --- src/lib/smart_pointers.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/smart_pointers.rs b/src/lib/smart_pointers.rs index 59eee16..50f3763 100644 --- a/src/lib/smart_pointers.rs +++ b/src/lib/smart_pointers.rs @@ -1,5 +1,8 @@ use std::cell::UnsafeCell; +// Further coverage of Cell, RefCell and Rc (https://youtu.be/8O0Nt9qY_vo): +// https://gist.github.com/jonhoo/7cfdfe581e5108b79c2a4e9fbde38de8 + pub(crate) struct Cell { value: UnsafeCell, } From 1c655788ad7e25886382e5508e229d6f7458f5fb Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Mon, 14 Nov 2022 18:44:14 +0530 Subject: [PATCH 07/79] WIP start on Traits --- Cargo.toml | 1 + src/lib.rs | 11 ++++++----- src/lib/cli.rs | 6 +++++- src/lib/traits.rs | 10 ++++++++++ src/main.rs | 31 ++++++++++++++++++++++++------- src/traits.rs | 0 6 files changed, 46 insertions(+), 13 deletions(-) create mode 100644 src/lib/traits.rs create mode 100644 src/traits.rs diff --git a/Cargo.toml b/Cargo.toml index 0d48074..51a86b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" publish = false [dependencies] +anyhow = "1.0.66" clap = { version = "3", features = ["derive"] } dotenv = "0.15.0" env_logger = "0.9.0" diff --git a/src/lib.rs b/src/lib.rs index 4268ad0..e699d5e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,8 +1,9 @@ -pub(crate) mod builder; -pub(crate) mod cli; -pub(crate) mod dispatch; -pub(crate) mod oop_pattern; -pub(crate) mod smart_pointers; +pub mod builder; +pub mod cli; +pub mod dispatch; +pub mod oop_pattern; +pub mod smart_pointers; +pub mod traits; //mod generics_1; //mod iterators_1; //mod trait_objects_1; diff --git a/src/lib/cli.rs b/src/lib/cli.rs index 0ad9940..e045bb2 100644 --- a/src/lib/cli.rs +++ b/src/lib/cli.rs @@ -1,3 +1,4 @@ +use anyhow::{Error, Result}; use clap::{Parser, Subcommand}; use std::str::FromStr; @@ -22,8 +23,11 @@ pub(crate) enum Commands { /// Smart pointers SmartPointers, + + /// Traits + Traits, } -pub(crate) fn runner(mut mk: impl FnMut() -> T) -> T { +pub(crate) fn runner(mut mk: impl FnMut() -> Result) -> Result { mk() } diff --git a/src/lib/traits.rs b/src/lib/traits.rs new file mode 100644 index 0000000..ddc2b85 --- /dev/null +++ b/src/lib/traits.rs @@ -0,0 +1,10 @@ +use anyhow::{Error, Result}; +use std::fmt::Display; + +pub fn x(b: Box) -> Box { + b +} + +pub fn runner() -> Result<()> { + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index 288a4b6..3480b73 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,16 +6,17 @@ #![deny(unreachable_pub, private_in_public, unstable_features)] #![warn(rust_2018_idioms, future_incompatible, nonstandard_style)] +use anyhow::{Error, Result}; use clap::Parser; use lib::cli::{runner, Args, Commands}; -use lib::{builder::TaskManagerBuilder, dispatch::*, oop_pattern::*, smart_pointers::*}; +use lib::{builder::TaskManagerBuilder, dispatch::*, oop_pattern::*, smart_pointers::*, traits}; use log::{debug, info}; use std::io::Read; use std::sync::Arc; -mod lib; +pub mod lib; -fn main() { +fn main() -> Result<()> { env_logger::init(); let cli = Args::parse(); @@ -44,7 +45,9 @@ fn main() { let x: &dyn Hei = &"hei"; x.weird(); say_hei(x); - }), + + Ok(()) + })?, Some(Commands::Builder) => runner(|| { info!("Tutorial: Builder pattern\n"); @@ -52,7 +55,9 @@ fn main() { debug!("Task manager.count: {}", task_manager.count()); assert_eq!(*task_manager.count(), 10); - }), + + Ok(()) + })?, Some(Commands::TypeState) => { runner(|| { info!("Tutorial: OOP design pattern with Type State\n"); @@ -70,7 +75,8 @@ fn main() { let post = post.approve(); assert_eq!("I ate fish at lunch", post.content()); - }) + Ok(()) + })? } Some(Commands::SmartPointers) => runner(|| { info!("Tutorial: Smart pointers\n"); @@ -111,7 +117,18 @@ fn main() { // BytesToString is used as a new-type to convert Vec to a String let contents: String = BytesToString::new(x.get()).into(); assert_eq!(contents, "Häagen-Dazs".to_string()); - }), + + Ok(()) + })?, + Some(Commands::Traits) => runner(|| { + info!("Tutorial: Traits\n"); + + traits::runner()?; + + Ok(()) + })?, _ => info!("Command not found"), }; + + Ok(()) } diff --git a/src/traits.rs b/src/traits.rs new file mode 100644 index 0000000..e69de29 From 6538841c039fa444a05f71b531fdc5df73d9ac49 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Mon, 14 Nov 2022 18:54:56 +0530 Subject: [PATCH 08/79] Add notes on use of `impl Trait` on a fn parameter --- src/lib/traits.rs | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/lib/traits.rs b/src/lib/traits.rs index ddc2b85..a7b63f0 100644 --- a/src/lib/traits.rs +++ b/src/lib/traits.rs @@ -5,6 +5,43 @@ pub fn x(b: Box) -> Box { b } +#[derive(Debug)] +struct Device(u8); + +impl Device { + fn new(id: u8) -> Self { + Self(id) + } +} + +// This is added to satisfy the trait bound on `x` +impl Display for Device { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_fmt(format_args!("Device ({})", self.0)) + } +} + pub fn runner() -> Result<()> { + let device = Device::new(1); + + // If we try to run `let resp = x(Box::new(device));`, notice that we have not satisfied the trait bound that is specified on `x`. + // + // error[E0277]: `Device` doesn't implement `std::fmt::Display` + // --> src/lib/traits.rs:19:18 + // | + // 19 | let resp = x(Box::new(device)); + // | - ^^^^^^^^^^^^^^^^ `Device` cannot be formatted with the default formatter + // | | + // | required by a bound introduced by this call + // | + // = help: the trait `std::fmt::Display` is not implemented for `Device` + // = note: in format strings you may be able to use `{:?}` (or {:#?} for pretty-print) instead + // note: required by a bound in `x` + // --> src/lib/traits.rs:4:22 + // | + // 4 | pub fn x(b: Box) -> Box { + // | ^^^^^^^ required by this bound in `x` + let resp = x(Box::new(device)); + Ok(()) } From 193c439a27e606f75ec6afe66fd7b683c6d0a8ba Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Wed, 16 Nov 2022 14:28:29 +0530 Subject: [PATCH 09/79] FIX organisations of modules --- src/builder.rs | 0 src/cli.rs | 0 src/dispatch.rs | 0 src/{lib.rs => lib.rs_old} | 1 + src/lib/cli.rs | 3 + src/lib/conversion/mod.rs | 46 ++++++++++ src/lib/mod.rs | 7 ++ src/lib/traits.rs | 102 ++++++++++++++++++++++- src/main.rs | 8 ++ src/oop_pattern.rs | 0 src/{ => sandbox}/generics_1/mod.rs | 0 src/{ => sandbox}/iterators_1/mod.rs | 2 +- src/sandbox/mod.rs | 3 + src/{ => sandbox}/trait_objects_1/mod.rs | 0 src/smart_pointers.rs | 0 src/traits.rs | 0 16 files changed, 168 insertions(+), 4 deletions(-) delete mode 100644 src/builder.rs delete mode 100644 src/cli.rs delete mode 100644 src/dispatch.rs rename src/{lib.rs => lib.rs_old} (89%) create mode 100644 src/lib/conversion/mod.rs create mode 100644 src/lib/mod.rs delete mode 100644 src/oop_pattern.rs rename src/{ => sandbox}/generics_1/mod.rs (100%) rename src/{ => sandbox}/iterators_1/mod.rs (99%) create mode 100644 src/sandbox/mod.rs rename src/{ => sandbox}/trait_objects_1/mod.rs (100%) delete mode 100644 src/smart_pointers.rs delete mode 100644 src/traits.rs diff --git a/src/builder.rs b/src/builder.rs deleted file mode 100644 index e69de29..0000000 diff --git a/src/cli.rs b/src/cli.rs deleted file mode 100644 index e69de29..0000000 diff --git a/src/dispatch.rs b/src/dispatch.rs deleted file mode 100644 index e69de29..0000000 diff --git a/src/lib.rs b/src/lib.rs_old similarity index 89% rename from src/lib.rs rename to src/lib.rs_old index e699d5e..29f68c1 100644 --- a/src/lib.rs +++ b/src/lib.rs_old @@ -1,5 +1,6 @@ pub mod builder; pub mod cli; +pub mod conversion; pub mod dispatch; pub mod oop_pattern; pub mod smart_pointers; diff --git a/src/lib/cli.rs b/src/lib/cli.rs index e045bb2..d1ed59d 100644 --- a/src/lib/cli.rs +++ b/src/lib/cli.rs @@ -26,6 +26,9 @@ pub(crate) enum Commands { /// Traits Traits, + + /// Conversion + Conversion, } pub(crate) fn runner(mut mk: impl FnMut() -> Result) -> Result { diff --git a/src/lib/conversion/mod.rs b/src/lib/conversion/mod.rs new file mode 100644 index 0000000..233f2bd --- /dev/null +++ b/src/lib/conversion/mod.rs @@ -0,0 +1,46 @@ +use anyhow::Result; + +pub fn runner() -> Result<()> { + lesson_1()?; + lesson_2()?; + + Ok(()) +} + +pub fn lesson_1() -> Result<()> { + let value: u8 = 10; + let resp: u16 = value.into(); + assert_eq!(resp as u8, value); + + let resp2 = u16::from(value); + assert_eq!(resp, resp2); + + Ok(()) +} + +// This is not possible as we cannot convert from u32 to u16 without truncation and data loss. +// +// --> src/lib/conversion/mod.rs:13:21 +// | +// 13 | let resp: u16 = value.into(); +// | ^^^^^ ---- required by a bound introduced by this call +// | | +// | the trait `From` is not implemented for `u16` +// | +// = help: the following other types implement trait `From`: +// > +// > +// > +// > +// > +// > +// > +// > +// and 67 others +// = note: required for `u32` to implement `Into` +pub fn lesson_2() -> Result<()> { + let value: u32 = 10; + // let resp: u16 = value.into(); + + Ok(()) +} diff --git a/src/lib/mod.rs b/src/lib/mod.rs new file mode 100644 index 0000000..53ad4fa --- /dev/null +++ b/src/lib/mod.rs @@ -0,0 +1,7 @@ +pub mod builder; +pub mod cli; +pub mod conversion; +pub mod dispatch; +pub mod oop_pattern; +pub mod smart_pointers; +pub mod traits; diff --git a/src/lib/traits.rs b/src/lib/traits.rs index a7b63f0..0a16329 100644 --- a/src/lib/traits.rs +++ b/src/lib/traits.rs @@ -1,6 +1,11 @@ use anyhow::{Error, Result}; -use std::fmt::Display; +use log::info; +use std::{ + error::Error as StdError, + fmt::{self, Display}, +}; +// Lesson 1 pub fn x(b: Box) -> Box { b } @@ -21,7 +26,100 @@ impl Display for Device { } } +// Error +pub type BoxError = Box; + +#[derive(Debug)] +pub struct BlanketError { + inner: BoxError, +} + +impl BlanketError { + /// Create a new `Error` from a boxable error. + pub fn new(error: impl Into) -> Self { + Self { + inner: error.into(), + } + } + + /// Convert an `Error` back into the underlying boxed trait object. + pub fn into_inner(self) -> BoxError { + self.inner + } +} + +impl fmt::Display for BlanketError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.inner.fmt(f) + } +} + +impl StdError for BlanketError { + fn source(&self) -> Option<&(dyn StdError + 'static)> { + Some(&*self.inner) + } +} + +// Lesson 2: Traits +pub trait RTC { + type Error: ErrorType; +} + +pub trait ErrorType: Display { + /// Error type + type Error: std::error::Error; +} + +// What does this do exactly? +impl ErrorType for &mut T { + type Error = T::Error; +} + +impl ErrorType for BlanketError { + type Error = Self; +} + +struct RTCDevice(u8); + +impl RTC for RTCDevice { + type Error = BlanketError; +} + +impl fmt::Display for RTCDevice { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl ErrorType for RTCDevice { + type Error = BlanketError; +} + pub fn runner() -> Result<()> { + lesson_1_add_trait_bound_to_parameter(); + + lesson_2(); + + Ok(()) +} + +fn test_mut_t(device: T, message: &str) +where + T: ErrorType, +{ + info!("{}", message); +} + +fn lesson_2() { + let device = RTCDevice(1); + test_mut_t::(device, "This is a standard use of `T: Trait`"); + + // This works with the "forwarding impl" for `&mut T`. It is handy to note that `T: Trait`, doesn't automatically mean `&mut T: Trait`. You have to write a "forwarding impl" for that. These are fairly common. &mut has them for Iterator, Write, Display, Debug, and more, for example + let mut device = RTCDevice(1); + test_mut_t::<&mut RTCDevice>(&mut device, "Here we are using `&mut T: Trait`"); +} + +fn lesson_1_add_trait_bound_to_parameter() { let device = Device::new(1); // If we try to run `let resp = x(Box::new(device));`, notice that we have not satisfied the trait bound that is specified on `x`. @@ -42,6 +140,4 @@ pub fn runner() -> Result<()> { // 4 | pub fn x(b: Box) -> Box { // | ^^^^^^^ required by this bound in `x` let resp = x(Box::new(device)); - - Ok(()) } diff --git a/src/main.rs b/src/main.rs index 3480b73..58dac8e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,6 +15,7 @@ use std::io::Read; use std::sync::Arc; pub mod lib; +// pub mod sandbox; fn main() -> Result<()> { env_logger::init(); @@ -127,6 +128,13 @@ fn main() -> Result<()> { Ok(()) })?, + Some(Commands::Conversion) => runner(|| { + info!("Tutorial: Conversion\n"); + + lib::conversion::runner()?; + + Ok(()) + })?, _ => info!("Command not found"), }; diff --git a/src/oop_pattern.rs b/src/oop_pattern.rs deleted file mode 100644 index e69de29..0000000 diff --git a/src/generics_1/mod.rs b/src/sandbox/generics_1/mod.rs similarity index 100% rename from src/generics_1/mod.rs rename to src/sandbox/generics_1/mod.rs diff --git a/src/iterators_1/mod.rs b/src/sandbox/iterators_1/mod.rs similarity index 99% rename from src/iterators_1/mod.rs rename to src/sandbox/iterators_1/mod.rs index e12e98b..ef2d405 100644 --- a/src/iterators_1/mod.rs +++ b/src/sandbox/iterators_1/mod.rs @@ -17,7 +17,7 @@ fn shoes_into_iter(shoes: MyVec>, shoe_size: u32) -> MyVec> } #[derive(PartialEq, Debug, Clone)] -struct MyVec(T); +pub struct MyVec(T); struct MyVecIter<'a, T> { vector: &'a T, diff --git a/src/sandbox/mod.rs b/src/sandbox/mod.rs new file mode 100644 index 0000000..99c03fc --- /dev/null +++ b/src/sandbox/mod.rs @@ -0,0 +1,3 @@ +pub mod generics_1; +pub mod iterators_1; +pub mod trait_objects_1; diff --git a/src/trait_objects_1/mod.rs b/src/sandbox/trait_objects_1/mod.rs similarity index 100% rename from src/trait_objects_1/mod.rs rename to src/sandbox/trait_objects_1/mod.rs diff --git a/src/smart_pointers.rs b/src/smart_pointers.rs deleted file mode 100644 index e69de29..0000000 diff --git a/src/traits.rs b/src/traits.rs deleted file mode 100644 index e69de29..0000000 From 1e1b354b337ae61289f3a2321976a7b0ad0aff90 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Wed, 16 Nov 2022 16:27:39 +0530 Subject: [PATCH 10/79] WIP initial approach --- src/lib/conversion/mod.rs | 89 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/src/lib/conversion/mod.rs b/src/lib/conversion/mod.rs index 233f2bd..2fa572d 100644 --- a/src/lib/conversion/mod.rs +++ b/src/lib/conversion/mod.rs @@ -3,6 +3,7 @@ use anyhow::Result; pub fn runner() -> Result<()> { lesson_1()?; lesson_2()?; + embedded_rtc::lesson_3()?; Ok(()) } @@ -44,3 +45,91 @@ pub fn lesson_2() -> Result<()> { Ok(()) } + +pub mod embedded_rtc { + use super::*; + use std::fmt; + + #[derive(Debug, Clone, Copy)] + pub enum Weekday { + Sunday = 1, + Monday = 2, + Tuesday = 4, + Wednesday = 8, + Thursday = 16, + Friday = 32, + Saturday = 64, + } + + impl fmt::Display for Weekday { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.to_s()) + } + } + + impl Weekday { + pub fn value(&self) -> u8 { + *self as u8 + } + + pub fn to_s(&self) -> String { + match self { + Self::Sunday => "Sunday".to_string(), + Self::Monday => "Monday".to_string(), + Self::Tuesday => "Tuesday".to_string(), + Self::Wednesday => "Wednesday".to_string(), + Self::Thursday => "Thursday".to_string(), + Self::Friday => "Friday".to_string(), + Self::Saturday => "Saturday".to_string(), + } + } + + // Returns the first 3-letters of the day of the week + pub fn to_short(&self) -> Result { + let day = self.to_s(); + + // let resp: Vec = day + // .chars() + // .enumerate() + // .map(|(i, x)| if i <= 2 { x } else { char::default() }) + // .collect(); + // let res = resp[0..3].to_vec(); + + let result = String::from_utf8(day[0..3].as_bytes().to_vec())?; + println!("{:?}", result); + + Ok(result) + } + } + + impl From for Weekday { + fn from(day: u8) -> Self { + match day { + 1 => Self::Sunday, + 2 => Self::Monday, + 4 => Self::Tuesday, + 8 => Self::Wednesday, + 16 => Self::Thursday, + 32 => Self::Friday, + 64 => Self::Saturday, + _ => Self::Sunday, + } + } + } + + pub fn lesson_3() -> Result<()> { + let weekday_num = 4 as u8; + + // This is possible since we added the `From<_>` trait above + // via `impl From for Weekday { //... }` + let weekday = Weekday::from(weekday_num); + + let weekday_value = weekday.value(); + assert_eq!(weekday_value, 4); + assert_eq!(weekday.to_s(), "Tuesday"); + + weekday.to_short(); + + Ok(()) + } +} From bb093b425d09f5ddf379bee087d39e3a3946ba40 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Wed, 16 Nov 2022 16:30:02 +0530 Subject: [PATCH 11/79] Add method to provide short version of the weekday --- src/lib/conversion/mod.rs | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/lib/conversion/mod.rs b/src/lib/conversion/mod.rs index 2fa572d..f585ff9 100644 --- a/src/lib/conversion/mod.rs +++ b/src/lib/conversion/mod.rs @@ -87,16 +87,7 @@ pub mod embedded_rtc { // Returns the first 3-letters of the day of the week pub fn to_short(&self) -> Result { let day = self.to_s(); - - // let resp: Vec = day - // .chars() - // .enumerate() - // .map(|(i, x)| if i <= 2 { x } else { char::default() }) - // .collect(); - // let res = resp[0..3].to_vec(); - - let result = String::from_utf8(day[0..3].as_bytes().to_vec())?; - println!("{:?}", result); + let result: String = day.chars().take(3).collect(); Ok(result) } @@ -127,8 +118,7 @@ pub mod embedded_rtc { let weekday_value = weekday.value(); assert_eq!(weekday_value, 4); assert_eq!(weekday.to_s(), "Tuesday"); - - weekday.to_short(); + assert_eq!(weekday.to_short()?, "Tue"); Ok(()) } From 753b7367effbd64499dc363a4b7e04e7ecfc1125 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Thu, 17 Nov 2022 16:12:32 +0530 Subject: [PATCH 12/79] FIX: Make sure lib.rs is implemented correctly --- src/{lib => }/builder.rs | 12 ++++----- src/{lib => }/cli.rs | 8 +++--- src/{lib/conversion/mod.rs => conversion.rs} | 3 +++ src/{lib => }/dispatch.rs | 12 ++++----- src/{lib.rs_old => lib.rs} | 0 src/lib/mod.rs | 7 ------ src/main.rs | 11 ++++----- src/{lib => }/oop_pattern.rs | 26 ++++++++++---------- src/{lib => }/smart_pointers.rs | 26 ++++++++++---------- src/{lib => }/traits.rs | 0 10 files changed, 50 insertions(+), 55 deletions(-) rename src/{lib => }/builder.rs (69%) rename src/{lib => }/cli.rs (76%) rename src/{lib/conversion/mod.rs => conversion.rs} (97%) rename src/{lib => }/dispatch.rs (71%) rename src/{lib.rs_old => lib.rs} (100%) delete mode 100644 src/lib/mod.rs rename src/{lib => }/oop_pattern.rs (57%) rename src/{lib => }/smart_pointers.rs (73%) rename src/{lib => }/traits.rs (100%) diff --git a/src/lib/builder.rs b/src/builder.rs similarity index 69% rename from src/lib/builder.rs rename to src/builder.rs index bcda5bf..f4217ad 100644 --- a/src/lib/builder.rs +++ b/src/builder.rs @@ -1,25 +1,25 @@ #[derive(Debug)] -pub(crate) struct TaskManager { +pub struct TaskManager { state: String, count: usize, } impl TaskManager { /// Get task count - pub(crate) fn count(&self) -> &usize { + pub fn count(&self) -> &usize { &self.count } } #[derive(Default)] -pub(crate) struct TaskManagerBuilder { +pub struct TaskManagerBuilder { state: String, count: usize, } impl TaskManagerBuilder { /// Creates a new TaskManagerBuilder - pub(crate) fn new() -> Self { + pub fn new() -> Self { Self { state: "initialized".to_string(), count: 0, @@ -27,13 +27,13 @@ impl TaskManagerBuilder { } /// Sets the task count - pub(crate) fn count(mut self, value: usize) -> Self { + pub fn count(mut self, value: usize) -> Self { self.count = value; self } /// Creates a new TaskManager - pub(crate) fn build(self) -> TaskManager { + pub fn build(self) -> TaskManager { TaskManager { state: self.state, count: self.count, diff --git a/src/lib/cli.rs b/src/cli.rs similarity index 76% rename from src/lib/cli.rs rename to src/cli.rs index d1ed59d..bb78952 100644 --- a/src/lib/cli.rs +++ b/src/cli.rs @@ -5,13 +5,13 @@ use std::str::FromStr; /// Program to run rust tutorials #[derive(Parser, Debug)] #[clap(author, version, about, long_about = None)] -pub(crate) struct Args { +pub struct Args { #[clap(subcommand)] - pub(crate) command: Option, + pub command: Option, } #[derive(Subcommand, Debug)] -pub(crate) enum Commands { +pub enum Commands { /// Tutorial on dynamic dispatch Dispatch, @@ -31,6 +31,6 @@ pub(crate) enum Commands { Conversion, } -pub(crate) fn runner(mut mk: impl FnMut() -> Result) -> Result { +pub fn runner(mut mk: impl FnMut() -> Result) -> Result { mk() } diff --git a/src/lib/conversion/mod.rs b/src/conversion.rs similarity index 97% rename from src/lib/conversion/mod.rs rename to src/conversion.rs index f585ff9..81f2295 100644 --- a/src/lib/conversion/mod.rs +++ b/src/conversion.rs @@ -16,6 +16,9 @@ pub fn lesson_1() -> Result<()> { let resp2 = u16::from(value); assert_eq!(resp, resp2); + let formal = >::into(value); + let shorter = value as u16; + Ok(()) } diff --git a/src/lib/dispatch.rs b/src/dispatch.rs similarity index 71% rename from src/lib/dispatch.rs rename to src/dispatch.rs index 882a500..0a709ec 100644 --- a/src/lib/dispatch.rs +++ b/src/dispatch.rs @@ -1,5 +1,5 @@ /// Say hello in Norwegian -pub(crate) trait Hei { +pub trait Hei { fn hei(&self); fn weird(&self); @@ -37,23 +37,23 @@ impl Hei for String { } } -pub(crate) fn say_hei(s: &dyn Hei) { +pub fn say_hei(s: &dyn Hei) { s.hei() } -pub(crate) fn strlen>(s: S) -> usize { +pub fn strlen>(s: S) -> usize { s.as_ref().len() } -pub(crate) fn strlen2(s: String) -> usize { +pub fn strlen2(s: String) -> usize { s.len() } // examples of trait objects -pub(crate) fn strlen_dyn2(s: Box>) -> usize { +pub fn strlen_dyn2(s: Box>) -> usize { s.as_ref().as_ref().len() } -pub(crate) fn strlen_dyn(s: &dyn AsRef) -> usize { +pub fn strlen_dyn(s: &dyn AsRef) -> usize { s.as_ref().len() } diff --git a/src/lib.rs_old b/src/lib.rs similarity index 100% rename from src/lib.rs_old rename to src/lib.rs diff --git a/src/lib/mod.rs b/src/lib/mod.rs deleted file mode 100644 index 53ad4fa..0000000 --- a/src/lib/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub mod builder; -pub mod cli; -pub mod conversion; -pub mod dispatch; -pub mod oop_pattern; -pub mod smart_pointers; -pub mod traits; diff --git a/src/main.rs b/src/main.rs index 58dac8e..ca2eb83 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,14 +8,13 @@ use anyhow::{Error, Result}; use clap::Parser; -use lib::cli::{runner, Args, Commands}; -use lib::{builder::TaskManagerBuilder, dispatch::*, oop_pattern::*, smart_pointers::*, traits}; use log::{debug, info}; use std::io::Read; use std::sync::Arc; - -pub mod lib; -// pub mod sandbox; +use tutorials::cli::{runner, Args, Commands}; +use tutorials::{ + builder::TaskManagerBuilder, dispatch::*, oop_pattern::*, smart_pointers::*, traits, +}; fn main() -> Result<()> { env_logger::init(); @@ -131,7 +130,7 @@ fn main() -> Result<()> { Some(Commands::Conversion) => runner(|| { info!("Tutorial: Conversion\n"); - lib::conversion::runner()?; + tutorials::conversion::runner()?; Ok(()) })?, diff --git a/src/lib/oop_pattern.rs b/src/oop_pattern.rs similarity index 57% rename from src/lib/oop_pattern.rs rename to src/oop_pattern.rs index be0b36e..2826c62 100644 --- a/src/lib/oop_pattern.rs +++ b/src/oop_pattern.rs @@ -1,51 +1,51 @@ -pub(crate) struct Post { +pub struct Post { content: String, } -pub(crate) struct DraftPost { +pub struct DraftPost { content: String, } impl Post { - pub(crate) fn new() -> DraftPost { + pub fn new() -> DraftPost { DraftPost { content: String::new(), } } - pub(crate) fn content(&self) -> &str { + pub fn content(&self) -> &str { &self.content } } impl DraftPost { - pub(crate) fn add_text(&mut self, text: &str) { + pub fn add_text(&mut self, text: &str) { self.content.push_str(text); } - pub(crate) fn request_review(self) -> PendingReviewPost { + pub fn request_review(self) -> PendingReviewPost { PendingReviewPost { content: self.content, } } } -pub(crate) struct PendingReviewPost { +pub struct PendingReviewPost { content: String, } impl PendingReviewPost { - pub(crate) fn review(&self) -> &String { + pub fn review(&self) -> &String { &self.content } - pub(crate) fn approve(self) -> Post { + pub fn approve(self) -> Post { Post { content: self.content, } } - pub(crate) fn reject(self, changes: &str) -> RequestChangesPost { + pub fn reject(self, changes: &str) -> RequestChangesPost { RequestChangesPost { content: self.content, changes: changes.to_string(), @@ -53,17 +53,17 @@ impl PendingReviewPost { } } -pub(crate) struct RequestChangesPost { +pub struct RequestChangesPost { content: String, changes: String, } impl RequestChangesPost { - pub(crate) fn get_feedback(&self) -> String { + pub fn get_feedback(&self) -> String { format!("Make changes to '{}' as {}", &self.content, &self.changes) } - pub(crate) fn replace_text(&mut self, text: &str) -> PendingReviewPost { + pub fn replace_text(&mut self, text: &str) -> PendingReviewPost { PendingReviewPost { content: text.to_string(), } diff --git a/src/lib/smart_pointers.rs b/src/smart_pointers.rs similarity index 73% rename from src/lib/smart_pointers.rs rename to src/smart_pointers.rs index 50f3763..19e72f2 100644 --- a/src/lib/smart_pointers.rs +++ b/src/smart_pointers.rs @@ -3,7 +3,7 @@ use std::cell::UnsafeCell; // Further coverage of Cell, RefCell and Rc (https://youtu.be/8O0Nt9qY_vo): // https://gist.github.com/jonhoo/7cfdfe581e5108b79c2a4e9fbde38de8 -pub(crate) struct Cell { +pub struct Cell { value: UnsafeCell, } @@ -12,19 +12,19 @@ pub(crate) struct Cell { // https://doc.rust-lang.org/std/cell/struct.UnsafeCell.html#impl-Sync impl Cell { - pub(crate) fn new(value: T) -> Self { + pub fn new(value: T) -> Self { Cell { value: UnsafeCell::new(value), } } - pub(crate) fn set(&self, value: T) { + pub fn set(&self, value: T) { // SAFETY: we know no-one else is concurrently mutating self (because !Sync) // SAFETY: we're not invalidating any references as we are not sharing any. unsafe { *self.value.get() = value }; } - pub(crate) fn get(&self) -> T + pub fn get(&self) -> T where T: Copy, { @@ -35,49 +35,49 @@ impl Cell { } /// Contrived example storing a String as Vec -pub(crate) struct Message { +pub struct Message { content: String, bytes: Vec, } impl Message { - pub(crate) fn update(mut self, content: &str) -> Self { + pub fn update(mut self, content: &str) -> Self { let bytes: Vec = content.to_string().as_bytes().to_vec(); self.bytes = bytes; self } - pub(crate) fn content_from_bytes(&self) -> Option { + pub fn content_from_bytes(&self) -> Option { Some(String::from_utf8(self.bytes.clone()).ok()?) } - pub(crate) fn content(&self) -> &String { + pub fn content(&self) -> &String { &self.content } - pub(crate) fn bytes(&self) -> &Vec { + pub fn bytes(&self) -> &Vec { &self.bytes } } -pub(crate) struct MessageBuilder { +pub struct MessageBuilder { content: String, } impl MessageBuilder { - pub(crate) fn new() -> Self { + pub fn new() -> Self { Self { content: String::default(), } } - pub(crate) fn content(mut self, content: &str) -> Self { + pub fn content(mut self, content: &str) -> Self { self.content = content.to_string(); self } - pub(crate) fn build(&self) -> Message { + pub fn build(&self) -> Message { Message { content: self.content.to_string(), bytes: vec![0], diff --git a/src/lib/traits.rs b/src/traits.rs similarity index 100% rename from src/lib/traits.rs rename to src/traits.rs From 335b359291d71b2ed93fa8f8c9985cd0780f469e Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Mon, 21 Nov 2022 16:36:42 +0530 Subject: [PATCH 13/79] Example for use of traits and trait objects from `esp_idf_hal` --- src/traits.rs | 541 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 540 insertions(+), 1 deletion(-) diff --git a/src/traits.rs b/src/traits.rs index 0a16329..4bbd6bb 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -97,11 +97,550 @@ impl ErrorType for RTCDevice { pub fn runner() -> Result<()> { lesson_1_add_trait_bound_to_parameter(); - lesson_2(); + lesson_3::run()?; Ok(()) } +/// Use `esp_idf_hal` as an example for advanced used of Traits and trait objects +mod lesson_3 { + use crate::into_ref; + use anyhow::Result; + + mod core { + + #[macro_export] + #[allow(unused_macros)] + macro_rules! into_ref { + ($($name:ident),*) => { + $( + let $name = $name.into_ref(); + )* + } + } + + #[allow(unused_macros)] + macro_rules! impl_peripheral_trait { + ($type:ident) => { + unsafe impl Send for $type {} + + impl $crate::traits::lesson_3::peripheral::sealed::Sealed for $type {} + + impl $crate::traits::lesson_3::peripheral::Peripheral for $type { + type P = $type; + + #[inline] + unsafe fn clone_unchecked(&mut self) -> Self::P { + $type { ..*self } + } + } + }; + } + + #[allow(unused_macros)] + macro_rules! impl_peripheral { + ($type:ident) => { + pub struct $type(::core::marker::PhantomData<*const ()>); + + impl $type { + /// # Safety + /// + /// Care should be taken not to instnatiate this peripheralinstance, if it is already instantiated and used elsewhere + #[inline(always)] + pub unsafe fn new() -> Self { + $type(::core::marker::PhantomData) + } + } + + $crate::traits::lesson_3::core::impl_peripheral_trait!($type); + }; + } + + #[allow(unused_imports)] + pub(crate) use impl_peripheral; + #[allow(unused_imports)] + pub(crate) use impl_peripheral_trait; + #[allow(unused_imports)] + pub(crate) use into_ref; + } + + mod peripheral { + use core::marker::PhantomData; + use core::ops::{Deref, DerefMut}; + + pub struct PeripheralRef<'a, T> { + inner: T, + _lifetime: PhantomData<&'a mut T>, + } + + impl<'a, T> PeripheralRef<'a, T> { + #[inline] + pub fn new(inner: T) -> Self { + Self { + inner, + _lifetime: PhantomData, + } + } + + /// Unsafely clone (duplicate) a Peripheral singleton. + /// + /// # Safety + /// + /// This returns an owned clone of the Peripheral. You must manually ensure + /// only one copy of the Peripheral is in use at a time. For example, don't + /// create two SPI drivers on `SPI1`, because they will "fight" each other. + /// + /// You should strongly prefer using `reborrow()` instead. It returns a + /// `PeripheralRef` that borrows `self`, which allows the borrow checker + /// to enforce this at compile time. + pub unsafe fn clone_unchecked(&mut self) -> PeripheralRef<'a, T> + where + T: Peripheral

, + { + PeripheralRef::new(self.inner.clone_unchecked()) + } + + /// Reborrow into a "child" PeripheralRef. + /// + /// `self` will stay borrowed until the child PeripheralRef is dropped. + pub fn reborrow(&mut self) -> PeripheralRef<'_, T> + where + T: Peripheral

, + { + // safety: we're returning the clone inside a new PeripheralRef that borrows + // self, so user code can't use both at the same time. + PeripheralRef::new(unsafe { self.inner.clone_unchecked() }) + } + + /// Map the inner Peripheral using `Into`. + /// + /// This converts from `PeripheralRef<'a, T>` to `PeripheralRef<'a, U>`, using an + /// `Into` impl to convert from `T` to `U`. + /// + /// For example, this can be useful to degrade GPIO pins: converting from PeripheralRef<'a, PB11>` to `PeripheralRef<'a, AnyPin>`. + #[inline] + pub fn map_into(self) -> PeripheralRef<'a, U> + where + T: Into, + { + PeripheralRef { + inner: self.inner.into(), + _lifetime: PhantomData, + } + } + } + + impl<'a, T> Deref for PeripheralRef<'a, T> { + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.inner + } + } + + impl<'a, T> DerefMut for PeripheralRef<'a, T> { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } + } + + /// Trait for any type that can be used as a Peripheral of type `P`. + /// + /// This is used in driver constructors, to allow passing either owned Peripherals (e.g. `TWISPI0`), + /// or borrowed Peripherals (e.g. `&mut TWISPI0`). + /// + /// For example, if you have a driver with a constructor like this: + /// + /// ```ignore + /// impl<'d, T: Instance> Twim<'d, T> { + /// pub fn new( + /// twim: impl Peripheral

+ 'd, + /// irq: impl Peripheral

+ 'd, + /// sda: impl Peripheral

+ 'd, + /// scl: impl Peripheral

+ 'd, + /// config: Config, + /// ) -> Self { .. } + /// } + /// ``` + /// + /// You may call it with owned Peripherals, which yields an instance that can live forever (`'static`): + /// + /// ```ignore + /// let mut twi: Twim<'static, ...> = Twim::new(p.TWISPI0, irq, p.P0_03, p.P0_04, config); + /// ``` + /// + /// Or you may call it with borrowed Peripherals, which yields an instance that can only live for as long + /// as the borrows last: + /// + /// ```ignore + /// let mut twi: Twim<'_, ...> = Twim::new(&mut p.TWISPI0, &mut irq, &mut p.P0_03, &mut p.P0_04, config); + /// ``` + /// + /// # Implementation details, for HAL authors + /// + /// When writing a HAL, the intended way to use this trait is to take `impl Peripheral

` in + /// the HAL's public API (such as driver constructors), calling `.into_ref()` to obtain a `PeripheralRef`, + /// and storing that in the driver struct. + /// + /// `.into_ref()` on an owned `T` yields a `PeripheralRef<'static, T>`. + /// `.into_ref()` on an `&'a mut T` yields a `PeripheralRef<'a, T>`. + pub trait Peripheral: Sized + sealed::Sealed { + /// Peripheral singleton type + type P; + + /// Unsafely clone (duplicate) a Peripheral singleton. + /// + /// # Safety + /// + /// This returns an owned clone of the Peripheral. You must manually ensure + /// only one copy of the Peripheral is in use at a time. For example, don't + /// create two SPI drivers on `SPI1`, because they will "fight" each other. + /// + /// You should strongly prefer using `into_ref()` instead. It returns a + /// `PeripheralRef`, which allows the borrow checker to enforce this at compile time. + unsafe fn clone_unchecked(&mut self) -> Self::P; + + /// Convert a value into a `PeripheralRef`. + /// + /// When called on an owned `T`, yields a `PeripheralRef<'static, T>`. + /// When called on an `&'a mut T`, yields a `PeripheralRef<'a, T>`. + #[inline] + fn into_ref<'a>(mut self) -> PeripheralRef<'a, Self::P> + where + Self: 'a, + { + PeripheralRef::new(unsafe { self.clone_unchecked() }) + } + } + + impl sealed::Sealed for T {} + + impl Peripheral for T + where + T::Target: Peripheral, + { + type P = ::P; + + #[inline] + unsafe fn clone_unchecked(&mut self) -> Self::P { + self.deref_mut().clone_unchecked() + } + } + + pub(crate) mod sealed { + pub trait Sealed {} + } + } + + mod gpio { + use super::core::impl_peripheral_trait; + use super::peripheral::{Peripheral, PeripheralRef}; + use anyhow::Result; + use core::marker::PhantomData; + + /// A trait implemented by every pin instance + pub trait Pin: Peripheral

+ Sized + Send + 'static { + fn pin(&self) -> i32; + } + + /// A marker trait designating a pin which is capable of + /// operating as an input pin + pub trait InputPin: Pin + Into { + fn downgrade_input(self) -> AnyInputPin { + self.into() + } + } + + /// A marker trait designating a pin which is capable of + /// operating as an output pin + pub trait OutputPin: Pin + Into { + fn downgrade_output(self) -> AnyOutputPin { + self.into() + } + } + + /// A marker trait designating a pin which is capable of + /// operating as an input and output pin + pub trait IOPin: InputPin + OutputPin + Into { + fn downgrade(self) -> AnyIOPin { + self.into() + } + } + + /// Generic Gpio input-output pin + pub struct AnyIOPin { + pin: i32, + _p: PhantomData<*const ()>, + } + + impl AnyIOPin { + /// # Safety + /// + /// Care should be taken not to instantiate this Pin, if it is + /// already instantiated and used elsewhere, or if it is not set + /// already in the mode of operation which is being instantiated + pub unsafe fn new(pin: i32) -> Self { + Self { + pin, + _p: PhantomData, + } + } + } + + impl_peripheral_trait!(AnyIOPin); + + impl Pin for AnyIOPin { + fn pin(&self) -> i32 { + self.pin + } + } + + impl InputPin for AnyIOPin {} + impl OutputPin for AnyIOPin {} + + /// Generic Gpio input pin + pub struct AnyInputPin { + pin: i32, + _p: PhantomData<*const ()>, + } + + impl AnyInputPin { + /// # Safety + /// + /// Care should be taken not to instantiate this Pin, if it is + /// already instantiated and used elsewhere, or if it is not set + /// already in the mode of operation which is being instantiated + pub unsafe fn new(pin: i32) -> Self { + Self { + pin, + _p: PhantomData, + } + } + } + + impl_peripheral_trait!(AnyInputPin); + + impl Pin for AnyInputPin { + fn pin(&self) -> i32 { + self.pin + } + } + + impl InputPin for AnyInputPin {} + + impl From for AnyInputPin { + fn from(pin: AnyIOPin) -> Self { + unsafe { Self::new(pin.pin()) } + } + } + + /// Generic Gpio output pin + pub struct AnyOutputPin { + pin: i32, + _p: PhantomData<*const ()>, + } + + impl AnyOutputPin { + /// # Safety + /// + /// Care should be taken not to instantiate this Pin, if it is + /// already instantiated and used elsewhere, or if it is not set + /// already in the mode of operation which is being instantiated + pub unsafe fn new(pin: i32) -> Self { + Self { + pin, + _p: PhantomData, + } + } + } + + impl_peripheral_trait!(AnyOutputPin); + + impl Pin for AnyOutputPin { + fn pin(&self) -> i32 { + self.pin + } + } + + impl OutputPin for AnyOutputPin {} + + impl From for AnyOutputPin { + fn from(pin: AnyIOPin) -> Self { + unsafe { Self::new(pin.pin()) } + } + } + + pub struct Output; + pub struct Input; + + /// A driver for a GPIO pin. + /// + /// The driver can set the pin as a disconnected/disabled one, input, or output pin, or both or analog. + /// On some chips (i.e. esp32 and esp32s*), the driver can also set the pin in RTC IO mode. + /// Depending on the current operating mode, different sets of functions are available. + /// + /// The mode-setting depends on the capabilities of the pin as well, i.e. input-only pins cannot be set + /// into output or input-output mode. + pub struct PinDriver<'d, T: Pin, MODE> { + pin: PeripheralRef<'d, T>, + _mode: PhantomData, + } + + impl<'d, T: InputPin> PinDriver<'d, T, Input> { + /// Creates the driver for a pin in input state. + #[inline] + pub fn input(pin: impl Peripheral

+ 'd) -> Result { + crate::into_ref!(pin); + + Self { + pin, + _mode: PhantomData, + } + .into_input() + } + } + + impl<'d, T: OutputPin> PinDriver<'d, T, Output> { + /// Creates the driver for a pin in output state. + #[inline] + pub fn output(pin: impl Peripheral

+ 'd) -> Result { + crate::into_ref!(pin); + + Self { + pin, + _mode: PhantomData, + } + .into_output() + } + } + + impl<'d, T: Pin, MODE> PinDriver<'d, T, MODE> { + /// Returns the pin number. + pub fn pin(&self) -> i32 { + self.pin.pin() + } + + /// Put the pin into input mode. + #[inline] + pub fn into_input(self) -> Result> + where + T: InputPin, + { + self.into_mode("input") + } + + /// Put the pin into output mode. + #[inline] + pub fn into_output(self) -> Result> + where + T: OutputPin, + { + self.into_mode("output") + } + + #[inline] + fn into_mode(mut self, mode: &str) -> Result> + where + T: Pin, + { + let pin = unsafe { self.pin.clone_unchecked() }; + + drop(self); + + if mode != "disabled" { + // esp!(unsafe { gpio_set_direction(pin.pin(), mode) })?; + } + + Ok(PinDriver { + pin, + _mode: PhantomData, + }) + } + } + + unsafe impl<'d, T: Pin, MODE> Send for PinDriver<'d, T, MODE> {} + + macro_rules! impl_input { + ($pxi:ident: $pin:expr) => { + $crate::traits::lesson_3::core::impl_peripheral!($pxi); + + impl $crate::traits::lesson_3::gpio::Pin for $pxi { + fn pin(&self) -> i32 { + $pin + } + } + + impl InputPin for $pxi {} + + impl From<$pxi> for AnyInputPin { + fn from(pin: $pxi) -> Self { + unsafe { Self::new(pin.pin()) } + } + } + }; + } + + macro_rules! impl_input_output { + ($pxi:ident: $pin:expr) => { + $crate::traits::lesson_3::gpio::impl_input!($pxi: $pin); + + impl OutputPin for $pxi {} + + impl IOPin for $pxi {} + + impl From<$pxi> for AnyOutputPin { + fn from(pin: $pxi) -> Self { + unsafe { Self::new(pin.pin()) } + } + } + + impl From<$pxi> for AnyIOPin { + fn from(pin: $pxi) -> Self { + unsafe { Self::new(pin.pin()) } + } + } + }; + } + + macro_rules! pin { + ($pxi:ident: $pin:expr, Input) => { + $crate::traits::lesson_3::gpio::impl_input!($pxi: $pin); + }; + + ($pxi:ident: $pin:expr, IO) => { + $crate::traits::lesson_3::gpio::impl_input_output!($pxi: $pin); + }; + } + + #[allow(unused_imports)] + pub(crate) use impl_input; + #[allow(unused_imports)] + pub(crate) use impl_input_output; + #[allow(unused_imports)] + pub(crate) use pin; + } + + pub fn run() -> Result<()> { + use self::core::*; + use gpio::*; + use peripheral::{Peripheral, PeripheralRef}; + + gpio::pin!(Gpio0:0, IO); + gpio::pin!(Gpio34:34, Input); + + unsafe { + let pin = Gpio0::new(); + gpio::PinDriver::output(pin); + + let pin2 = Gpio34::new(); + gpio::PinDriver::input(pin2); + } + + Ok(()) + } +} fn test_mut_t(device: T, message: &str) where From 22cc4f36c2be82fccf2c8a571355bc755deb4bb9 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Mon, 21 Nov 2022 17:19:36 +0530 Subject: [PATCH 14/79] Update example for using PeripheralRef --- src/traits.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/traits.rs b/src/traits.rs index 4bbd6bb..edaf16f 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -104,7 +104,10 @@ pub fn runner() -> Result<()> { } /// Use `esp_idf_hal` as an example for advanced used of Traits and trait objects mod lesson_3 { - use crate::into_ref; + use crate::{ + into_ref, + traits::lesson_3::peripheral::{Peripheral, PeripheralRef}, + }; use anyhow::Result; mod core { @@ -623,19 +626,23 @@ mod lesson_3 { } pub fn run() -> Result<()> { - use self::core::*; use gpio::*; - use peripheral::{Peripheral, PeripheralRef}; + use std::ops::Deref; gpio::pin!(Gpio0:0, IO); gpio::pin!(Gpio34:34, Input); unsafe { let pin = Gpio0::new(); - gpio::PinDriver::output(pin); + gpio::PinDriver::output(pin)?; - let pin2 = Gpio34::new(); - gpio::PinDriver::input(pin2); + let mut pin2 = Gpio34::new(); + let pin2_ref = pin2.into_ref(); + + let driver = gpio::PinDriver::input(pin2_ref)?; + driver.into_input()?; + + // let pin2_d = pin2_ref.deref(); } Ok(()) From c6ccbd49599a3c12491a45b614f161be4d698088 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Thu, 8 Dec 2022 09:12:08 +0530 Subject: [PATCH 15/79] Cleanup --- src/traits.rs | 74 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 9 deletions(-) diff --git a/src/traits.rs b/src/traits.rs index edaf16f..0ab0db8 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -337,7 +337,7 @@ mod lesson_3 { } } - mod gpio { + pub(crate) mod gpio { use super::core::impl_peripheral_trait; use super::peripheral::{Peripheral, PeripheralRef}; use anyhow::Result; @@ -475,8 +475,32 @@ mod lesson_3 { } } + pub trait InputMode { + const RTC: bool; + } + pub trait OutputMode { + const RTC: bool; + } + pub struct Output; pub struct Input; + pub struct InputOutput; + + impl InputMode for Input { + const RTC: bool = false; + } + + impl InputMode for InputOutput { + const RTC: bool = false; + } + + impl OutputMode for Output { + const RTC: bool = false; + } + + impl OutputMode for InputOutput { + const RTC: bool = false; + } /// A driver for a GPIO pin. /// @@ -505,6 +529,20 @@ mod lesson_3 { } } + impl<'d, T: InputPin + OutputPin> PinDriver<'d, T, InputOutput> { + /// Creates the driver for a pin in input-output state. + #[inline] + pub fn input_output(pin: impl Peripheral

+ 'd) -> Result { + crate::into_ref!(pin); + + Self { + pin, + _mode: PhantomData, + } + .into_input_output() + } + } + impl<'d, T: OutputPin> PinDriver<'d, T, Output> { /// Creates the driver for a pin in output state. #[inline] @@ -534,6 +572,15 @@ mod lesson_3 { self.into_mode("input") } + /// Put the pin into input + output mode. + #[inline] + pub fn into_input_output(self) -> Result> + where + T: InputPin + OutputPin, + { + self.into_mode("input_output") + } + /// Put the pin into output mode. #[inline] pub fn into_output(self) -> Result> @@ -561,6 +608,16 @@ mod lesson_3 { _mode: PhantomData, }) } + + /// Toggle pin output + #[inline] + pub fn toggle(&mut self) -> Result<()> + where + MODE: OutputMode, + { + // Todo + Ok(()) + } } unsafe impl<'d, T: Pin, MODE> Send for PinDriver<'d, T, MODE> {} @@ -633,16 +690,15 @@ mod lesson_3 { gpio::pin!(Gpio34:34, Input); unsafe { - let pin = Gpio0::new(); - gpio::PinDriver::output(pin)?; - - let mut pin2 = Gpio34::new(); - let pin2_ref = pin2.into_ref(); + let gpio0 = Gpio0::new(); + let mut io_pin = gpio0.downgrade(); - let driver = gpio::PinDriver::input(pin2_ref)?; - driver.into_input()?; + { + let mut pin_driver = PinDriver::input_output(&mut io_pin)?; - // let pin2_d = pin2_ref.deref(); + pin_driver.toggle()?; + } + // NOTE: by using a block here, the `pin_driver` is implicitly dropped when execution reaches the end of the block. This allowes for the drowngraded pin to be used without clashing with the borrow-checker. } Ok(()) From 9bb2638c7539aee4a8770160e4b52b4e99c92f26 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sat, 22 Apr 2023 16:39:47 +0530 Subject: [PATCH 16/79] Add example of using closure to spawn parallel tasks with Tokio --- Cargo.toml | 6 ++ src/cli.rs | 3 + src/closures.rs | 92 +++++++++++++++++++++++++++++++ src/lib.rs | 1 + src/main.rs | 144 ++++++++++++++++++++++++++---------------------- 5 files changed, 180 insertions(+), 66 deletions(-) create mode 100644 src/closures.rs diff --git a/Cargo.toml b/Cargo.toml index 51a86b0..9817b11 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,12 @@ publish = false [dependencies] anyhow = "1.0.66" +futures = "0.3.19" +hyper = { version = "0.14.16", features = ["full"] } +hyper-rustls = { version = "0.23.0", features = ["http1", "http2"] } +tokio = { version = "1.17.0", features = ["full"] } +tokio-util = "0.7.7" +rustls = "0.21.0" clap = { version = "3", features = ["derive"] } dotenv = "0.15.0" env_logger = "0.9.0" diff --git a/src/cli.rs b/src/cli.rs index bb78952..8c092be 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -29,6 +29,9 @@ pub enum Commands { /// Conversion Conversion, + + /// Closures + Closures, } pub fn runner(mut mk: impl FnMut() -> Result) -> Result { diff --git a/src/closures.rs b/src/closures.rs new file mode 100644 index 0000000..679d077 --- /dev/null +++ b/src/closures.rs @@ -0,0 +1,92 @@ +use anyhow::{Error, Result}; +use log::info; + +#[tokio::main] +pub async fn runner() -> Result<()> { + lesson_1::run().await?; + + Ok(()) +} + +mod lesson_1 { + use super::*; + use std::{ + collections::BTreeMap, + fmt::{self, Debug, Display}, + sync::Arc, + time::Duration, + }; + use tokio::{sync::mpsc, task::JoinHandle}; + + #[derive(Debug)] + pub enum ActorMessage { + RunTask { + document: String, + rows: Vec, + alert: Option>>, + timestamp: String, + }, + } + + pub struct MyActor { + receiver: mpsc::Receiver, + next_id: u32, + } + + impl MyActor { + pub fn new(receiver: mpsc::Receiver) -> Self { + MyActor { + receiver, + next_id: 0, + } + } + pub fn spawn_tasks(mut f: F, d: D, r: R) -> JoinHandle<()> + where + F: FnMut(D, R) -> JoinHandle<()>, + { + f(d, r) + } + + async fn run_in_parallel( + task_name: TaskName, + items: ItemCollection, + mut fut: &impl Fn(TaskName, ItemCollection::Item) -> JoinHandle<()>, + ) -> Vec<()> + where + TaskName: Display + Clone, + ItemCollection: IntoIterator, + { + let futures: Vec<_> = items + .into_iter() + .map(|row| Self::spawn_tasks(&mut fut, task_name.clone(), row)) + .collect(); + + // do these futures in parallel and return them + let mut res = Vec::with_capacity(futures.len()); + for f in futures.into_iter() { + log::info!("run_in_parallel(): {:#?}", &f); + f.await.expect("Run `do_task` as a parallel task"); + res.push(()); + } + + res + } + + async fn do_task(task_name: String, mut el: u32) { + log::info!("do_task(): {} / {}", task_name, el); + } + } + + pub async fn run() -> Result<()> { + let task = "test".to_string(); + let items: Vec = vec![0, 1, 2]; + + let c = |t, elem| tokio::spawn(MyActor::do_task(t, elem)); + + let results = MyActor::run_in_parallel::, u32>(task, items, &c).await; + + log::info!("run(): done."); + + Ok(()) + } +} diff --git a/src/lib.rs b/src/lib.rs index 29f68c1..b8468f7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,6 @@ pub mod builder; pub mod cli; +pub mod closures; pub mod conversion; pub mod dispatch; pub mod oop_pattern; diff --git a/src/main.rs b/src/main.rs index ca2eb83..1bdad70 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,7 +13,7 @@ use std::io::Read; use std::sync::Arc; use tutorials::cli::{runner, Args, Commands}; use tutorials::{ - builder::TaskManagerBuilder, dispatch::*, oop_pattern::*, smart_pointers::*, traits, + builder::TaskManagerBuilder, closures, dispatch::*, oop_pattern::*, smart_pointers::*, traits, }; fn main() -> Result<()> { @@ -22,32 +22,34 @@ fn main() -> Result<()> { let cli = Args::parse(); match &cli.command { - Some(Commands::Dispatch) => runner(|| { - info!("Tutorial: Dynamic dispatch\n"); + Some(Commands::Dispatch) => { + runner(|| { + info!("Tutorial: Dynamic dispatch\n"); - let x: Box> = Box::new("hello".to_string()); - strlen_dyn2(x); + let x: Box> = Box::new("hello".to_string()); + strlen_dyn2(x); - // Use go-through pointer-indirection for something on the stack - let x: &dyn AsRef = &"hello".to_string(); - strlen_dyn(x); + // Use go-through pointer-indirection for something on the stack + let x: &dyn AsRef = &"hello".to_string(); + strlen_dyn(x); - // Use our Hei trait - let x: &dyn Hei = &"hei".to_string(); - x.weird(); - //x.need_sized(); // This is not object safe and therefore cannot be called on a trait-object - say_hei(x); + // Use our Hei trait + let x: &dyn Hei = &"hei".to_string(); + x.weird(); + //x.need_sized(); // This is not object safe and therefore cannot be called on a trait-object + say_hei(x); - // Demonstrate that sized functions work just fine on any standard implementation of the trait - let message = String::from("hello!"); - message.need_sized().to_string(); + // Demonstrate that sized functions work just fine on any standard implementation of the trait + let message = String::from("hello!"); + message.need_sized().to_string(); - let x: &dyn Hei = &"hei"; - x.weird(); - say_hei(x); + let x: &dyn Hei = &"hei"; + x.weird(); + say_hei(x); - Ok(()) - })?, + Ok(()) + })? + } Some(Commands::Builder) => runner(|| { info!("Tutorial: Builder pattern\n"); @@ -58,68 +60,71 @@ fn main() -> Result<()> { Ok(()) })?, - Some(Commands::TypeState) => { - runner(|| { - info!("Tutorial: OOP design pattern with Type State\n"); + Some(Commands::TypeState) => runner(|| { + info!("Tutorial: OOP design pattern with Type State\n"); - let mut post = Post::new(); - post.add_text("I ate a salad for lunch today"); + let mut post = Post::new(); + post.add_text("I ate a salad for lunch today"); - let post = post.request_review(); - assert_eq!("I ate a salad for lunch today", post.review()); + let post = post.request_review(); + assert_eq!("I ate a salad for lunch today", post.review()); - let mut post = post.reject("Salad isn't available today"); - assert_eq!("Make changes to 'I ate a salad for lunch today' as Salad isn't available today", post.get_feedback()); + let mut post = post.reject("Salad isn't available today"); + assert_eq!( + "Make changes to 'I ate a salad for lunch today' as Salad isn't available today", + post.get_feedback() + ); - let post = post.replace_text("I ate fish at lunch"); + let post = post.replace_text("I ate fish at lunch"); - let post = post.approve(); - assert_eq!("I ate fish at lunch", post.content()); - Ok(()) - })? - } - Some(Commands::SmartPointers) => runner(|| { - info!("Tutorial: Smart pointers\n"); + let post = post.approve(); + assert_eq!("I ate fish at lunch", post.content()); + Ok(()) + })?, + Some(Commands::SmartPointers) => { + runner(|| { + info!("Tutorial: Smart pointers\n"); - let new_message = MessageBuilder::new().content("hello").build(); - let new_message = new_message.update("foo"); + let new_message = MessageBuilder::new().content("hello").build(); + let new_message = new_message.update("foo"); - let byte_zero: u8 = 0; - assert_ne!(new_message.bytes(), &vec![byte_zero]); + let byte_zero: u8 = 0; + assert_ne!(new_message.bytes(), &vec![byte_zero]); - let message = new_message.update("Häagen-Dazs"); - assert_eq!( - message.content_from_bytes().unwrap(), - "Häagen-Dazs".to_string() - ); + let message = new_message.update("Häagen-Dazs"); + assert_eq!( + message.content_from_bytes().unwrap(), + "Häagen-Dazs".to_string() + ); - // Example of using a new-type to implement the Into trait - struct BytesToString { - value: String, - } + // Example of using a new-type to implement the Into trait + struct BytesToString { + value: String, + } - impl BytesToString { - pub(crate) fn new(value: &Vec) -> Self { - Self { - value: String::from_utf8(value.clone()).unwrap_or_default(), + impl BytesToString { + pub(crate) fn new(value: &Vec) -> Self { + Self { + value: String::from_utf8(value.clone()).unwrap_or_default(), + } } } - } - impl Into for BytesToString { - fn into(self) -> String { - self.value + impl Into for BytesToString { + fn into(self) -> String { + self.value + } } - } - let x = Cell::new(message.bytes()); + let x = Cell::new(message.bytes()); - // BytesToString is used as a new-type to convert Vec to a String - let contents: String = BytesToString::new(x.get()).into(); - assert_eq!(contents, "Häagen-Dazs".to_string()); + // BytesToString is used as a new-type to convert Vec to a String + let contents: String = BytesToString::new(x.get()).into(); + assert_eq!(contents, "Häagen-Dazs".to_string()); - Ok(()) - })?, + Ok(()) + })? + } Some(Commands::Traits) => runner(|| { info!("Tutorial: Traits\n"); @@ -134,6 +139,13 @@ fn main() -> Result<()> { Ok(()) })?, + Some(Commands::Closures) => runner(|| { + info!("Tutorial: Closures\n"); + + closures::runner()?; + + Ok(()) + })?, _ => info!("Command not found"), }; From 3f4f00a1f134c121c0eed33c049c42dc973d114e Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Tue, 16 May 2023 09:56:20 +0530 Subject: [PATCH 17/79] Add examples on using static/dynamic despatch with closures --- src/closures.rs | 63 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/src/closures.rs b/src/closures.rs index 679d077..9393092 100644 --- a/src/closures.rs +++ b/src/closures.rs @@ -4,6 +4,8 @@ use log::info; #[tokio::main] pub async fn runner() -> Result<()> { lesson_1::run().await?; + lesson_2::run().await?; + lesson_3::run().await?; Ok(()) } @@ -90,3 +92,64 @@ mod lesson_1 { Ok(()) } } + +mod lesson_2 { + use super::*; + use std::{fmt::Debug, fs::File, io::Write}; + trait Writeable: Write + Debug {} + impl Writeable for File {} + + fn foo(f: &mut (impl FnMut(i32) -> i32 + ?Sized)) { + dbg!(f(5)); + } + + fn test(v: i32) -> i32 { + dbg!(v) + } + + fn process_file(f: &mut (dyn Writeable)) { + dbg!(f); + } + + pub async fn run() -> Result<()> { + foo(&mut test); // static dispatch + + let dyn_func: &mut dyn FnMut(i32) -> i32 = &mut test; + foo(dyn_func); // dynamic dispatch + + // example with Write trait object + let dyn_f: &mut dyn Writeable = &mut File::create("/tmp/test.log").unwrap(); + process_file(dyn_f); + + log::info!("run(): done."); + + Ok(()) + } +} + +mod lesson_3 { + use super::*; + + pub async fn run() -> Result<()> { + fn closures( + mut f1: F1, + mut f2: F2, + mut f3: F3, + mut f4: impl FnMut(usize) -> usize, + ) -> i32 + where + F1: FnMut() -> f32, + F2: FnMut(i32) -> f32, + F3: FnMut(i32, i32) -> f32, + { + // (f1() + f2(10) + f3(20, 30)) as i32 + f4(10) as i32 + } + + let x = closures(|| 0.1, |x| (2 * x) as f32, |x, y| (x + y) as f32, |k| k); + + log::info!("run(): x: {}", x); + + Ok(()) + } +} From d94e1ad1a7c0de36e4f2f0d84054208887d6f125 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Tue, 23 May 2023 08:58:55 +0530 Subject: [PATCH 18/79] Add lesson 4 --- src/closures.rs | 54 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/src/closures.rs b/src/closures.rs index 9393092..667beec 100644 --- a/src/closures.rs +++ b/src/closures.rs @@ -6,6 +6,7 @@ pub async fn runner() -> Result<()> { lesson_1::run().await?; lesson_2::run().await?; lesson_3::run().await?; + lesson_4::run()?; Ok(()) } @@ -153,3 +154,56 @@ mod lesson_3 { Ok(()) } } + +mod lesson_4 { + use super::*; + + /* + Least Most + restrictive: restrictive + FnOnce() --> FnMut() --> Fn() + T &mut T &T + */ + + fn exec_once<'a, F: FnOnce(&'a str)>(mut f: F) { + f("hola") + } + + // https://practice.rs/functional-programing/closure.html#fn-fnmut-fnonce + pub fn run() -> Result<()> { + // Closure over FnOnce + let mut s = String::new(); + let c = |str| { + s; // closure is `FnOnce` because it moves the variable `s` out of its environment + () + }; + exec_once(c); + // exec_once(&c); // not allowed + + // Closure over FnMut + let mut s = String::new(); + let mut c = |str| { + s.push_str(str); + // closure is `FnMut` because it mutates the variable `s` here + // this closure implements `FnMut`, not `Fn` + }; + exec_once(&mut c); + exec_once(c); + // exec_once(&c); // not allowed + + // Closure over Fn + let mut s = String::new(); + let mut c = |str| { + let _ = &s; + () + }; + exec_once(c); + exec_once(&c); + exec_once(&mut c); + + // exec_mut(&mut update_string2); + // assert_eq!(s, "hola".to_string()); + + Ok(()) + } +} From 4f23d2a40f3616c0a6defa2b10c4c369a7ba6d1a Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sat, 15 Jul 2023 10:36:06 +0530 Subject: [PATCH 19/79] Add example for object-safe approach to calling generic method on trait object --- src/traits.rs | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/src/traits.rs b/src/traits.rs index 0ab0db8..5a09e33 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -99,6 +99,7 @@ pub fn runner() -> Result<()> { lesson_1_add_trait_bound_to_parameter(); lesson_2(); lesson_3::run()?; + lesson_4::run(); Ok(()) } @@ -743,3 +744,99 @@ fn lesson_1_add_trait_bound_to_parameter() { // | ^^^^^^^ required by this bound in `x` let resp = x(Box::new(device)); } + +// https://github.com/dtolnay/erased-serde/blob/master/explanation/main.rs +mod lesson_4 { + ///////////////////////////////////////////////////////////////////// + // Suppose these are the real traits from Serde. + + trait Querializer {} + + trait Generic { + // Not object safe because of this generic method. + fn generic_fn(&self, querializer: Q); + } + + // Note: extra constraint `where T: Querializer` does not seem to be needed. + impl<'a, T: ?Sized> Querializer for &'a T {} + + impl<'a, T: ?Sized> Generic for Box + where + T: Generic, + { + fn generic_fn(&self, querializer: Q) { + println!("generic_fn() on Box via the Generic Trait"); + + (**self).generic_fn(querializer) + } + } + + ///////////////////////////////////////////////////////////////////// + // This is an object-safe equivalent that interoperates seamlessly. + + trait ErasedGeneric { + fn erased_fn(&self, querializer: &dyn Querializer); + } + + impl Generic for dyn ErasedGeneric { + // Depending on the trait method signatures and the upstream + // impls, could also implement for: + // + // - &'a dyn ErasedGeneric + // - &'a (dyn ErasedGeneric + Send) + // - &'a (dyn ErasedGeneric + Sync) + // - &'a (dyn ErasedGeneric + Send + Sync) + // - Box + // - Box + // - Box + // - Box + fn generic_fn(&self, querializer: Q) { + self.erased_fn(&querializer) + } + } + + impl ErasedGeneric for T + where + T: Generic, + { + fn erased_fn(&self, querializer: &dyn Querializer) { + self.generic_fn(querializer) + } + } + + pub fn run() { + struct T; + impl Querializer for T {} + + #[derive(Debug)] + struct S { + size: usize, + }; + impl Generic for S { + fn generic_fn(&self, _querializer: Q) { + println!("querying the real S"); + } + } + + impl Generic for &S { + fn generic_fn(&self, _querializer: Q) { + println!("querying the real &S"); + let s = *self; + + dbg!("{:#?}", s); + } + } + + // Construct a trait object. + let trait_object: Box = Box::new(S { size: 0 }); + + // Seamlessly invoke the generic method on the trait object. + // + // THIS LINE LOOKS LIKE MAGIC. We have a value of type trait + // object and we are invoking a generic method on it. + trait_object.generic_fn(T); + + let trait_object: Box = Box::new(&S { size: 0 }); + trait_object.generic_fn(T); + } +} From 353781a690967dc76e167140baa73dd1d9a3ce1a Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sat, 15 Jul 2023 10:49:26 +0530 Subject: [PATCH 20/79] Cleanup --- src/traits.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/traits.rs b/src/traits.rs index 5a09e33..cee9113 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -765,7 +765,7 @@ mod lesson_4 { T: Generic, { fn generic_fn(&self, querializer: Q) { - println!("generic_fn() on Box via the Generic Trait"); + println!("impl<'a, T: ?Sized> Generic for Box: calling generic_fn()"); (**self).generic_fn(querializer) } @@ -791,6 +791,7 @@ mod lesson_4 { // - Box // - Box fn generic_fn(&self, querializer: Q) { + println!("impl Generic for dyn ErasedGeneric: call self.erased_fn()"); self.erased_fn(&querializer) } } @@ -800,6 +801,7 @@ mod lesson_4 { T: Generic, { fn erased_fn(&self, querializer: &dyn Querializer) { + println!("erased_fn() caling self.generic_fn"); self.generic_fn(querializer) } } @@ -836,6 +838,7 @@ mod lesson_4 { // object and we are invoking a generic method on it. trait_object.generic_fn(T); + println!(""); let trait_object: Box = Box::new(&S { size: 0 }); trait_object.generic_fn(T); } From 87cf99bd0c5c43302f04a6f657cd9289c54baab7 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sun, 16 Jul 2023 11:58:54 +0530 Subject: [PATCH 21/79] Add example with downcasting trait objects using dyn Any --- src/traits.rs | 132 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 131 insertions(+), 1 deletion(-) diff --git a/src/traits.rs b/src/traits.rs index cee9113..57d909f 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -99,7 +99,8 @@ pub fn runner() -> Result<()> { lesson_1_add_trait_bound_to_parameter(); lesson_2(); lesson_3::run()?; - lesson_4::run(); + // lesson_4::run(); + lesson_5::run(); Ok(()) } @@ -843,3 +844,132 @@ mod lesson_4 { trait_object.generic_fn(T); } } + +mod lesson_5 { + use std::any::Any; + ///////////////////////////////////////////////////////////////////// + // Suppose these are the real traits from Serde. + + trait Task {} + + trait Generic { + // Not object safe because of this generic method. + fn generic_fn(&self, task: Q); + } + + // Note: extra constraint `where T: Task` does not seem to be needed. + impl<'a, T: ?Sized> Task for &'a T {} + + impl<'a, T: ?Sized> Generic for Box + where + T: Generic, + { + fn generic_fn(&self, task: Q) { + println!("impl<'a, T: ?Sized> Generic for Box: calling generic_fn()"); + + (**self).generic_fn(task) + } + } + + ///////////////////////////////////////////////////////////////////// + // This is an object-safe equivalent that interoperates seamlessly. + + trait AsAny { + fn as_any(&self) -> &dyn Any; + } + + trait ErasedGeneric { + fn erased_fn(&self, task: &dyn Task); + + fn as_any(&self) -> &dyn Any; + } + + impl Generic for dyn ErasedGeneric { + // Depending on the trait method signatures and the upstream + // impls, could also implement for: + // + // - &'a dyn ErasedGeneric + // - &'a (dyn ErasedGeneric + Send) + // - &'a (dyn ErasedGeneric + Sync) + // - &'a (dyn ErasedGeneric + Send + Sync) + // - Box + // - Box + // - Box + // - Box + fn generic_fn(&self, task: Q) { + println!("impl Generic for dyn ErasedGeneric: call self.erased_fn()"); + self.erased_fn(&task) + } + } + + impl ErasedGeneric for T + where + T: Generic + 'static, + { + fn erased_fn(&self, task: &dyn Task) { + println!("erased_fn() caling self.generic_fn"); + self.generic_fn(task) + } + + fn as_any(&self) -> &dyn Any { + self + } + } + + pub fn run() { + #[derive(Debug)] + struct T; + impl Task for T {} + + #[derive(Debug)] + struct S { + size: usize, + }; + impl Generic for S { + fn generic_fn(&self, _task: Q) { + println!("querying the real S"); + + let s = self; + + dbg!("{:#?}", s); + } + } + + #[derive(Debug)] + struct M { + size: usize, + }; + impl Generic for M { + fn generic_fn(&self, _task: Q) { + println!("querying the real M"); + + let s = self; + + dbg!("{:#?}", s); + } + } + + struct Tasks { + tasks: Vec>, + } + + // Construct a trait object. + let mut tasks: Vec> = vec![]; + let trait_object1: Box = Box::new(S { size: 0 }); + let trait_object2: Box = Box::new(M { size: 0 }); + tasks.push(trait_object1); + tasks.push(trait_object2); + + for (index, task) in tasks.iter().enumerate() { + // task.generic_fn(T); + + if let Some(inner) = task.as_ref().as_any().downcast_ref::() { + println!("Downcast index {index} of type S: {:#?}", inner); + } + + if let Some(inner) = task.as_ref().as_any().downcast_ref::() { + println!("Downcast index {index} of type M: {:#?}", inner); + } + } + } +} From 79eda38cd3fc1d72d4005a7923da015128c45efe Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Thu, 10 Aug 2023 10:02:18 +0530 Subject: [PATCH 22/79] Add lesson 6 - digging into Axum --- Cargo.toml | 10 +++++ src/closures.rs | 2 +- src/traits.rs | 113 ++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 120 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9817b11..38a2aa0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,16 @@ clap = { version = "3", features = ["derive"] } dotenv = "0.15.0" env_logger = "0.9.0" log = "0.4.16" +http-body = "0.4.5" + +axum = { version = "0.6.18", features = ["headers", "tower-log"] } +axum-extra = "0.1.2" + +# Axum builds on the types in Tower +tower = { version = "0.4.13", features = ["limit", "load-shed", "filter", "util"] } +tower-http = { version = "0.4.0", features = ["trace", "cors", "catch-panic"] } +tower-layer = "0.3.1" +tower-service = "0.3.1" [dev-dependencies] test-log = "0.2.10" diff --git a/src/closures.rs b/src/closures.rs index 667beec..051d032 100644 --- a/src/closures.rs +++ b/src/closures.rs @@ -75,7 +75,7 @@ mod lesson_1 { res } - async fn do_task(task_name: String, mut el: u32) { + async fn do_task(task_name: String, el: u32) { log::info!("do_task(): {} / {}", task_name, el); } } diff --git a/src/traits.rs b/src/traits.rs index 57d909f..7a26eca 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -99,9 +99,11 @@ pub fn runner() -> Result<()> { lesson_1_add_trait_bound_to_parameter(); lesson_2(); lesson_3::run()?; - // lesson_4::run(); + // lesson_4::run();``````````````````````````````````` lesson_5::run(); + lesson6::run(); + Ok(()) } /// Use `esp_idf_hal` as an example for advanced used of Traits and trait objects @@ -814,7 +816,7 @@ mod lesson_4 { #[derive(Debug)] struct S { size: usize, - }; + } impl Generic for S { fn generic_fn(&self, _querializer: Q) { println!("querying the real S"); @@ -924,7 +926,7 @@ mod lesson_5 { #[derive(Debug)] struct S { size: usize, - }; + } impl Generic for S { fn generic_fn(&self, _task: Q) { println!("querying the real S"); @@ -938,7 +940,7 @@ mod lesson_5 { #[derive(Debug)] struct M { size: usize, - }; + } impl Generic for M { fn generic_fn(&self, _task: Q) { println!("querying the real M"); @@ -973,3 +975,106 @@ mod lesson_5 { } } } + +mod lesson6 { + use axum::{ + body::Body, + extract::{FromRequest, FromRequestParts, State}, + handler::{HandlerService as OtherHandlerService, Layered as OtherLayered}, + response::IntoResponse, + routing::get, + Router, + }; + use futures::Future; + use hyper::body::Buf; + use hyper::http::{Request, Response}; + use std::{convert::Infallible, fmt, marker::PhantomData, pin::Pin}; + use tower_layer::Layer; + use tower_service::Service; + + pub struct Layered { + pub layer: L, + pub handler: H, + pub _marker: PhantomData (T, S, B, B2)>, + } + + pub struct HandlerService { + pub handler: H, + pub state: S, + pub _marker: PhantomData (T, B)>, + } + + pub trait Handler: Clone + Send + Sized + 'static { + /// The type of future calling this handler returns. + type Future: Future> + Send + 'static; + + type RespType; + + /// Call the handler with the given request. + fn call(self, req: Request, state: S) -> Self::Future; + + /// Apply a [`tower::Layer`] to the handler. + fn layer(self, layer: L) -> Layered + where + L: Layer> + Clone, + L::Service: Service>, + { + Layered { + layer, + handler: self, + _marker: PhantomData, + } + } + + /// Convert the handler into a [`Service`] by providing the state + fn with_state( + self, + state: S, + _: PhantomData (T, B)>, + ) -> HandlerService { + // NOTE calling `new` is private + // HandlerService::new(self, state) + HandlerService { + handler: self, + state, + _marker: PhantomData, + } + } + } + + impl Handler<((),), S> for F + where + F: FnOnce() -> Fut + Clone + Send + 'static, + Fut: Future + Send, + Res: IntoResponse, + { + type Future = Pin> + Send>>; + type RespType = (); + + fn call(self, _req: Request, _state: S) -> Self::Future { + Box::pin(async move { + let real_resp = self().await.into_response(); + + let response = Response::builder() + .status(200) + .header("X-Custom-Foo", "Bar") + .body(()) + .unwrap(); + + response + }) + } + } + + pub fn run() { + // NOTE this breaks as we clash with existing expansion in axum. + // let app = Router::new().route("/", get(root)); + // + // 403 | top_level_handler_fn!(get, GET); + // | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `get` + } + + pub fn root() { + () + } +} From c4f483771676ab36685e8024159769212af143cc Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sat, 2 Sep 2023 11:51:18 +0530 Subject: [PATCH 23/79] Update Traits lesson 7 --- src/traits.rs | 103 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 101 insertions(+), 2 deletions(-) diff --git a/src/traits.rs b/src/traits.rs index 7a26eca..a0c3228 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -100,9 +100,11 @@ pub fn runner() -> Result<()> { lesson_2(); lesson_3::run()?; // lesson_4::run();``````````````````````````````````` - lesson_5::run(); + // lesson_5::run(); - lesson6::run(); + // lesson6::run(); + lesson7::run(); + lesson8::run(); Ok(()) } @@ -1078,3 +1080,100 @@ mod lesson6 { () } } + +mod lesson7 { + use super::*; + + // Each caller needs to care about the generic `F` and propagate this type up. + // Also anyone using `Wrapper` needs to generic and name the type of `F`. + // pub struct Wrapper { + // f: F, + // } + + // This leads to a cleaner interface + pub struct Wrapper { + f: Box, + } + + // Either the following + // + // trait X { + // fn foo(&self, f: impl Fn()); + // } + // + // or + // + // trait X { + // fn foo(&self, f: F); + // } + // + // fn quox(x: &dyn X) {} + // + // will not work as the trait needs to be object safe. + + trait X { + fn foo(&self, f: ClosureType) -> u8; + } + // Now, there is only a single 'foo' in the V-table. + + fn quox(x: &dyn X) {} + + pub struct Entity {} + + impl X for Entity { + fn foo(&self, f: ClosureType) -> u8 { + let x = (*f)(); + + x + } + } + + pub fn do_closure() -> u8 { + 10 + } + + pub type ClosureType<'a> = &'a dyn Fn() -> u8; + + pub fn run() { + let e = Entity {}; + + let x: u8 = 10; + let c = move || x * x; + let resp = e.foo(&c); + let resp2 = e.foo(&do_closure); + + info!("lesson 7: resp: {}", resp); + info!("lesson 7: resp2: {}", resp2); + } +} + +/// Nothing to do with Traits, needs to be stashed somewhere else. +mod lesson8 { + use std::sync::Arc; + + #[derive(Debug, Clone)] + struct Entity { + counter: Arc>, + } + + impl Entity { + pub fn new() -> Self { + Self { + counter: Arc::new(Box::new(0)), + } + } + } + + pub fn run() { + let new_box = Entity::new(); + + let static_ref: &'static mut u8 = Box::leak(Box::new(**new_box.counter)); + assert_eq!(static_ref, &0); + + *static_ref = 2; + assert_eq!(static_ref, &2); + + let counter = **new_box.counter; + assert_eq!(counter, 0); + } +} From 1461729209819a7c0a85c43ee54c9506f509cfd0 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Thu, 14 Sep 2023 10:07:12 +0530 Subject: [PATCH 24/79] Add example based on IMAP crate --- Cargo.toml | 4 + src/traits.rs | 787 +++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 789 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 38a2aa0..e07213e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,10 @@ dotenv = "0.15.0" env_logger = "0.9.0" log = "0.4.16" http-body = "0.4.5" +bufstream = "0.1.3" +base64 = "0.21" +imap-proto = "0.16.1" +rustls-connector = { version = "0.18.0", features = ["dangerous-configuration"] } axum = { version = "0.6.18", features = ["headers", "tower-log"] } axum-extra = "0.1.2" diff --git a/src/traits.rs b/src/traits.rs index a0c3228..91182c6 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -104,7 +104,9 @@ pub fn runner() -> Result<()> { // lesson6::run(); lesson7::run(); - lesson8::run(); + let _ = lesson8::run(); + lesson9::run(); + lesson10::run(); Ok(()) } @@ -1147,8 +1149,789 @@ mod lesson7 { } } +pub mod lesson8 { + mod error { + //! Error types. + + use std::error::Error as StdError; + use std::fmt; + use std::io::Error as IoError; + use std::net::TcpStream; + #[cfg(any(feature = "native-tls", feature = "rustls-tls"))] + use std::net::TcpStream; + use std::result; + use std::str::Utf8Error; + + use base64::DecodeError; + use bufstream::IntoInnerError as BufError; + use imap_proto::{types::ResponseCode, Response}; + #[cfg(feature = "native-tls")] + use native_tls::Error as TlsError; + #[cfg(feature = "native-tls")] + use native_tls::HandshakeError as TlsHandshakeError; + use rustls_connector::HandshakeError; + #[cfg(feature = "rustls-tls")] + use rustls_connector::HandshakeError as RustlsHandshakeError; + + /// A convenience wrapper around `Result` for `imap::Error`. + pub type Result = result::Result; + + /// A BAD response from the server, which indicates an error message from the server. + #[derive(Debug)] + #[non_exhaustive] + pub struct Bad { + /// Human-redable message included with the Bad response. + pub information: String, + /// A more specific error status code included with the Bad response. + pub code: Option>, + } + + impl fmt::Display for Bad { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.information) + } + } + + /// A NO response from the server, which indicates an operational error message from the server. + #[derive(Debug)] + #[non_exhaustive] + pub struct No { + /// Human-redable message included with the NO response. + pub information: String, + /// A more specific error status code included with the NO response. + pub code: Option>, + } + + impl fmt::Display for No { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.information) + } + } + + /// A BYE response from the server, which indicates it is going to hang up on us. + #[derive(Debug)] + #[non_exhaustive] + pub struct Bye { + /// Human-redable message included with the response. + pub information: String, + /// A more specific error status code included with the response. + pub code: Option>, + } + + impl fmt::Display for Bye { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.information) + } + } + /// A set of errors that can occur in the IMAP client + #[derive(Debug)] + #[non_exhaustive] + pub enum Error { + /// An `io::Error` that occurred while trying to read or write to a network stream. + Io(IoError), + /// An error from the `rustls` library during the TLS handshake. + #[cfg(feature = "rustls-tls")] + RustlsHandshake(RustlsHandshakeError), + RustlsHandshakeErr(HandshakeError), + + /// An error from the `native_tls` library during the TLS handshake. + #[cfg(feature = "native-tls")] + TlsHandshake(TlsHandshakeError), + /// An error from the `native_tls` library while managing the socket. + #[cfg(feature = "native-tls")] + Tls(TlsError), + /// A BAD response from the IMAP server. + Bad(Bad), + /// A NO response from the IMAP server. + No(No), + /// A BYE response from the IMAP server. + Bye(Bye), + /// The connection was terminated unexpectedly. + ConnectionLost, + /// Error parsing a server response. + Parse(ParseError), + /// Command inputs were not valid [IMAP + /// strings](https://tools.ietf.org/html/rfc3501#section-4.3). + Validate(ValidateError), + /// Error appending an e-mail. + Append, + /// An unexpected response was received. This could be a response from a command, + /// or an unsolicited response that could not be converted into a local type in + /// [`UnsolicitedResponse`](crate::types::UnsolicitedResponse). + Unexpected(Response<'static>), + /// In response to a STATUS command, the server sent OK without actually sending any STATUS + /// responses first. + MissingStatusResponse, + /// StartTls is not available on the server + StartTlsNotAvailable, + #[cfg(all(not(feature = "native-tls"), not(feature = "rustls-tls")))] + /// Returns when Tls is not configured + TlsNotConfigured, + } + + impl From for Error { + fn from(err: IoError) -> Error { + Error::Io(err) + } + } + + impl From for Error { + fn from(err: ParseError) -> Error { + Error::Parse(err) + } + } + + impl From> for Error { + fn from(err: BufError) -> Error { + Error::Io(err.into()) + } + } + + #[cfg(feature = "rustls-tls")] + impl From> for Error { + fn from(err: RustlsHandshakeError) -> Error { + Error::RustlsHandshake(err) + } + } + + #[cfg(feature = "native-tls")] + impl From> for Error { + fn from(err: TlsHandshakeError) -> Error { + Error::TlsHandshake(err) + } + } + + impl From> for Error { + fn from(err: HandshakeError) -> Error { + Error::RustlsHandshakeErr(err) + } + } + + #[cfg(feature = "native-tls")] + impl From for Error { + fn from(err: TlsError) -> Error { + Error::Tls(err) + } + } + + impl<'a> From> for Error { + fn from(err: Response<'a>) -> Error { + Error::Unexpected(err.into_owned()) + } + } + + impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + Error::Io(ref e) => fmt::Display::fmt(e, f), + #[cfg(feature = "rustls-tls")] + Error::RustlsHandshake(ref e) => fmt::Display::fmt(e, f), + Error::RustlsHandshakeErr(ref e) => fmt::Display::fmt(e, f), + #[cfg(feature = "native-tls")] + Error::Tls(ref e) => fmt::Display::fmt(e, f), + #[cfg(feature = "native-tls")] + Error::TlsHandshake(ref e) => fmt::Display::fmt(e, f), + Error::Validate(ref e) => fmt::Display::fmt(e, f), + Error::Parse(ref e) => fmt::Display::fmt(e, f), + Error::No(ref data) => write!(f, "No Response: {}", data), + Error::Bad(ref data) => write!(f, "Bad Response: {}", data), + Error::Bye(ref data) => write!(f, "Bye Response: {}", data), + Error::ConnectionLost => f.write_str("Connection Lost"), + Error::Append => f.write_str("Could not append mail to mailbox"), + Error::Unexpected(ref r) => write!(f, "Unexpected Response: {:?}", r), + Error::MissingStatusResponse => write!(f, "Missing STATUS Response"), + Error::StartTlsNotAvailable => { + write!(f, "StartTls is not available on the server") + } + #[cfg(all(not(feature = "native-tls"), not(feature = "rustls-tls")))] + Error::TlsNotConfigured => write!(f, "No Tls feature is available"), + } + } + } + + impl StdError for Error { + #[allow(deprecated)] + fn description(&self) -> &str { + match *self { + Error::Io(ref e) => e.description(), + #[cfg(feature = "rustls-tls")] + Error::RustlsHandshake(ref e) => e.description(), + Error::RustlsHandshakeErr(ref e) => e.description(), + #[cfg(feature = "native-tls")] + Error::Tls(ref e) => e.description(), + #[cfg(feature = "native-tls")] + Error::TlsHandshake(ref e) => e.description(), + Error::Parse(ref e) => e.description(), + Error::Validate(ref e) => e.description(), + Error::Bad(_) => "Bad Response", + Error::No(_) => "No Response", + Error::Bye(_) => "Bye Response", + Error::ConnectionLost => "Connection lost", + Error::Append => "Could not append mail to mailbox", + Error::Unexpected(_) => "Unexpected Response", + Error::MissingStatusResponse => "Missing STATUS Response", + Error::StartTlsNotAvailable => "StartTls is not available on the server", + #[cfg(all(not(feature = "native-tls"), not(feature = "rustls-tls")))] + Error::TlsNotConfigured => "No Tls feature is available", + } + } + + fn cause(&self) -> Option<&dyn StdError> { + match *self { + Error::Io(ref e) => Some(e), + #[cfg(feature = "rustls-tls")] + Error::RustlsHandshake(ref e) => Some(e), + Error::RustlsHandshakeErr(ref e) => Some(e), + #[cfg(feature = "native-tls")] + Error::Tls(ref e) => Some(e), + #[cfg(feature = "native-tls")] + Error::TlsHandshake(ref e) => Some(e), + Error::Parse(ParseError::DataNotUtf8(_, ref e)) => Some(e), + _ => None, + } + } + } + + /// An error occured while trying to parse a server response. + #[derive(Debug)] + pub enum ParseError { + /// Indicates an error parsing the status response. Such as OK, NO, and BAD. + Invalid(Vec), + /// The client could not find or decode the server's authentication challenge. + Authentication(String, Option), + /// The client received data that was not UTF-8 encoded. + DataNotUtf8(Vec, Utf8Error), + } + + impl fmt::Display for ParseError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + ParseError::Invalid(_) => f.write_str("Unable to parse status response"), + ParseError::Authentication(_, _) => { + f.write_str("Unable to parse authentication response") + } + ParseError::DataNotUtf8(_, _) => { + f.write_str("Unable to parse data as UTF-8 text") + } + } + } + } + + impl StdError for ParseError { + fn description(&self) -> &str { + match *self { + ParseError::Invalid(_) => "Unable to parse status response", + ParseError::Authentication(_, _) => "Unable to parse authentication response", + ParseError::DataNotUtf8(_, _) => "Unable to parse data as UTF-8 text", + } + } + + fn cause(&self) -> Option<&dyn StdError> { + match *self { + ParseError::Authentication(_, Some(ref e)) => Some(e), + _ => None, + } + } + } + + /// An [invalid character](https://tools.ietf.org/html/rfc3501#section-4.3) was found in a command + /// argument. + #[derive(Debug)] + pub struct ValidateError { + /// the synopsis of the invalid command + pub(crate) command_synopsis: String, + /// the name of the invalid argument + pub(crate) argument: String, + /// the invalid character contained in the argument + pub(crate) offending_char: char, + } + + impl fmt::Display for ValidateError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // print character in debug form because invalid ones are often whitespaces + write!( + f, + "Invalid character {:?} in argument '{}' of command '{}'", + self.offending_char, self.argument, self.command_synopsis + ) + } + } + + impl StdError for ValidateError { + fn description(&self) -> &str { + "Invalid character in command argument" + } + + fn cause(&self) -> Option<&dyn StdError> { + None + } + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn validate_error_display() { + assert_eq!( + ValidateError { + command_synopsis: "COMMAND arg1 arg2".to_owned(), + argument: "arg2".to_string(), + offending_char: '\n' + } + .to_string(), + "Invalid character '\\n' in argument 'arg2' of command 'COMMAND arg1 arg2'" + ); + } + } + } + + mod client { + use crate::traits::lesson8::error::{Error, Result}; + use bufstream::BufStream; + use std::{ + io::{Read, Write}, + ops::{Deref, DerefMut}, + }; + + const INITIAL_TAG: u32 = 0; + const CR: u8 = 0x0d; + const LF: u8 = 0x0a; + + /// An (unauthenticated) handle to talk to an IMAP server. This is what you get when first + /// connecting. A succesfull call to [`Client::login`] or [`Client::authenticate`] will return a + /// [`Session`] instance that provides the usual IMAP methods. + // Both `Client` and `Session` deref to [`Connection`](struct.Connection.html), the underlying + // primitives type. + #[derive(Debug)] + pub struct Client { + conn: Connection, + } + + /// The underlying primitives type. Both `Client`(unauthenticated) and `Session`(after succesful + /// login) use a `Connection` internally for the TCP stream primitives. + #[derive(Debug)] + #[doc(hidden)] + pub struct Connection { + pub(crate) stream: BufStream, + tag: u32, + + /// Enable debug mode for this connection so that all client-server interactions are printed to + /// `STDERR`. + pub debug: bool, + + /// Tracks if we have read a greeting. + pub greeting_read: bool, + } + + // `Deref` instances are so we can make use of the same underlying primitives in `Client` and + // `Session` + impl Deref for Client { + type Target = Connection; + + fn deref(&self) -> &Connection { + &self.conn + } + } + + impl DerefMut for Client { + fn deref_mut(&mut self) -> &mut Connection { + &mut self.conn + } + } + + impl Client { + /// Creates a new client over the given stream. + /// + /// This method primarily exists for writing tests that mock the underlying transport, + /// but can also be used to support IMAP over custom tunnels. If you do not need to do + /// that, then it is simpler to use the [`ClientBuilder`](crate::ClientBuilder) to get + /// a new client. + /// + /// For an example, see `examples/timeout.rs` which uses a custom timeout on the + /// tcp stream. + /// + /// **Note:** In case you do need to use `Client::new` instead of the `ClientBuilder` + /// you will need to listen for the IMAP protocol server greeting before authenticating: + /// + /// ```rust,no_run + /// # use imap::Client; + /// # use std::io; + /// # use std::net::TcpStream; + /// # {} #[cfg(feature = "native-tls")] + /// # fn main() { + /// # let server = "imap.example.com"; + /// # let username = ""; + /// # let password = ""; + /// # let tcp = TcpStream::connect((server, 993)).unwrap(); + /// # use native_tls::TlsConnector; + /// # let ssl_connector = TlsConnector::builder().build().unwrap(); + /// # let tls = TlsConnector::connect(&ssl_connector, server.as_ref(), tcp).unwrap(); + /// let mut client = Client::new(tls); + /// client.read_greeting().unwrap(); + /// let session = client.login(username, password).unwrap(); + /// # } + /// ``` + pub fn new(stream: T) -> Client { + Client { + conn: Connection { + stream: BufStream::new(stream), + tag: INITIAL_TAG, + debug: false, + greeting_read: false, + }, + } + } + } + + impl Connection { + /// Read the greeting from the connection. Needs to be done after `connect`ing. + /// + /// Panics if called more than once on the same `Connection`. + pub fn read_greeting(&mut self) -> Result> { + assert!(!self.greeting_read, "Greeting can only be read once"); + + let mut v = Vec::new(); + self.readline(&mut v)?; + self.greeting_read = true; + + Ok(v) + } + + pub(crate) fn readline(&mut self, into: &mut Vec) -> Result { + use std::io::BufRead; + let read = self.stream.read_until(LF, into)?; + if read == 0 { + return Err(Error::ConnectionLost); + } + + if self.debug { + // Remove CRLF + let len = into.len(); + let line = &into[(len - read)..(len - 2)]; + eprintln!("S: {}", String::from_utf8_lossy(line)); + } + + Ok(read) + } + } + } + + mod conn { + use super::*; + use crate::traits::lesson8::extensions::idle::SetReadTimeout; + use std::fmt::{Debug, Formatter}; + use std::io::{Read, Write}; + + /// Imap connection trait of a read/write stream + pub trait ImapConnection: Read + Write + Send + SetReadTimeout + private::Sealed {} + + impl ImapConnection for T where T: Read + Write + Send + SetReadTimeout {} + + impl Debug for dyn ImapConnection { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "Imap connection") + } + } + + /// A boxed connection type + pub type Connection = Box; + + mod private { + use super::{Read, SetReadTimeout, Write}; + + pub trait Sealed {} + + impl Sealed for T where T: Read + Write + SetReadTimeout {} + } + } + + pub mod extensions { + pub mod idle { + use rustls::{ClientConnection, StreamOwned}; + + use crate::traits::lesson8::{ + conn::Connection, + error::{Error, Result}, + }; + use std::{net::TcpStream, ops::DerefMut, time::Duration}; + + /// Must be implemented for a transport in order for a `Session` to use IDLE. + pub trait SetReadTimeout { + /// Set the timeout for subsequent reads to the given one. + /// + /// If `timeout` is `None`, the read timeout should be removed. + /// + /// See also `std::net::TcpStream::set_read_timeout`. + fn set_read_timeout(&mut self, timeout: Option) -> Result<()>; + } + + impl<'a> SetReadTimeout for Connection { + fn set_read_timeout(&mut self, timeout: Option) -> Result<()> { + self.deref_mut().set_read_timeout(timeout) + } + } + + impl<'a> SetReadTimeout for TcpStream { + fn set_read_timeout(&mut self, timeout: Option) -> Result<()> { + TcpStream::set_read_timeout(self, timeout).map_err(Error::Io) + } + } + } + } + + use self::{client::Client, extensions::idle::SetReadTimeout}; + use conn::{Connection, ImapConnection}; + use error::{Error, Result}; + use std::io::{Read, Write}; + use std::net::TcpStream; + + /// The connection mode we are going to use + #[derive(Clone, Debug, PartialEq, Eq)] + pub enum ConnectionMode { + /// Automatically detect what connection mode should be used. + /// This will use TLS if the port is 993, and StartTLls if the server says it's available. + /// If only Plaintext is available it will error out. + #[cfg(any(feature = "native-tls", feature = "rustls-tls"))] + AutoTls, + /// Automatically detect what connection mode should be used. + /// This will use TLS if the port is 993, and StartTLls if the server says it's available. + /// Finally it will fallback to Plaintext + Auto, + /// A plain unencrypted Tcp connection + Plaintext, + /// an encrypted TLS connection + #[cfg(any(feature = "native-tls", feature = "rustls-tls"))] + Tls, + /// a start tls connection + #[cfg(any(feature = "native-tls", feature = "rustls-tls"))] + StartTls, + } + + /// The tls backend to use, either explicit or auto (default) + #[cfg(any(feature = "native-tls", feature = "rustls-tls"))] + #[derive(Clone, Debug, Eq, PartialEq)] + #[non_exhaustive] + pub enum TlsKind { + /// Use the NativeTLS backend + #[cfg(feature = "native-tls")] + Native, + /// Use the Rustls backend + #[cfg(feature = "rustls-tls")] + Rust, + /// Use whatever backend is available (uses rustls if both are available) + Any, + } + + #[derive(Clone)] + pub struct ClientBuilder + where + D: AsRef, + { + domain: D, + port: u16, + mode: ConnectionMode, + #[cfg(any(feature = "native-tls", feature = "rustls-tls"))] + tls_kind: TlsKind, + #[cfg(any(feature = "native-tls", feature = "rustls-tls"))] + skip_tls_verify: bool, + } + + impl ClientBuilder + where + D: AsRef, + { + /// Make a new `ClientBuilder` using the given domain and port. + pub fn new(domain: D, port: u16) -> Self { + ClientBuilder { + domain, + port, + #[cfg(any(feature = "native-tls", feature = "rustls-tls"))] + mode: ConnectionMode::AutoTls, + #[cfg(all(not(feature = "native-tls"), not(feature = "rustls-tls")))] + mode: ConnectionMode::Auto, + #[cfg(any(feature = "native-tls", feature = "rustls-tls"))] + tls_kind: TlsKind::Any, + #[cfg(any(feature = "native-tls", feature = "rustls-tls"))] + skip_tls_verify: false, + } + } + + /// Sets the Connection mode to use for this connection + pub fn mode(&mut self, mode: ConnectionMode) -> &mut Self { + self.mode = mode; + self + } + + /// Sets the TLS backend to use for this connection. + #[cfg(any(feature = "native-tls", feature = "rustls-tls"))] + pub fn tls_kind(&mut self, kind: TlsKind) -> &mut Self { + self.tls_kind = kind; + self + } + + /// Controls the use of certificate validation. + /// + /// Defaults to `false`. + /// + /// # Warning + /// + /// You should only use this as a last resort as it allows another server to impersonate the + /// server you think you're talking to, which would include being able to receive your + /// credentials. + /// + /// See [`native_tls::TlsConnectorBuilder::danger_accept_invalid_certs`], + /// [`native_tls::TlsConnectorBuilder::danger_accept_invalid_hostnames`], + /// [`rustls::ClientConfig::dangerous`] + #[cfg(any(feature = "native-tls", feature = "rustls-tls"))] + pub fn danger_skip_tls_verify(&mut self, skip_tls_verify: bool) -> &mut Self { + self.skip_tls_verify = skip_tls_verify; + self + } + + /// Make a [`Client`] using the configuration. + /// + /// ```no_run + /// # use imap::ClientBuilder; + /// # {} #[cfg(feature = "rustls-tls")] + /// # fn main() -> Result<(), imap::Error> { + /// let client = ClientBuilder::new("imap.example.com", 143) + /// .starttls().connect()?; + /// # Ok(()) + /// # } + /// ``` + pub fn connect(&self) -> Result> { + #[cfg(any(feature = "native-tls", feature = "rustls-tls"))] + return self.connect_with(|_domain, tcp| self.build_tls_connection(tcp)); + #[cfg(all(not(feature = "native-tls"), not(feature = "rustls-tls")))] + return self.connect_with(|_domain, _tcp| -> Result { + return Err(Error::TlsNotConfigured); + }); + } + + /// Make a [`Client`] using a custom initialization. This function is intended + /// to be used if your TLS setup requires custom work such as adding private CAs + /// or other specific TLS parameters. + /// + /// Note, if the connection does not end up as TLS, handshake will not be called. + /// This can happen if connection mode is set to Tcp, or the server does not support starttls. + /// + /// The `handshake` argument should accept two parameters: + /// + /// - domain: [`&str`] + /// - tcp: [`TcpStream`] + /// + /// and yield a `Result` where `C` is `Read + Write + Send + SetReadTimeout + 'static,`. + /// It should only perform TLS initialization over the given `tcp` socket and return the + /// encrypted stream object, such as a [`native_tls::TlsStream`] or a + /// [`rustls_connector::TlsStream`]. + /// + /// If the caller is using `STARTTLS` and previously called [`starttls`](Self::starttls) + /// then the `tcp` socket given to the `handshake` function will be connected and will + /// have initiated the `STARTTLS` handshake. + /// + /// ```no_run + /// # use imap::ClientBuilder; + /// # use rustls_connector::RustlsConnector; + /// # {} #[cfg(feature = "rustls-tls")] + /// # fn main() -> Result<(), imap::Error> { + /// let client = ClientBuilder::new("imap.example.com", 143) + /// .starttls() + /// .connect_with(|domain, tcp| { + /// let ssl_conn = RustlsConnector::new_with_native_certs()?; + /// Ok(ssl_conn.connect(domain, tcp)?) + /// })?; + /// # Ok(()) + /// # } + /// ``` + #[allow(unused_variables)] + pub fn connect_with(&self, handshake: F) -> Result> + where + F: FnOnce(&str, TcpStream) -> Result, + C: Read + Write + Send + SetReadTimeout + 'static, + { + #[allow(unused_mut)] + let mut greeting_read = false; + let tcp = TcpStream::connect((self.domain.as_ref(), self.port))?; + + let stream: Connection = match self.mode { + #[cfg(any(feature = "native-tls", feature = "rustls-tls"))] + ConnectionMode::AutoTls => { + #[cfg(any(feature = "native-tls", feature = "rustls-tls"))] + if self.port == 993 { + Box::new(handshake(self.domain.as_ref(), tcp)?) + } else { + let (stream, upgraded) = self.upgrade_tls(Client::new(tcp), handshake)?; + greeting_read = true; + + if !upgraded { + Err(Error::StartTlsNotAvailable)? + } + stream + } + #[cfg(all(not(feature = "native-tls"), not(feature = "rustls-tls")))] + Err(Error::StartTlsNotAvailable)? + } + ConnectionMode::Auto => { + #[cfg(any(feature = "native-tls", feature = "rustls-tls"))] + if self.port == 993 { + Box::new(handshake(self.domain.as_ref(), tcp)?) + } else { + let (stream, _upgraded) = self.upgrade_tls(Client::new(tcp), handshake)?; + greeting_read = true; + + stream + } + #[cfg(all(not(feature = "native-tls"), not(feature = "rustls-tls")))] + Box::new(tcp) + } + ConnectionMode::Plaintext => Box::new(tcp), + #[cfg(any(feature = "native-tls", feature = "rustls-tls"))] + ConnectionMode::StartTls => { + let (stream, upgraded) = self.upgrade_tls(Client::new(tcp), handshake)?; + greeting_read = true; + + if !upgraded { + Err(Error::StartTlsNotAvailable)? + } + stream + } + #[cfg(any(feature = "native-tls", feature = "rustls-tls"))] + ConnectionMode::Tls => Box::new(handshake(self.domain.as_ref(), tcp)?), + }; + + let mut client = Client::new(stream); + if !greeting_read { + client.read_greeting()?; + } else { + client.greeting_read = true; + } + + Ok(client) + } + } + + pub fn run() -> Result<()> { + let _client = ClientBuilder::new("imap.example.com", 143).connect_with(|domain, tcp| { + let ssl_conn = rustls_connector::RustlsConnector::new_with_native_certs()?; + Ok::(ssl_conn.connect(domain, tcp)?.sock) + })?; + + Ok(()) + } +} + +mod lesson9 { + use std::error::Error; + use std::result::Result; + + pub fn run() -> Result<(), &'static dyn Error> { + Ok(()) + } +} + /// Nothing to do with Traits, needs to be stashed somewhere else. -mod lesson8 { +mod lesson10 { use std::sync::Arc; #[derive(Debug, Clone)] From 5157035e524ededf32c2cb7092fca69532cdc45b Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Tue, 19 Mar 2024 17:33:00 +0530 Subject: [PATCH 25/79] Add example for pbkdf AES enc (still in progress, may have mistakes) --- Cargo.toml | 8 ++ src/challenge1.rs | 251 ++++++++++++++++++++++++++++++++++++++++++++++ src/cli.rs | 3 + src/lib.rs | 1 + src/main.rs | 10 +- src/traits.rs | 2 +- 6 files changed, 273 insertions(+), 2 deletions(-) create mode 100644 src/challenge1.rs diff --git a/Cargo.toml b/Cargo.toml index e07213e..d0a8415 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,14 @@ bufstream = "0.1.3" base64 = "0.21" imap-proto = "0.16.1" rustls-connector = { version = "0.18.0", features = ["dangerous-configuration"] } +hmac = { version = "0.13.0-pre.3" } +pbkdf2 = { version = "0.13.0-pre.0" } +sha2 = { version = "0.11.0-pre.3" } +hex-literal = "^0.4" +hex = "^0.4" +aes-gcm-siv = "^0.11" +aes = "^0.8" +heapless = "^0.8" axum = { version = "0.6.18", features = ["headers", "tower-log"] } axum-extra = "0.1.2" diff --git a/src/challenge1.rs b/src/challenge1.rs new file mode 100644 index 0000000..d1655ac --- /dev/null +++ b/src/challenge1.rs @@ -0,0 +1,251 @@ +/// Example at: +/// https://gist.github.com/bsodmike/46fc344fa8d5fcb6e30f544bd4409df5 +use anyhow::{Context, Error, Result}; +use hmac::{digest::core_api::CoreWrapper, EagerHash, Hmac, HmacCore, KeyInit}; +use log::trace; +use pbkdf2::pbkdf2; +use sha2::Sha512; + +type PrfHasher = Sha512; + +const PBKDF_ROUNDS: u32 = 600_000; +const KEY_BUFF_SIZE: usize = 20; + +pub fn runner() -> Result<()> { + example1::run()?; + + Ok(()) +} + +pub mod aes { + use aes::cipher; + use aes::cipher::generic_array::GenericArray; + use aes_gcm_siv::aead::Buffer; + use aes_gcm_siv::AesGcmSiv; + use aes_gcm_siv::{ + aead::{AeadInPlace, KeyInit, OsRng}, + Aes256GcmSiv, Nonce, + }; + use std::io::Read; + use std::marker::PhantomData; + + #[derive(Debug)] + /// FIXME: Allow swiching out the `A` array type. + pub(crate) struct AesVecBuffer<'a, A> { + inner: Vec, + _life: PhantomData<&'a A>, + } + + impl<'a, A> AesVecBuffer<'a, A> { + pub fn inner(&mut self) -> &mut Vec { + &mut self.inner + } + } + + impl<'a, A> aes_gcm_siv::aead::Buffer for AesVecBuffer<'a, A> { + fn extend_from_slice(&mut self, other: &[u8]) -> aes_gcm_siv::aead::Result<()> { + Ok(self.inner.extend(other)) + } + + fn truncate(&mut self, len: usize) { + self.inner.truncate(len) + } + + fn len(&self) -> usize { + self.as_ref().len() + } + + fn is_empty(&self) -> bool { + self.as_ref().is_empty() + } + } + + impl<'a, A> AsRef<[u8]> for AesVecBuffer<'a, A> { + fn as_ref(&self) -> &[u8] { + &self.inner + } + } + + impl<'a, A> AsMut<[u8]> for AesVecBuffer<'a, A> { + fn as_mut(&mut self) -> &mut [u8] { + &mut self.inner[..] + } + } + + impl<'a, A, const N: usize> PartialEq<[u8; N]> for AesVecBuffer<'a, A> { + fn eq(&self, other: &[u8; N]) -> bool { + self.inner.eq(other) + } + + fn ne(&self, other: &[u8; N]) -> bool { + !self.eq(other) + } + } + + pub(crate) struct AesEncrypter<'a> { + cipher: AesGcmSiv, + nonce: String, + buffer: AesVecBuffer<'a, ()>, + } + + impl<'a> AesEncrypter<'a> { + pub fn new(nonce: String, plaintext: &'a str) -> Self { + let key = Aes256GcmSiv::generate_key(&mut OsRng); + let cipher = Aes256GcmSiv::new(&key); + + // Note: buffer needs 16-bytes overhead for auth tag tag + let inner: heapless::Vec = heapless::Vec::new(); + // let inner: Vec = Vec::new(); + let mut buffer = AesVecBuffer::<()> { + inner: inner.to_vec(), + _life: PhantomData, + }; + buffer.extend_from_slice(plaintext.as_bytes()).unwrap(); + + Self { + cipher, + nonce, + buffer, + } + } + + #[allow(dead_code)] + pub fn buffer(&mut self) -> &mut AesVecBuffer<'a, ()> { + &mut self.buffer + } + + pub fn encrypt_in_place(&mut self) -> anyhow::Result<()> { + let mut bytes = self.nonce.as_bytes(); + let mut short_nonce = [0u8; 12]; + bytes.read_exact(&mut short_nonce)?; + // trace!("Len: {:?}", short_nonce.len()); + let nonce: &GenericArray = Nonce::from_slice(&short_nonce[..]); // 96-bits; unique per message + + // Encrypt `buffer` in-place, replacing the plaintext contents with ciphertext + Ok(self + .cipher + .encrypt_in_place(nonce, b"", &mut self.buffer) + .expect("Encrypt cipher in place")) + } + + pub fn decrypt_in_place(&mut self) -> anyhow::Result<()> { + let mut bytes = self.nonce.as_bytes(); + let mut short_nonce = [0u8; 12]; + bytes.read_exact(&mut short_nonce)?; + + let nonce: &GenericArray = Nonce::from_slice(&short_nonce[..]); // 96-bits; unique per message + + // Encrypt `buffer` in-place, replacing the plaintext contents with ciphertext + Ok(self + .cipher + .decrypt_in_place(nonce, b"", &mut self.buffer) + .expect("Decrypt cipher in place")) + } + } +} + +use crate::challenge1::aes::AesEncrypter; +use crate::challenge1::encrypter::{Encryptable, Encrypter}; + +pub(crate) struct EncrypterState<'a>(pub(crate) &'a str, pub(crate) &'a str); + +impl<'a> EncrypterState<'a> { + pub(crate) fn new(password: &'a str, salt: &'a str) -> Self { + Self(password, salt) + } +} + +pub(crate) fn get_encrypter<'a>(state: EncrypterState<'a>, plaintext: &'a str) -> AesEncrypter<'a> { + // Create pbkdf + let buf = [0u8; 20]; + let mut buf_boxed = Box::new(buf); + let mut encrypter = Encrypter::<()>::new(&mut buf_boxed); + let pbkdf_key = encrypter.pbkdf_key(state.0, state.1); + let pbkdf_key_hex = hex::encode(pbkdf_key); + trace!("Key: {}", &pbkdf_key_hex); + + AesEncrypter::new(pbkdf_key_hex.clone(), plaintext) +} + +fn process_pbkdf_key( + buf_ptr: &mut Box<[u8; KEY_BUFF_SIZE]>, + password: &str, + salt: &str, // fmt +) -> anyhow::Result<()> +where + CoreWrapper>: KeyInit, + H: hmac::EagerHash, + ::Core: Sync, +{ + let buf = buf_ptr.as_mut(); + + pbkdf2::>( + &password.to_string().as_bytes(), + &salt.to_string().as_bytes(), + PBKDF_ROUNDS, + buf, + // fmt + ) + .context("HMAC can be initialized with any key length")?; + + Ok(()) +} + +pub mod encrypter { + use super::*; + use std::marker::PhantomData; + + #[derive(Debug)] + pub(crate) struct Encrypter<'a, S> { + key: &'a mut Box<[u8; KEY_BUFF_SIZE]>, + _phat: PhantomData<&'a S>, + } + + impl<'a, S> Encrypter<'a, S> { + pub fn new(buf: &'a mut Box<[u8; KEY_BUFF_SIZE]>) -> Self { + Self { + key: buf, + _phat: PhantomData, + } + } + } + + pub trait Encryptable { + type KeyBuf; + + fn pbkdf_key(&mut self, password: &str, salt: &str) -> Self::KeyBuf; + } + + impl Encryptable for Encrypter<'_, T> { + type KeyBuf = [u8; KEY_BUFF_SIZE]; + + fn pbkdf_key(&mut self, password: &str, salt: &str) -> Self::KeyBuf { + process_pbkdf_key::(&mut self.key, password, salt).unwrap(); + + **self.key + } + } +} + +pub mod example1 { + use super::*; + + pub fn run() -> Result<()> { + let mut enc = get_encrypter(EncrypterState::new("password", "salt"), "plaintext message"); + enc.encrypt_in_place().unwrap(); + // `buffer` now contains the message ciphertext + trace!("Encrypted cipher text: {}", hex::encode(&enc.buffer())); + assert_ne!(enc.buffer(), b"plaintext message"); + + // Decrypt `buffer` in-place, replacing its ciphertext context with the original plaintext + enc.decrypt_in_place().unwrap(); + let m = enc.buffer().inner(); + trace!( + "Decrypted plaintext: {}", + String::from_utf8(m.to_vec()).unwrap() + ); + assert_eq!(enc.buffer().as_ref(), b"plaintext message"); + + Ok(()) + } +} diff --git a/src/cli.rs b/src/cli.rs index 8c092be..74c4500 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -32,6 +32,9 @@ pub enum Commands { /// Closures Closures, + + /// Challenge1 + Challenge1, } pub fn runner(mut mk: impl FnMut() -> Result) -> Result { diff --git a/src/lib.rs b/src/lib.rs index b8468f7..47bd262 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,3 +9,4 @@ pub mod traits; //mod generics_1; //mod iterators_1; //mod trait_objects_1; +pub mod challenge1; diff --git a/src/main.rs b/src/main.rs index 1bdad70..ab3838b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,7 +3,7 @@ // Disabled for use of UnsafeCell //#![forbid(unsafe_code)] #![allow(unused_imports)] -#![deny(unreachable_pub, private_in_public, unstable_features)] +#![deny(unreachable_pub, unstable_features)] #![warn(rust_2018_idioms, future_incompatible, nonstandard_style)] use anyhow::{Error, Result}; @@ -11,6 +11,7 @@ use clap::Parser; use log::{debug, info}; use std::io::Read; use std::sync::Arc; +use tutorials::challenge1; use tutorials::cli::{runner, Args, Commands}; use tutorials::{ builder::TaskManagerBuilder, closures, dispatch::*, oop_pattern::*, smart_pointers::*, traits, @@ -146,6 +147,13 @@ fn main() -> Result<()> { Ok(()) })?, + Some(Commands::Challenge1) => runner(|| { + info!("Tutorial: Challenge1\n"); + + challenge1::runner()?; + + Ok(()) + })?, _ => info!("Command not found"), }; diff --git a/src/traits.rs b/src/traits.rs index 91182c6..71469f1 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -105,7 +105,7 @@ pub fn runner() -> Result<()> { // lesson6::run(); lesson7::run(); let _ = lesson8::run(); - lesson9::run(); + let _ = lesson9::run(); lesson10::run(); Ok(()) From 480c439f63ac19686ee2d176b2fbf4e0b16b800c Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sat, 23 Mar 2024 14:28:59 +0530 Subject: [PATCH 26/79] Initial addition for PartialEq --- src/cli.rs | 3 +++ src/lib.rs | 1 + src/main.rs | 10 +++++++++- src/partial_eq.rs | 45 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 src/partial_eq.rs diff --git a/src/cli.rs b/src/cli.rs index 74c4500..fb0fdcd 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -35,6 +35,9 @@ pub enum Commands { /// Challenge1 Challenge1, + + /// PartialEq example + PartialEq1, } pub fn runner(mut mk: impl FnMut() -> Result) -> Result { diff --git a/src/lib.rs b/src/lib.rs index 47bd262..45ad951 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,3 +10,4 @@ pub mod traits; //mod iterators_1; //mod trait_objects_1; pub mod challenge1; +pub mod partial_eq; diff --git a/src/main.rs b/src/main.rs index ab3838b..4818489 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,11 +11,11 @@ use clap::Parser; use log::{debug, info}; use std::io::Read; use std::sync::Arc; -use tutorials::challenge1; use tutorials::cli::{runner, Args, Commands}; use tutorials::{ builder::TaskManagerBuilder, closures, dispatch::*, oop_pattern::*, smart_pointers::*, traits, }; +use tutorials::{challenge1, partial_eq}; fn main() -> Result<()> { env_logger::init(); @@ -154,6 +154,14 @@ fn main() -> Result<()> { Ok(()) })?, + Some(Commands::PartialEq1) => runner(|| { + info!("Tutorial: PartialEq1\n"); + + partial_eq::runner()?; + + Ok(()) + })?, + _ => info!("Command not found"), }; diff --git a/src/partial_eq.rs b/src/partial_eq.rs new file mode 100644 index 0000000..6295d8e --- /dev/null +++ b/src/partial_eq.rs @@ -0,0 +1,45 @@ +use anyhow::{Ok, Result}; + +#[derive(Debug, PartialEq, Default)] +pub struct List { + memory: Vec, +} + +impl List +where + List: PartialEq, +{ + pub fn new() -> Self { + List { memory: Vec::new() } + } + // push() add to end of list + pub fn push(&mut self, value: T) { + self.memory.push(value); + } + + pub fn is_equal_to(&mut self, other: &List) -> bool { + self == other + } +} + +pub fn runner() -> Result<()> { + example1::run()?; + + Ok(()) +} + +pub mod example1 { + use super::*; + + pub fn run() -> Result<()> { + let mut list1 = List::>::new(); + let mut list2 = List::>::new(); + + list1.push(Some(1)); + + let are_they_equal = list1.is_equal_to(&mut list2); + assert!(are_they_equal); + + Ok(()) + } +} From 065b0d9f57b44da46df4dbe8709ddd7422b130cb Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sat, 23 Mar 2024 15:00:12 +0530 Subject: [PATCH 27/79] FIX tests --- src/traits.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/traits.rs b/src/traits.rs index 71469f1..f10fe25 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -1486,7 +1486,7 @@ pub mod lesson8 { } } - mod client { + pub mod client { use crate::traits::lesson8::error::{Error, Result}; use bufstream::BufStream; use std::{ @@ -1555,7 +1555,7 @@ pub mod lesson8 { /// you will need to listen for the IMAP protocol server greeting before authenticating: /// /// ```rust,no_run - /// # use imap::Client; + /// # use tutorials::traits::lesson8::client::Client; /// # use std::io; /// # use std::net::TcpStream; /// # {} #[cfg(feature = "native-tls")] @@ -1792,7 +1792,7 @@ pub mod lesson8 { /// Make a [`Client`] using the configuration. /// /// ```no_run - /// # use imap::ClientBuilder; + /// # use tutorials::traits::lesson8::ClientBuilder; /// # {} #[cfg(feature = "rustls-tls")] /// # fn main() -> Result<(), imap::Error> { /// let client = ClientBuilder::new("imap.example.com", 143) @@ -1831,7 +1831,7 @@ pub mod lesson8 { /// have initiated the `STARTTLS` handshake. /// /// ```no_run - /// # use imap::ClientBuilder; + /// # use tutorials::traits::lesson8::ClientBuilder; /// # use rustls_connector::RustlsConnector; /// # {} #[cfg(feature = "rustls-tls")] /// # fn main() -> Result<(), imap::Error> { From 861ad332a8eb9cf903252bf8e9fc2cec966602e8 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Thu, 22 Aug 2024 01:45:10 +0530 Subject: [PATCH 28/79] Add example for correctly handling panics with Tokio threads --- Cargo.toml | 4 ++++ LICENSE | 3 ++- README.md | 7 +++++++ rust-toolchain.toml | 2 ++ src/bin/tokio-panic-handling.rs | 29 +++++++++++++++++++++++++++++ src/main.rs | 6 ++++++ 6 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 rust-toolchain.toml create mode 100644 src/bin/tokio-panic-handling.rs diff --git a/Cargo.toml b/Cargo.toml index d0a8415..53b3eaa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,10 @@ version = "0.1.0" edition = "2021" publish = false +[[bin]] +name = "tokio-panic-handling" +path = "src/bin/tokio-panic-handling.rs" + [dependencies] anyhow = "1.0.66" futures = "0.3.19" diff --git a/LICENSE b/LICENSE index 03bf8a9..8092844 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,7 @@ The MIT License (MIT) -Copyright (c) 2021 Michael de Silva (https://desilva.io/about) +Copyright (c) Michael de Silva (https://desilva.io/about) +Email: michael@cyberdynea.io Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/README.md b/README.md index 127311a..4895b46 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,13 @@ This repo means to serve as a playground for expressing concepts quickly, in a "Rust by Example" format. +## Examples + +Run the following with `cargo r --bin ` + +1. `tokio-panic-handling` + + ## License Information diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..66eee23 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,2 @@ +[toolchain] +channel = "beta" \ No newline at end of file diff --git a/src/bin/tokio-panic-handling.rs b/src/bin/tokio-panic-handling.rs new file mode 100644 index 0000000..30fc769 --- /dev/null +++ b/src/bin/tokio-panic-handling.rs @@ -0,0 +1,29 @@ +//! Correctly handle panic failures in threads spawned with Tokio. + +use std::time::Duration; +use tokio::time::sleep; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let control_path_1 = tokio::spawn(async { + println!("control path (path_1)"); + for i in (0..5).rev() { + println!("(path_1) ...{i}"); + sleep(Duration::from_millis(1000)).await; + } + panic!("(path_1) BOOOOOOOOOOOOOM!!!!"); + }); + + let control_path_2 = tokio::spawn(async { + println!("control path (path_2)"); + loop { + sleep(Duration::from_millis(222)).await; + println!("(path_2)"); + } + }); + + tokio::select! { // as a bonus you get error handling with _?_ + v = control_path_1 => v?, + v = control_path_2 => v?, + } +} diff --git a/src/main.rs b/src/main.rs index 4818489..9aecba0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,6 +22,12 @@ fn main() -> Result<()> { let cli = Args::parse(); + // FIXME - what was this about now? + // let f = vec![]; + // let s = || { + // // + // }; + match &cli.command { Some(Commands::Dispatch) => { runner(|| { From 16340c378565f86cfa8fe65310db7d942184c28e Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Thu, 22 Aug 2024 23:55:34 +0530 Subject: [PATCH 29/79] Add example demonstrating in memory searching of Strings and data --- Cargo.toml | 5 ++ src/bin/string-search.rs | 178 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 183 insertions(+) create mode 100644 src/bin/string-search.rs diff --git a/Cargo.toml b/Cargo.toml index 53b3eaa..30e82ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,10 @@ publish = false name = "tokio-panic-handling" path = "src/bin/tokio-panic-handling.rs" +[[bin]] +name = "string-search" +path = "src/bin/string-search.rs" + [dependencies] anyhow = "1.0.66" futures = "0.3.19" @@ -34,6 +38,7 @@ hex = "^0.4" aes-gcm-siv = "^0.11" aes = "^0.8" heapless = "^0.8" +critical-section = { version = "1.1", features = ["std"]} axum = { version = "0.6.18", features = ["headers", "tower-log"] } axum-extra = "0.1.2" diff --git a/src/bin/string-search.rs b/src/bin/string-search.rs new file mode 100644 index 0000000..814c3b6 --- /dev/null +++ b/src/bin/string-search.rs @@ -0,0 +1,178 @@ +//! Challenge: +//! +//! - Assume two users are using a site like eBay, but the concept is that user Bob can post an item for sale via a text input field and offer a sell price. +//! - The bidding action is performed purely in memory, so ignore I/O and other complexities. +//! - Think in terms of C++ and pointers. This is a quiz on simple data structures. +//! - Optimise for Big O performance. +//! +//! Design notes +//! +//! - User input is assumed to be all lower case, alphanumerals only, with single spaces. The assumption is that this binary operates on clean data. Validating and sanitising input is a further refinement. +//! - TBD +//! +//! Optimisations: +//! +//! Create small and big benchmarks: +//! +//! - Use https://github.com/bheisler/criterion.rs +//! - Use https://github.com/nvzqz/divan +//! +#![allow(unused_imports, dead_code, unused_variables)] + +use anyhow::anyhow; +use anyhow::Error; +use std::cell::RefCell; +use std::hash::Hash; +use std::{ + collections::HashMap, + io::{BufRead, Write}, + ops::Deref, + pin::Pin, + sync::LazyLock, +}; +use tokio::sync::Mutex; + +pub static ENTRY_MAP: LazyLock>>> = + LazyLock::new(|| Mutex::new(Some(HashMap::new()))); + +pub struct Entries(HashMap); + +impl Entries { + pub async fn add(text: String, amount: f64) { + let index_lock: &mut Option> = &mut *ENTRY_MAP.lock().await; + if let Some(entries) = index_lock { + let mut existing_data = entries.clone(); + existing_data.insert(text, amount); + + *entries = existing_data; + } + } + + pub async fn add_many(items: HashMap<&str, f64>) { + let index_lock: &mut Option> = &mut *ENTRY_MAP.lock().await; + if let Some(entries) = index_lock { + let new: HashMap = items + .into_iter() + .map(|el| (el.0.to_string(), el.1)) + .collect(); + *entries = new; + } + } + + pub async fn search(input: &str, amount: f64) -> Result<(String, f64, f64), Error> { + let text = input.to_string(); + + let index_lock: &mut Option> = &mut *ENTRY_MAP.lock().await; + if let Some(entries) = index_lock { + let needle = text; + let haystack: Vec = entries.into_iter().map(|el| el.0.to_string()).collect(); + + if haystack.contains(&needle) { + if let Some(cost) = entries.get(&needle) { + if amount >= *cost { + Ok((needle, amount, *cost)) + } else { + return Err(anyhow!( + "Item {} costs more than your offer of ${}", + &needle, + &amount + )); + } + } else { + unreachable!() + } + } else { + return Err(anyhow!("Item {} is not available!", &needle)); + } + } else { + // This would be classed as an internal error, so ideally I would mark this as `unreachable!()`. + return Err(anyhow!("Internal error")); + } + } +} + +#[tokio::main] +pub async fn main() -> Result<(), Error> { + Ok(()) +} + +#[cfg(test)] +pub mod tests { + use super::*; + + #[tokio::test] + async fn match_existing_available_item() { + let items = HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); + Entries::add_many(items).await; + + { + let index_lock: &mut Option> = &mut *ENTRY_MAP.lock().await; + if let Some(entries) = index_lock { + dbg!(&entries); + assert!(entries.len() > 0); + }; + } + + // Success, the buyer gets an instant match! + let (item, bid, ask) = Entries::search("banana", 20.00).await.unwrap(); + } + + #[tokio::test] + async fn panic_expect_error_for_low_bid() { + let items = HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); + Entries::add_many(items).await; + + { + let index_lock: &mut Option> = &mut *ENTRY_MAP.lock().await; + if let Some(entries) = index_lock { + dbg!(&entries); + assert!(entries.len() > 0); + }; + } + + if let Err(err) = Entries::search("banana", 8.23).await { + assert_eq!( + err.to_string(), + String::from("Item banana costs more than your offer of $8.23") + ) + } + } + + #[should_panic] + #[tokio::test] + async fn handle_mismatching_category() { + let items = HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); + Entries::add_many(items).await; + + { + let index_lock: &mut Option> = &mut *ENTRY_MAP.lock().await; + if let Some(entries) = index_lock { + dbg!(&entries); + assert!(entries.len() > 0); + }; + } + + Entries::search("fruit", 20.00).await.unwrap(); + } + + #[tokio::test] + async fn panic_mismatching_category_partial_text() { + let items = HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); + Entries::add_many(items).await; + + { + let index_lock: &mut Option> = &mut *ENTRY_MAP.lock().await; + if let Some(entries) = index_lock { + dbg!(&entries); + assert!(entries.len() > 0); + }; + } + + if let Err(err) = Entries::search("red appl", 8.23).await { + assert_eq!( + err.to_string(), + String::from("Item red appl is not available!") + ) + } + } +} From 7c012894cfc852e23225ce9068b7f33283e674fd Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Fri, 23 Aug 2024 10:36:59 +0530 Subject: [PATCH 30/79] WIP: Add benchmarking option for flamegraphing --- Cargo.toml | 4 +- flamegraph.svg | 491 +++++++++++++++++++++++++++++++++++++++ src/bin/string-search.rs | 89 +++++-- 3 files changed, 567 insertions(+), 17 deletions(-) create mode 100644 flamegraph.svg diff --git a/Cargo.toml b/Cargo.toml index 30e82ad..569f50a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,6 @@ hex = "^0.4" aes-gcm-siv = "^0.11" aes = "^0.8" heapless = "^0.8" -critical-section = { version = "1.1", features = ["std"]} axum = { version = "0.6.18", features = ["headers", "tower-log"] } axum-extra = "0.1.2" @@ -51,3 +50,6 @@ tower-service = "0.3.1" [dev-dependencies] test-log = "0.2.10" + +[profile.release] +debug = true \ No newline at end of file diff --git a/flamegraph.svg b/flamegraph.svg new file mode 100644 index 0000000..e9e54e7 --- /dev/null +++ b/flamegraph.svg @@ -0,0 +1,491 @@ +Flame Graph Reset ZoomSearch all (1 samples, 100%)libsystem_pthread.dylib`thread_start (1 samples, 100.00%)libsystem_pthread.dylib`thread_startlibsystem_pthread.dylib`_pthread_start (1 samples, 100.00%)libsystem_pthread.dylib`_pthread_startstring_search-f39db1bfbe9ebcc5`std::sys::pal::unix::thread::Thread::new::thread_start (1 samples, 100.00%)string_search-f39db1bfbe9ebcc5`std::sys::pal::unix::thread::Thread::new::thread_startstring_search-f39db1bfbe9ebcc5`core::ops::function::FnOnce::call_once{{vtable.shim}} (1 samples, 100.00%)string_search-f39db1bfbe9ebcc5`core::ops::function::FnOnce::call_once{{vtable.shim}}string_search-f39db1bfbe9ebcc5`std::sys::backtrace::__rust_begin_short_backtrace (1 samples, 100.00%)string_search-f39db1bfbe9ebcc5`std::sys::backtrace::__rust_begin_short_backtracestring_search-f39db1bfbe9ebcc5`test::run_test::_{{closure}} (1 samples, 100.00%)string_search-f39db1bfbe9ebcc5`test::run_test::_{{closure}}string_search-f39db1bfbe9ebcc5`test::__rust_begin_short_backtrace (1 samples, 100.00%)string_search-f39db1bfbe9ebcc5`test::__rust_begin_short_backtracestring_search-f39db1bfbe9ebcc5`string_search::tests::match_existing_available_item_with_benchmarking (1 samples, 100.00%)string_search-f39db1bfbe9ebcc5`string_search::tests::match_existing_available_item_with_benchmarking \ No newline at end of file diff --git a/src/bin/string-search.rs b/src/bin/string-search.rs index 814c3b6..94c5b24 100644 --- a/src/bin/string-search.rs +++ b/src/bin/string-search.rs @@ -19,10 +19,12 @@ //! #![allow(unused_imports, dead_code, unused_variables)] +use crate::benchmarking::contains; use anyhow::anyhow; use anyhow::Error; use std::cell::RefCell; use std::hash::Hash; +use std::hint::black_box; use std::{ collections::HashMap, io::{BufRead, Write}, @@ -59,7 +61,11 @@ impl Entries { } } - pub async fn search(input: &str, amount: f64) -> Result<(String, f64, f64), Error> { + pub async fn search( + input: &str, + amount: f64, + benchmark: bool, + ) -> Result<(String, f64, f64), Error> { let text = input.to_string(); let index_lock: &mut Option> = &mut *ENTRY_MAP.lock().await; @@ -67,22 +73,48 @@ impl Entries { let needle = text; let haystack: Vec = entries.into_iter().map(|el| el.0.to_string()).collect(); - if haystack.contains(&needle) { - if let Some(cost) = entries.get(&needle) { - if amount >= *cost { - Ok((needle, amount, *cost)) + if benchmark { + let hay_slice: Vec = haystack; + let hay_slice = hay_slice.as_slice(); + + if black_box(contains( + black_box(hay_slice), + black_box(needle.to_string()), + )) { + if let Some(cost) = entries.get(&needle) { + if amount >= *cost { + Ok((needle, amount, *cost)) + } else { + return Err(anyhow!( + "Item {} costs more than your offer of ${}", + &needle, + &amount + )); + } } else { - return Err(anyhow!( - "Item {} costs more than your offer of ${}", - &needle, - &amount - )); + unreachable!() } } else { - unreachable!() + return Err(anyhow!("Item {} is not available!", &needle)); } } else { - return Err(anyhow!("Item {} is not available!", &needle)); + if haystack.contains(&needle) { + if let Some(cost) = entries.get(&needle) { + if amount >= *cost { + Ok((needle, amount, *cost)) + } else { + return Err(anyhow!( + "Item {} costs more than your offer of ${}", + &needle, + &amount + )); + } + } else { + unreachable!() + } + } else { + return Err(anyhow!("Item {} is not available!", &needle)); + } } } else { // This would be classed as an internal error, so ideally I would mark this as `unreachable!()`. @@ -93,9 +125,17 @@ impl Entries { #[tokio::main] pub async fn main() -> Result<(), Error> { + // let _ = metrics::match_existing_available_item().await; + Ok(()) } +pub mod benchmarking { + pub fn contains(haystack: &[String], needle: String) -> bool { + haystack.iter().any(|x| x == &needle) + } +} + #[cfg(test)] pub mod tests { use super::*; @@ -114,7 +154,24 @@ pub mod tests { } // Success, the buyer gets an instant match! - let (item, bid, ask) = Entries::search("banana", 20.00).await.unwrap(); + let (item, bid, ask) = Entries::search("banana", 20.00, false).await.unwrap(); + } + + #[tokio::test] + async fn match_existing_available_item_with_benchmarking() { + let items = HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); + Entries::add_many(items).await; + + { + let index_lock: &mut Option> = &mut *ENTRY_MAP.lock().await; + if let Some(entries) = index_lock { + dbg!(&entries); + assert!(entries.len() > 0); + }; + } + + // Success, the buyer gets an instant match! + let (item, bid, ask) = Entries::search("banana", 20.00, true).await.unwrap(); } #[tokio::test] @@ -130,7 +187,7 @@ pub mod tests { }; } - if let Err(err) = Entries::search("banana", 8.23).await { + if let Err(err) = Entries::search("banana", 8.23, false).await { assert_eq!( err.to_string(), String::from("Item banana costs more than your offer of $8.23") @@ -152,7 +209,7 @@ pub mod tests { }; } - Entries::search("fruit", 20.00).await.unwrap(); + Entries::search("fruit", 20.00, false).await.unwrap(); } #[tokio::test] @@ -168,7 +225,7 @@ pub mod tests { }; } - if let Err(err) = Entries::search("red appl", 8.23).await { + if let Err(err) = Entries::search("red appl", 8.23, false).await { assert_eq!( err.to_string(), String::from("Item red appl is not available!") From d6ad6bb0de78efc40ad998bdb75b5554437777a8 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Fri, 23 Aug 2024 10:40:35 +0530 Subject: [PATCH 31/79] Remove comment in main --- src/bin/string-search.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/bin/string-search.rs b/src/bin/string-search.rs index 94c5b24..b367fcb 100644 --- a/src/bin/string-search.rs +++ b/src/bin/string-search.rs @@ -125,8 +125,6 @@ impl Entries { #[tokio::main] pub async fn main() -> Result<(), Error> { - // let _ = metrics::match_existing_available_item().await; - Ok(()) } From 44d082da6d76311a7d22869b86e4d31d7362dad8 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Fri, 23 Aug 2024 11:15:38 +0530 Subject: [PATCH 32/79] FIXME: testing in release mode causes ICE --- src/bin/string-search.rs | 57 ++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/src/bin/string-search.rs b/src/bin/string-search.rs index b367fcb..6a57801 100644 --- a/src/bin/string-search.rs +++ b/src/bin/string-search.rs @@ -34,30 +34,30 @@ use std::{ }; use tokio::sync::Mutex; -pub static ENTRY_MAP: LazyLock>>> = - LazyLock::new(|| Mutex::new(Some(HashMap::new()))); +pub static ENTRY_MAP: LazyLock>> = + LazyLock::new(|| Mutex::new(Some(Entries(HashMap::new())))); pub struct Entries(HashMap); impl Entries { pub async fn add(text: String, amount: f64) { - let index_lock: &mut Option> = &mut *ENTRY_MAP.lock().await; + let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; if let Some(entries) = index_lock { - let mut existing_data = entries.clone(); + let mut existing_data = entries.0.clone(); existing_data.insert(text, amount); - *entries = existing_data; + entries.0 = existing_data; } } pub async fn add_many(items: HashMap<&str, f64>) { - let index_lock: &mut Option> = &mut *ENTRY_MAP.lock().await; + let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; if let Some(entries) = index_lock { let new: HashMap = items .into_iter() .map(|el| (el.0.to_string(), el.1)) .collect(); - *entries = new; + entries.0 = new; } } @@ -68,10 +68,15 @@ impl Entries { ) -> Result<(String, f64, f64), Error> { let text = input.to_string(); - let index_lock: &mut Option> = &mut *ENTRY_MAP.lock().await; + let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; if let Some(entries) = index_lock { let needle = text; - let haystack: Vec = entries.into_iter().map(|el| el.0.to_string()).collect(); + let haystack: Vec = entries + .0 + .clone() + .into_iter() + .map(|el| el.0.to_string()) + .collect(); if benchmark { let hay_slice: Vec = haystack; @@ -81,7 +86,7 @@ impl Entries { black_box(hay_slice), black_box(needle.to_string()), )) { - if let Some(cost) = entries.get(&needle) { + if let Some(cost) = entries.0.get(&needle) { if amount >= *cost { Ok((needle, amount, *cost)) } else { @@ -99,7 +104,7 @@ impl Entries { } } else { if haystack.contains(&needle) { - if let Some(cost) = entries.get(&needle) { + if let Some(cost) = entries.0.get(&needle) { if amount >= *cost { Ok((needle, amount, *cost)) } else { @@ -144,10 +149,10 @@ pub mod tests { Entries::add_many(items).await; { - let index_lock: &mut Option> = &mut *ENTRY_MAP.lock().await; + let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; if let Some(entries) = index_lock { - dbg!(&entries); - assert!(entries.len() > 0); + dbg!(&entries.0); + assert!(entries.0.len() > 0); }; } @@ -161,10 +166,10 @@ pub mod tests { Entries::add_many(items).await; { - let index_lock: &mut Option> = &mut *ENTRY_MAP.lock().await; + let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; if let Some(entries) = index_lock { - dbg!(&entries); - assert!(entries.len() > 0); + dbg!(&entries.0); + assert!(entries.0.len() > 0); }; } @@ -178,10 +183,10 @@ pub mod tests { Entries::add_many(items).await; { - let index_lock: &mut Option> = &mut *ENTRY_MAP.lock().await; + let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; if let Some(entries) = index_lock { - dbg!(&entries); - assert!(entries.len() > 0); + dbg!(&entries.0); + assert!(entries.0.len() > 0); }; } @@ -200,10 +205,10 @@ pub mod tests { Entries::add_many(items).await; { - let index_lock: &mut Option> = &mut *ENTRY_MAP.lock().await; + let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; if let Some(entries) = index_lock { - dbg!(&entries); - assert!(entries.len() > 0); + dbg!(&entries.0); + assert!(entries.0.len() > 0); }; } @@ -216,10 +221,10 @@ pub mod tests { Entries::add_many(items).await; { - let index_lock: &mut Option> = &mut *ENTRY_MAP.lock().await; + let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; if let Some(entries) = index_lock { - dbg!(&entries); - assert!(entries.len() > 0); + dbg!(&entries.0); + assert!(entries.0.len() > 0); }; } From a8bb3d1168faee255ca9656e5cdcbda715131758 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Fri, 23 Aug 2024 11:56:16 +0530 Subject: [PATCH 33/79] FIXME: use `black_box` correctly --- .gitignore | 2 + flamegraph.svg | 491 --------------------------------------- src/bin/string-search.rs | 75 ++---- 3 files changed, 21 insertions(+), 547 deletions(-) delete mode 100644 flamegraph.svg diff --git a/.gitignore b/.gitignore index ff0d847..f48622d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ /target /.vscode Cargo.lock + +flamegraph.svg \ No newline at end of file diff --git a/flamegraph.svg b/flamegraph.svg deleted file mode 100644 index e9e54e7..0000000 --- a/flamegraph.svg +++ /dev/null @@ -1,491 +0,0 @@ -Flame Graph Reset ZoomSearch all (1 samples, 100%)libsystem_pthread.dylib`thread_start (1 samples, 100.00%)libsystem_pthread.dylib`thread_startlibsystem_pthread.dylib`_pthread_start (1 samples, 100.00%)libsystem_pthread.dylib`_pthread_startstring_search-f39db1bfbe9ebcc5`std::sys::pal::unix::thread::Thread::new::thread_start (1 samples, 100.00%)string_search-f39db1bfbe9ebcc5`std::sys::pal::unix::thread::Thread::new::thread_startstring_search-f39db1bfbe9ebcc5`core::ops::function::FnOnce::call_once{{vtable.shim}} (1 samples, 100.00%)string_search-f39db1bfbe9ebcc5`core::ops::function::FnOnce::call_once{{vtable.shim}}string_search-f39db1bfbe9ebcc5`std::sys::backtrace::__rust_begin_short_backtrace (1 samples, 100.00%)string_search-f39db1bfbe9ebcc5`std::sys::backtrace::__rust_begin_short_backtracestring_search-f39db1bfbe9ebcc5`test::run_test::_{{closure}} (1 samples, 100.00%)string_search-f39db1bfbe9ebcc5`test::run_test::_{{closure}}string_search-f39db1bfbe9ebcc5`test::__rust_begin_short_backtrace (1 samples, 100.00%)string_search-f39db1bfbe9ebcc5`test::__rust_begin_short_backtracestring_search-f39db1bfbe9ebcc5`string_search::tests::match_existing_available_item_with_benchmarking (1 samples, 100.00%)string_search-f39db1bfbe9ebcc5`string_search::tests::match_existing_available_item_with_benchmarking \ No newline at end of file diff --git a/src/bin/string-search.rs b/src/bin/string-search.rs index 6a57801..75a23a0 100644 --- a/src/bin/string-search.rs +++ b/src/bin/string-search.rs @@ -17,6 +17,13 @@ //! - Use https://github.com/bheisler/criterion.rs //! - Use https://github.com/nvzqz/divan //! +//! Running Tests +//! +//! +//! Benchmarking and Profiling +//! +//! - Run with `cargo flamegraph --root --unit-test string-search -- tests::match_existing_available_item`. +//! #![allow(unused_imports, dead_code, unused_variables)] use crate::benchmarking::contains; @@ -78,48 +85,22 @@ impl Entries { .map(|el| el.0.to_string()) .collect(); - if benchmark { - let hay_slice: Vec = haystack; - let hay_slice = hay_slice.as_slice(); - - if black_box(contains( - black_box(hay_slice), - black_box(needle.to_string()), - )) { - if let Some(cost) = entries.0.get(&needle) { - if amount >= *cost { - Ok((needle, amount, *cost)) - } else { - return Err(anyhow!( - "Item {} costs more than your offer of ${}", - &needle, - &amount - )); - } + if haystack.contains(&needle) { + if let Some(cost) = entries.0.get(&needle) { + if amount >= *cost { + Ok((needle, amount, *cost)) } else { - unreachable!() + return Err(anyhow!( + "Item {} costs more than your offer of ${}", + &needle, + &amount + )); } } else { - return Err(anyhow!("Item {} is not available!", &needle)); + unreachable!() } } else { - if haystack.contains(&needle) { - if let Some(cost) = entries.0.get(&needle) { - if amount >= *cost { - Ok((needle, amount, *cost)) - } else { - return Err(anyhow!( - "Item {} costs more than your offer of ${}", - &needle, - &amount - )); - } - } else { - unreachable!() - } - } else { - return Err(anyhow!("Item {} is not available!", &needle)); - } + return Err(anyhow!("Item {} is not available!", &needle)); } } else { // This would be classed as an internal error, so ideally I would mark this as `unreachable!()`. @@ -155,26 +136,8 @@ pub mod tests { assert!(entries.0.len() > 0); }; } - - // Success, the buyer gets an instant match! - let (item, bid, ask) = Entries::search("banana", 20.00, false).await.unwrap(); - } - - #[tokio::test] - async fn match_existing_available_item_with_benchmarking() { - let items = HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); - Entries::add_many(items).await; - - { - let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; - if let Some(entries) = index_lock { - dbg!(&entries.0); - assert!(entries.0.len() > 0); - }; - } - // Success, the buyer gets an instant match! - let (item, bid, ask) = Entries::search("banana", 20.00, true).await.unwrap(); + let (item, bid, ask) = black_box(Entries::search("banana", 20.00, true).await.unwrap()); } #[tokio::test] From 2624695eeddaefe1b6f8ae87788cf298b45f51fd Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Fri, 23 Aug 2024 18:18:27 +0530 Subject: [PATCH 34/79] WIP initial profiling with flamegraph --- Cargo.toml | 1 + src/bin/string-search.rs | 21 +++++++++++++-- src/lib.rs | 2 ++ src/utils.rs | 1 + src/utils/generate.rs | 58 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 src/utils.rs create mode 100644 src/utils/generate.rs diff --git a/Cargo.toml b/Cargo.toml index 569f50a..0e89130 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ hex = "^0.4" aes-gcm-siv = "^0.11" aes = "^0.8" heapless = "^0.8" +rand = "0.8.5" axum = { version = "0.6.18", features = ["headers", "tower-log"] } axum-extra = "0.1.2" diff --git a/src/bin/string-search.rs b/src/bin/string-search.rs index 75a23a0..76d749c 100644 --- a/src/bin/string-search.rs +++ b/src/bin/string-search.rs @@ -77,6 +77,8 @@ impl Entries { let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; if let Some(entries) = index_lock { + // println!("Number of items to search: {}", entries.0.len()); + let needle = text; let haystack: Vec = entries .0 @@ -122,11 +124,24 @@ pub mod benchmarking { #[cfg(test)] pub mod tests { + extern crate tutorials; + use super::*; + use tutorials::utils::generate::{float_nums, phrases}; #[tokio::test] async fn match_existing_available_item() { - let items = HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); + static PHRASE_COUNT: i32 = 10_000_000; + let rng_phrases = phrases(PHRASE_COUNT); + let items: HashMap<&str, f64> = rng_phrases + .iter() + .map(|phrase| (phrase.as_str(), float_nums())) + .collect(); + + // let items: HashMap<&str, f64> = + // HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); + // dbg!(&items); + let hm_keys = &items.clone().into_keys().collect::>(); Entries::add_many(items).await; { @@ -136,8 +151,10 @@ pub mod tests { assert!(entries.0.len() > 0); }; } + // Success, the buyer gets an instant match! - let (item, bid, ask) = black_box(Entries::search("banana", 20.00, true).await.unwrap()); + + let (item, bid, ask) = black_box(Entries::search(hm_keys[0], 120.00, true).await.unwrap()); } #[tokio::test] diff --git a/src/lib.rs b/src/lib.rs index 45ad951..a1fc10d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,3 +11,5 @@ pub mod traits; //mod trait_objects_1; pub mod challenge1; pub mod partial_eq; + +pub mod utils; diff --git a/src/utils.rs b/src/utils.rs new file mode 100644 index 0000000..118c66d --- /dev/null +++ b/src/utils.rs @@ -0,0 +1 @@ +pub mod generate; \ No newline at end of file diff --git a/src/utils/generate.rs b/src/utils/generate.rs new file mode 100644 index 0000000..28cd23e --- /dev/null +++ b/src/utils/generate.rs @@ -0,0 +1,58 @@ +//! The MIT License (MIT) +//! +//! Copyright (c) Michael de Silva (https://desilva.io/about) +//! Email: michael@cyberdynea.io +//! +//! Permission is hereby granted, free of charge, to any person obtaining a copy of +//! this software and associated documentation files (the "Software"), to deal in +//! the Software without restriction, including without limitation the rights to +//! use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +//! the Software, and to permit persons to whom the Software is furnished to do so, +//! subject to the following conditions: +//! +//! The above copyright notice and this permission notice shall be included in all +//! copies or substantial portions of the Software. +//! +//! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +//! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +//! FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +//! COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +//! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +//! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +use rand::distributions::{Alphanumeric, DistString}; +use rand::prelude::SliceRandom; +use rand::thread_rng; + +pub fn phrases(count: i32) -> Vec { + let mut rng = thread_rng(); + let mut nums: Vec = (1..8).collect(); + + let data: Vec = (0..count) + .into_iter() + .map(|_| { + nums.shuffle(&mut rng); + + let phrase: Vec = (0..nums[0]) + .into_iter() + .map(|_| -> String { + nums.shuffle(&mut rng); + Alphanumeric.sample_string(&mut rand::thread_rng(), nums[0] as usize) + }) + .collect(); + + phrase.join(" ") + }) + .collect(); + + data +} + +pub fn float_nums() -> f64 { + let mut rng = thread_rng(); + let mut nums: Vec = (19..100).collect(); + + nums.shuffle(&mut rng); + format!("{}.{}", &nums[0], &nums[1]) + .parse() + .expect("Unable to parse String to f64!") +} From ad249bab55bddc6341df7d362ccb016811cfc7be Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Fri, 23 Aug 2024 21:07:19 +0530 Subject: [PATCH 35/79] WIP setup core and another bin to assist with heaptrack memory profiling --- Cargo.toml | 4 ++ src/bin/string-search-heaptrack.rs | 35 +++++++++ src/bin/string-search.rs | 111 +++-------------------------- src/lib.rs | 3 + src/string_search/core.rs | 82 +++++++++++++++++++++ 5 files changed, 135 insertions(+), 100 deletions(-) create mode 100644 src/bin/string-search-heaptrack.rs create mode 100644 src/string_search/core.rs diff --git a/Cargo.toml b/Cargo.toml index 0e89130..70a5cf3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,10 @@ path = "src/bin/tokio-panic-handling.rs" name = "string-search" path = "src/bin/string-search.rs" +[[bin]] +name = "string-search-heaptrack" +path = "src/bin/string-search-heaptrack.rs" + [dependencies] anyhow = "1.0.66" futures = "0.3.19" diff --git a/src/bin/string-search-heaptrack.rs b/src/bin/string-search-heaptrack.rs new file mode 100644 index 0000000..f08a21c --- /dev/null +++ b/src/bin/string-search-heaptrack.rs @@ -0,0 +1,35 @@ +use crate::tutorials::string_search::core::*; +use crate::tutorials::utils::generate::{float_nums, phrases}; +use anyhow::Error; +use std::collections::HashMap; +use std::hint::black_box; + +extern crate tutorials; + +#[tokio::main] +pub async fn main() -> Result<(), Error> { + static PHRASE_COUNT: i32 = 50; + let rng_phrases = phrases(PHRASE_COUNT); + let items: HashMap<&str, f64> = rng_phrases + .iter() + .map(|phrase| (phrase.as_str(), float_nums())) + .collect(); + + let hm_keys = &items.clone().into_keys().collect::>(); + Entries::add_many(items).await; + + { + let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; + if let Some(entries) = index_lock { + assert!(entries.0.len() > 0); + }; + } + + // Success, the buyer gets an instant match! + let _ = black_box( + Entries::search(hm_keys[0], 120.00, true) + .await + .expect("Unable to search through entries!"), + ); + Ok(()) +} diff --git a/src/bin/string-search.rs b/src/bin/string-search.rs index 76d749c..e16725a 100644 --- a/src/bin/string-search.rs +++ b/src/bin/string-search.rs @@ -17,121 +17,33 @@ //! - Use https://github.com/bheisler/criterion.rs //! - Use https://github.com/nvzqz/divan //! -//! Running Tests +//! Running Tests, Benchmarking and Profiling //! +//! - Tests: `cargo t --bin string-search` +//! - Flamegraph: `cargo flamegraph --root --flamechart --verbose --bin string-search-heaptrack` +//! - Heaptrack: TBD //! -//! Benchmarking and Profiling -//! -//! - Run with `cargo flamegraph --root --unit-test string-search -- tests::match_existing_available_item`. -//! -#![allow(unused_imports, dead_code, unused_variables)] +// #![allow(unused_imports, dead_code, unused_variables)] -use crate::benchmarking::contains; -use anyhow::anyhow; use anyhow::Error; -use std::cell::RefCell; -use std::hash::Hash; -use std::hint::black_box; -use std::{ - collections::HashMap, - io::{BufRead, Write}, - ops::Deref, - pin::Pin, - sync::LazyLock, -}; -use tokio::sync::Mutex; - -pub static ENTRY_MAP: LazyLock>> = - LazyLock::new(|| Mutex::new(Some(Entries(HashMap::new())))); - -pub struct Entries(HashMap); - -impl Entries { - pub async fn add(text: String, amount: f64) { - let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; - if let Some(entries) = index_lock { - let mut existing_data = entries.0.clone(); - existing_data.insert(text, amount); - - entries.0 = existing_data; - } - } - pub async fn add_many(items: HashMap<&str, f64>) { - let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; - if let Some(entries) = index_lock { - let new: HashMap = items - .into_iter() - .map(|el| (el.0.to_string(), el.1)) - .collect(); - entries.0 = new; - } - } - - pub async fn search( - input: &str, - amount: f64, - benchmark: bool, - ) -> Result<(String, f64, f64), Error> { - let text = input.to_string(); - - let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; - if let Some(entries) = index_lock { - // println!("Number of items to search: {}", entries.0.len()); - - let needle = text; - let haystack: Vec = entries - .0 - .clone() - .into_iter() - .map(|el| el.0.to_string()) - .collect(); - - if haystack.contains(&needle) { - if let Some(cost) = entries.0.get(&needle) { - if amount >= *cost { - Ok((needle, amount, *cost)) - } else { - return Err(anyhow!( - "Item {} costs more than your offer of ${}", - &needle, - &amount - )); - } - } else { - unreachable!() - } - } else { - return Err(anyhow!("Item {} is not available!", &needle)); - } - } else { - // This would be classed as an internal error, so ideally I would mark this as `unreachable!()`. - return Err(anyhow!("Internal error")); - } - } -} +extern crate tutorials; #[tokio::main] pub async fn main() -> Result<(), Error> { Ok(()) } -pub mod benchmarking { - pub fn contains(haystack: &[String], needle: String) -> bool { - haystack.iter().any(|x| x == &needle) - } -} - #[cfg(test)] pub mod tests { - extern crate tutorials; - - use super::*; - use tutorials::utils::generate::{float_nums, phrases}; + use crate::tutorials::string_search::core::*; + use crate::tutorials::utils::generate::{float_nums, phrases}; + use std::collections::HashMap; + use std::hint::black_box; #[tokio::test] async fn match_existing_available_item() { - static PHRASE_COUNT: i32 = 10_000_000; + static PHRASE_COUNT: i32 = 10_000; let rng_phrases = phrases(PHRASE_COUNT); let items: HashMap<&str, f64> = rng_phrases .iter() @@ -147,7 +59,6 @@ pub mod tests { { let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; if let Some(entries) = index_lock { - dbg!(&entries.0); assert!(entries.0.len() > 0); }; } diff --git a/src/lib.rs b/src/lib.rs index a1fc10d..d1fdad8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,4 +12,7 @@ pub mod traits; pub mod challenge1; pub mod partial_eq; +pub mod string_search { + pub mod core; +} pub mod utils; diff --git a/src/string_search/core.rs b/src/string_search/core.rs new file mode 100644 index 0000000..4ae98f3 --- /dev/null +++ b/src/string_search/core.rs @@ -0,0 +1,82 @@ +// #![allow(unused_imports, dead_code, unused_variables)] + +use anyhow::anyhow; +use anyhow::Error; +use std::{collections::HashMap, sync::LazyLock}; +use tokio::sync::Mutex; + +pub static ENTRY_MAP: LazyLock>> = + LazyLock::new(|| Mutex::new(Some(Entries(HashMap::new())))); + +pub struct Entries(pub HashMap); + +impl Entries { + pub async fn add(text: String, amount: f64) { + let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; + if let Some(entries) = index_lock { + let mut existing_data = entries.0.clone(); + existing_data.insert(text, amount); + + entries.0 = existing_data; + } + } + + pub async fn add_many(items: HashMap<&str, f64>) { + let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; + if let Some(entries) = index_lock { + let new: HashMap = items + .into_iter() + .map(|el| (el.0.to_string(), el.1)) + .collect(); + entries.0 = new; + } + } + + pub async fn search( + input: &str, + amount: f64, + benchmark: bool, + ) -> Result<(String, f64, f64), Error> { + let text = input.to_string(); + + let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; + if let Some(entries) = index_lock { + // println!("Number of items to search: {}", entries.0.len()); + + let needle = text; + let haystack: Vec = entries + .0 + .clone() + .into_iter() + .map(|el| el.0.to_string()) + .collect(); + + if haystack.contains(&needle) { + if let Some(cost) = entries.0.get(&needle) { + if amount >= *cost { + Ok((needle, amount, *cost)) + } else { + return Err(anyhow!( + "Item {} costs more than your offer of ${}", + &needle, + &amount + )); + } + } else { + unreachable!() + } + } else { + return Err(anyhow!("Item {} is not available!", &needle)); + } + } else { + // This would be classed as an internal error, so ideally I would mark this as `unreachable!()`. + return Err(anyhow!("Internal error")); + } + } +} + +pub mod benchmarking { + pub fn contains(haystack: &[String], needle: String) -> bool { + haystack.iter().any(|x| x == &needle) + } +} From f5b5625f014e55510a42f962af765231fa7f1cba Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Fri, 23 Aug 2024 21:16:48 +0530 Subject: [PATCH 36/79] FIX: Strip out need for async, KISS...! --- src/bin/string-search-heaptrack.rs | 9 ++---- src/bin/string-search.rs | 44 ++++++++++++++---------------- src/string_search/core.rs | 25 ++++++----------- 3 files changed, 32 insertions(+), 46 deletions(-) diff --git a/src/bin/string-search-heaptrack.rs b/src/bin/string-search-heaptrack.rs index f08a21c..3166d9a 100644 --- a/src/bin/string-search-heaptrack.rs +++ b/src/bin/string-search-heaptrack.rs @@ -16,20 +16,17 @@ pub async fn main() -> Result<(), Error> { .collect(); let hm_keys = &items.clone().into_keys().collect::>(); - Entries::add_many(items).await; + Entries::add_many(items); { - let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; - if let Some(entries) = index_lock { + if let Ok(entries) = &mut ENTRY_MAP.lock() { assert!(entries.0.len() > 0); }; } // Success, the buyer gets an instant match! let _ = black_box( - Entries::search(hm_keys[0], 120.00, true) - .await - .expect("Unable to search through entries!"), + Entries::search(hm_keys[0], 120.00, true).expect("Unable to search through entries!"), ); Ok(()) } diff --git a/src/bin/string-search.rs b/src/bin/string-search.rs index e16725a..ad43ae8 100644 --- a/src/bin/string-search.rs +++ b/src/bin/string-search.rs @@ -41,8 +41,8 @@ pub mod tests { use std::collections::HashMap; use std::hint::black_box; - #[tokio::test] - async fn match_existing_available_item() { + #[test] + fn match_existing_available_item() { static PHRASE_COUNT: i32 = 10_000; let rng_phrases = phrases(PHRASE_COUNT); let items: HashMap<&str, f64> = rng_phrases @@ -54,34 +54,32 @@ pub mod tests { // HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); // dbg!(&items); let hm_keys = &items.clone().into_keys().collect::>(); - Entries::add_many(items).await; + Entries::add_many(items); { - let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; - if let Some(entries) = index_lock { + if let Ok(entries) = &mut ENTRY_MAP.lock() { assert!(entries.0.len() > 0); }; } // Success, the buyer gets an instant match! - let (item, bid, ask) = black_box(Entries::search(hm_keys[0], 120.00, true).await.unwrap()); + let (item, bid, ask) = black_box(Entries::search(hm_keys[0], 120.00, true).unwrap()); } - #[tokio::test] - async fn panic_expect_error_for_low_bid() { + #[test] + fn panic_expect_error_for_low_bid() { let items = HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); - Entries::add_many(items).await; + Entries::add_many(items); { - let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; - if let Some(entries) = index_lock { + if let Ok(entries) = &mut ENTRY_MAP.lock() { dbg!(&entries.0); assert!(entries.0.len() > 0); }; } - if let Err(err) = Entries::search("banana", 8.23, false).await { + if let Err(err) = Entries::search("banana", 8.23, false) { assert_eq!( err.to_string(), String::from("Item banana costs more than your offer of $8.23") @@ -90,36 +88,34 @@ pub mod tests { } #[should_panic] - #[tokio::test] - async fn handle_mismatching_category() { + #[test] + fn handle_mismatching_category() { let items = HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); - Entries::add_many(items).await; + Entries::add_many(items); { - let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; - if let Some(entries) = index_lock { + if let Ok(entries) = &mut ENTRY_MAP.lock() { dbg!(&entries.0); assert!(entries.0.len() > 0); }; } - Entries::search("fruit", 20.00, false).await.unwrap(); + Entries::search("fruit", 20.00, false).unwrap(); } - #[tokio::test] - async fn panic_mismatching_category_partial_text() { + #[test] + fn panic_mismatching_category_partial_text() { let items = HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); - Entries::add_many(items).await; + Entries::add_many(items); { - let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; - if let Some(entries) = index_lock { + if let Ok(entries) = &mut ENTRY_MAP.lock() { dbg!(&entries.0); assert!(entries.0.len() > 0); }; } - if let Err(err) = Entries::search("red appl", 8.23, false).await { + if let Err(err) = Entries::search("red appl", 8.23, false) { assert_eq!( err.to_string(), String::from("Item red appl is not available!") diff --git a/src/string_search/core.rs b/src/string_search/core.rs index 4ae98f3..e1e572e 100644 --- a/src/string_search/core.rs +++ b/src/string_search/core.rs @@ -2,18 +2,17 @@ use anyhow::anyhow; use anyhow::Error; +use std::sync::Mutex; use std::{collections::HashMap, sync::LazyLock}; -use tokio::sync::Mutex; -pub static ENTRY_MAP: LazyLock>> = - LazyLock::new(|| Mutex::new(Some(Entries(HashMap::new())))); +pub static ENTRY_MAP: LazyLock> = + LazyLock::new(|| Mutex::new(Entries(HashMap::new()))); pub struct Entries(pub HashMap); impl Entries { - pub async fn add(text: String, amount: f64) { - let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; - if let Some(entries) = index_lock { + pub fn add(text: String, amount: f64) { + if let Ok(entries) = &mut ENTRY_MAP.lock() { let mut existing_data = entries.0.clone(); existing_data.insert(text, amount); @@ -21,9 +20,8 @@ impl Entries { } } - pub async fn add_many(items: HashMap<&str, f64>) { - let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; - if let Some(entries) = index_lock { + pub fn add_many(items: HashMap<&str, f64>) { + if let Ok(entries) = &mut ENTRY_MAP.lock() { let new: HashMap = items .into_iter() .map(|el| (el.0.to_string(), el.1)) @@ -32,15 +30,10 @@ impl Entries { } } - pub async fn search( - input: &str, - amount: f64, - benchmark: bool, - ) -> Result<(String, f64, f64), Error> { + pub fn search(input: &str, amount: f64, _: bool) -> Result<(String, f64, f64), Error> { let text = input.to_string(); - let index_lock: &mut Option = &mut *ENTRY_MAP.lock().await; - if let Some(entries) = index_lock { + if let Ok(entries) = &mut ENTRY_MAP.lock() { // println!("Number of items to search: {}", entries.0.len()); let needle = text; From 461b20bfb27c1f55b26eab8eb9e9e481fa49d63d Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Fri, 23 Aug 2024 21:33:24 +0530 Subject: [PATCH 37/79] FIX: Remove unused argument --- src/bin/string-search-heaptrack.rs | 5 ++--- src/bin/string-search.rs | 10 +++++----- src/string_search/core.rs | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/bin/string-search-heaptrack.rs b/src/bin/string-search-heaptrack.rs index 3166d9a..05df740 100644 --- a/src/bin/string-search-heaptrack.rs +++ b/src/bin/string-search-heaptrack.rs @@ -25,8 +25,7 @@ pub async fn main() -> Result<(), Error> { } // Success, the buyer gets an instant match! - let _ = black_box( - Entries::search(hm_keys[0], 120.00, true).expect("Unable to search through entries!"), - ); + let _ = + black_box(Entries::search(hm_keys[0], 120.00).expect("Unable to search through entries!")); Ok(()) } diff --git a/src/bin/string-search.rs b/src/bin/string-search.rs index ad43ae8..546908c 100644 --- a/src/bin/string-search.rs +++ b/src/bin/string-search.rs @@ -54,7 +54,7 @@ pub mod tests { // HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); // dbg!(&items); let hm_keys = &items.clone().into_keys().collect::>(); - Entries::add_many(items); + Entries::add_many(black_box(items)); { if let Ok(entries) = &mut ENTRY_MAP.lock() { @@ -64,7 +64,7 @@ pub mod tests { // Success, the buyer gets an instant match! - let (item, bid, ask) = black_box(Entries::search(hm_keys[0], 120.00, true).unwrap()); + let (item, bid, ask) = black_box(Entries::search(hm_keys[0], 120.00).unwrap()); } #[test] @@ -79,7 +79,7 @@ pub mod tests { }; } - if let Err(err) = Entries::search("banana", 8.23, false) { + if let Err(err) = Entries::search("banana", 8.23) { assert_eq!( err.to_string(), String::from("Item banana costs more than your offer of $8.23") @@ -100,7 +100,7 @@ pub mod tests { }; } - Entries::search("fruit", 20.00, false).unwrap(); + Entries::search("fruit", 20.00).unwrap(); } #[test] @@ -115,7 +115,7 @@ pub mod tests { }; } - if let Err(err) = Entries::search("red appl", 8.23, false) { + if let Err(err) = Entries::search("red appl", 8.23) { assert_eq!( err.to_string(), String::from("Item red appl is not available!") diff --git a/src/string_search/core.rs b/src/string_search/core.rs index e1e572e..d8d15ed 100644 --- a/src/string_search/core.rs +++ b/src/string_search/core.rs @@ -30,7 +30,7 @@ impl Entries { } } - pub fn search(input: &str, amount: f64, _: bool) -> Result<(String, f64, f64), Error> { + pub fn search(input: &str, amount: f64) -> Result<(String, f64, f64), Error> { let text = input.to_string(); if let Ok(entries) = &mut ENTRY_MAP.lock() { From c0f57cf817af1e0632c3cfb5637852ffaae2c613 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sat, 24 Aug 2024 10:23:07 +0530 Subject: [PATCH 38/79] Remove use of LazyLock and switch to `&mut self` to avoid extra cloning of HashMap --- src/bin/string-search-heaptrack.rs | 15 +++--- src/bin/string-search.rs | 52 +++++------------- src/string_search/core.rs | 85 ++++++++++++++---------------- 3 files changed, 59 insertions(+), 93 deletions(-) diff --git a/src/bin/string-search-heaptrack.rs b/src/bin/string-search-heaptrack.rs index 05df740..a92f282 100644 --- a/src/bin/string-search-heaptrack.rs +++ b/src/bin/string-search-heaptrack.rs @@ -16,16 +16,15 @@ pub async fn main() -> Result<(), Error> { .collect(); let hm_keys = &items.clone().into_keys().collect::>(); - Entries::add_many(items); - { - if let Ok(entries) = &mut ENTRY_MAP.lock() { - assert!(entries.0.len() > 0); - }; - } + let mut entries = Entries::new(); + entries.add_many(black_box(items)); // Success, the buyer gets an instant match! - let _ = - black_box(Entries::search(hm_keys[0], 120.00).expect("Unable to search through entries!")); + let _ = black_box( + entries + .search(hm_keys[0], 120.00) + .expect("Unable to search through entries!"), + ); Ok(()) } diff --git a/src/bin/string-search.rs b/src/bin/string-search.rs index 546908c..362f0d6 100644 --- a/src/bin/string-search.rs +++ b/src/bin/string-search.rs @@ -49,37 +49,23 @@ pub mod tests { .iter() .map(|phrase| (phrase.as_str(), float_nums())) .collect(); - - // let items: HashMap<&str, f64> = - // HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); - // dbg!(&items); let hm_keys = &items.clone().into_keys().collect::>(); - Entries::add_many(black_box(items)); - { - if let Ok(entries) = &mut ENTRY_MAP.lock() { - assert!(entries.0.len() > 0); - }; - } + let mut entries = Entries::new(); + entries.add_many(black_box(items)); + assert!(entries.len() > 0); // Success, the buyer gets an instant match! - - let (item, bid, ask) = black_box(Entries::search(hm_keys[0], 120.00).unwrap()); + let _ = black_box(entries.search(hm_keys[0], 120.00).unwrap()); } #[test] fn panic_expect_error_for_low_bid() { let items = HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); - Entries::add_many(items); - - { - if let Ok(entries) = &mut ENTRY_MAP.lock() { - dbg!(&entries.0); - assert!(entries.0.len() > 0); - }; - } + let mut entries = Entries::new(); + entries.add_many(black_box(items)); - if let Err(err) = Entries::search("banana", 8.23) { + if let Err(err) = entries.search("banana", 8.23) { assert_eq!( err.to_string(), String::from("Item banana costs more than your offer of $8.23") @@ -91,31 +77,19 @@ pub mod tests { #[test] fn handle_mismatching_category() { let items = HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); - Entries::add_many(items); + let mut entries = Entries::new(); + entries.add_many(black_box(items)); - { - if let Ok(entries) = &mut ENTRY_MAP.lock() { - dbg!(&entries.0); - assert!(entries.0.len() > 0); - }; - } - - Entries::search("fruit", 20.00).unwrap(); + entries.search("fruit", 20.00).unwrap(); } #[test] fn panic_mismatching_category_partial_text() { let items = HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); - Entries::add_many(items); - - { - if let Ok(entries) = &mut ENTRY_MAP.lock() { - dbg!(&entries.0); - assert!(entries.0.len() > 0); - }; - } + let mut entries = Entries::new(); + entries.add_many(black_box(items)); - if let Err(err) = Entries::search("red appl", 8.23) { + if let Err(err) = entries.search("red appl", 8.23) { assert_eq!( err.to_string(), String::from("Item red appl is not available!") diff --git a/src/string_search/core.rs b/src/string_search/core.rs index d8d15ed..2c9abed 100644 --- a/src/string_search/core.rs +++ b/src/string_search/core.rs @@ -1,69 +1,62 @@ -// #![allow(unused_imports, dead_code, unused_variables)] - use anyhow::anyhow; use anyhow::Error; -use std::sync::Mutex; -use std::{collections::HashMap, sync::LazyLock}; - -pub static ENTRY_MAP: LazyLock> = - LazyLock::new(|| Mutex::new(Entries(HashMap::new()))); +use std::collections::HashMap; pub struct Entries(pub HashMap); impl Entries { - pub fn add(text: String, amount: f64) { - if let Ok(entries) = &mut ENTRY_MAP.lock() { - let mut existing_data = entries.0.clone(); - existing_data.insert(text, amount); + pub fn new() -> Self { + Self(HashMap::new()) + } - entries.0 = existing_data; - } + pub fn len(&self) -> usize { + self.0.len() } - pub fn add_many(items: HashMap<&str, f64>) { - if let Ok(entries) = &mut ENTRY_MAP.lock() { - let new: HashMap = items - .into_iter() - .map(|el| (el.0.to_string(), el.1)) - .collect(); - entries.0 = new; - } + pub fn add(&mut self, text: String, amount: f64) { + let mut existing_data = self.0.clone(); + existing_data.insert(text, amount); + + self.0 = existing_data; + } + + pub fn add_many(&mut self, items: HashMap<&str, f64>) { + let new: HashMap = items + .into_iter() + .map(|el| (el.0.to_string(), el.1)) + .collect(); + self.0 = new; } - pub fn search(input: &str, amount: f64) -> Result<(String, f64, f64), Error> { + pub fn search(&mut self, input: &str, amount: f64) -> Result<(String, f64, f64), Error> { let text = input.to_string(); - if let Ok(entries) = &mut ENTRY_MAP.lock() { - // println!("Number of items to search: {}", entries.0.len()); + // println!("Number of items to search: {}", entries.0.len()); - let needle = text; - let haystack: Vec = entries - .0 - .clone() - .into_iter() - .map(|el| el.0.to_string()) - .collect(); + let needle = text; + let haystack: Vec = self + .0 + .clone() + .into_iter() + .map(|el| el.0.to_string()) + .collect(); - if haystack.contains(&needle) { - if let Some(cost) = entries.0.get(&needle) { - if amount >= *cost { - Ok((needle, amount, *cost)) - } else { - return Err(anyhow!( - "Item {} costs more than your offer of ${}", - &needle, - &amount - )); - } + if haystack.contains(&needle) { + if let Some(cost) = self.0.get(&needle) { + if amount >= *cost { + Ok((needle, amount, *cost)) } else { - unreachable!() + return Err(anyhow!( + "Item {} costs more than your offer of ${}", + &needle, + &amount + )); } } else { - return Err(anyhow!("Item {} is not available!", &needle)); + unreachable!() } } else { - // This would be classed as an internal error, so ideally I would mark this as `unreachable!()`. - return Err(anyhow!("Internal error")); + return Err(anyhow!("Item {} is not available!", &needle)); } } } From 64a7b112d5ce9c98e417e283e354c29d4fd6805d Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sun, 25 Aug 2024 00:24:51 +0530 Subject: [PATCH 39/79] FIX: remove unnecessary cloning of HashMap --- src/string_search/core.rs | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/src/string_search/core.rs b/src/string_search/core.rs index 2c9abed..299f1a3 100644 --- a/src/string_search/core.rs +++ b/src/string_search/core.rs @@ -1,5 +1,6 @@ use anyhow::anyhow; use anyhow::Error; +use rand::seq::IteratorRandom; use std::collections::HashMap; pub struct Entries(pub HashMap); @@ -13,33 +14,25 @@ impl Entries { self.0.len() } - pub fn add(&mut self, text: String, amount: f64) { + pub fn add(&mut self, info: (String, f64)) { + let (text, amount) = info; let mut existing_data = self.0.clone(); existing_data.insert(text, amount); self.0 = existing_data; } - pub fn add_many(&mut self, items: HashMap<&str, f64>) { - let new: HashMap = items - .into_iter() - .map(|el| (el.0.to_string(), el.1)) - .collect(); + pub fn add_many<'a>(&mut self, items: impl IntoIterator) { + let new: HashMap = + items.into_iter().map(|(k, v)| (k.to_string(), v)).collect(); self.0 = new; } pub fn search(&mut self, input: &str, amount: f64) -> Result<(String, f64, f64), Error> { let text = input.to_string(); - - // println!("Number of items to search: {}", entries.0.len()); - let needle = text; - let haystack: Vec = self - .0 - .clone() - .into_iter() - .map(|el| el.0.to_string()) - .collect(); + let haystack: Vec = self.0.iter().map(|el| el.0.to_string()).collect(); + log::debug!("Number of items to search: {}", self.0.len()); if haystack.contains(&needle) { if let Some(cost) = self.0.get(&needle) { From 525f8cd979b31aada295609a7ad6f97afb5846de Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sun, 25 Aug 2024 15:59:06 +0530 Subject: [PATCH 40/79] FIX: remove stupid approach at creating a f64 range --- src/bin/string-search-heaptrack.rs | 11 ++++++++--- src/bin/string-search.rs | 10 ++++++++-- src/utils/generate.rs | 12 +----------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/bin/string-search-heaptrack.rs b/src/bin/string-search-heaptrack.rs index a92f282..0bedfd3 100644 --- a/src/bin/string-search-heaptrack.rs +++ b/src/bin/string-search-heaptrack.rs @@ -1,18 +1,23 @@ use crate::tutorials::string_search::core::*; -use crate::tutorials::utils::generate::{float_nums, phrases}; +use crate::tutorials::utils::generate::phrases; use anyhow::Error; +use rand::{thread_rng, Rng}; use std::collections::HashMap; use std::hint::black_box; +pub fn rng_amount() -> f64 { + let mut rng = thread_rng(); + rng.gen_range(19.0..1.0e2) +} extern crate tutorials; #[tokio::main] pub async fn main() -> Result<(), Error> { - static PHRASE_COUNT: i32 = 50; + static PHRASE_COUNT: i32 = 50_000_000; let rng_phrases = phrases(PHRASE_COUNT); let items: HashMap<&str, f64> = rng_phrases .iter() - .map(|phrase| (phrase.as_str(), float_nums())) + .map(|phrase| (phrase.as_str(), rng_amount())) .collect(); let hm_keys = &items.clone().into_keys().collect::>(); diff --git a/src/bin/string-search.rs b/src/bin/string-search.rs index 362f0d6..dbd50af 100644 --- a/src/bin/string-search.rs +++ b/src/bin/string-search.rs @@ -37,17 +37,23 @@ pub async fn main() -> Result<(), Error> { #[cfg(test)] pub mod tests { use crate::tutorials::string_search::core::*; - use crate::tutorials::utils::generate::{float_nums, phrases}; + use crate::tutorials::utils::generate::phrases; + use rand::{thread_rng, Rng}; use std::collections::HashMap; use std::hint::black_box; + pub fn rng_amount() -> f64 { + let mut rng = thread_rng(); + rng.gen_range(19.0..1.0e2) + } + #[test] fn match_existing_available_item() { static PHRASE_COUNT: i32 = 10_000; let rng_phrases = phrases(PHRASE_COUNT); let items: HashMap<&str, f64> = rng_phrases .iter() - .map(|phrase| (phrase.as_str(), float_nums())) + .map(|phrase| (phrase.as_str(), rng_amount())) .collect(); let hm_keys = &items.clone().into_keys().collect::>(); diff --git a/src/utils/generate.rs b/src/utils/generate.rs index 28cd23e..b9a6232 100644 --- a/src/utils/generate.rs +++ b/src/utils/generate.rs @@ -21,7 +21,7 @@ //! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. use rand::distributions::{Alphanumeric, DistString}; use rand::prelude::SliceRandom; -use rand::thread_rng; +use rand::{thread_rng}; pub fn phrases(count: i32) -> Vec { let mut rng = thread_rng(); @@ -46,13 +46,3 @@ pub fn phrases(count: i32) -> Vec { data } - -pub fn float_nums() -> f64 { - let mut rng = thread_rng(); - let mut nums: Vec = (19..100).collect(); - - nums.shuffle(&mut rng); - format!("{}.{}", &nums[0], &nums[1]) - .parse() - .expect("Unable to parse String to f64!") -} From 3b7b6e278adcb3a663ca7953b8866feb26278838 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sun, 25 Aug 2024 16:11:04 +0530 Subject: [PATCH 41/79] FIX: use rand to generate a random max word length --- src/utils/generate.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/utils/generate.rs b/src/utils/generate.rs index b9a6232..bf4835c 100644 --- a/src/utils/generate.rs +++ b/src/utils/generate.rs @@ -20,23 +20,20 @@ //! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN //! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. use rand::distributions::{Alphanumeric, DistString}; -use rand::prelude::SliceRandom; -use rand::{thread_rng}; +use rand::{thread_rng, Rng}; pub fn phrases(count: i32) -> Vec { let mut rng = thread_rng(); - let mut nums: Vec = (1..8).collect(); + let max_word_characters = rng.gen_range(0..8); let data: Vec = (0..count) .into_iter() .map(|_| { - nums.shuffle(&mut rng); - - let phrase: Vec = (0..nums[0]) + let phrase: Vec = (0..max_word_characters) .into_iter() .map(|_| -> String { - nums.shuffle(&mut rng); - Alphanumeric.sample_string(&mut rand::thread_rng(), nums[0] as usize) + Alphanumeric + .sample_string(&mut rand::thread_rng(), max_word_characters as usize) }) .collect(); From 2250f26c3cf46565e250fcfb33198bb18cc3a18e Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sun, 25 Aug 2024 16:15:54 +0530 Subject: [PATCH 42/79] Add script to run heaptrack --- run_heaptrack.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 run_heaptrack.sh diff --git a/run_heaptrack.sh b/run_heaptrack.sh new file mode 100755 index 0000000..cca6197 --- /dev/null +++ b/run_heaptrack.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -e + +cargo build --bin string-search-heaptrack --release +heaptrack target/release/string-search-heaptrack \ No newline at end of file From 7f73e6e72ea483c6df31233cc1dd4560a7b7464c Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sun, 25 Aug 2024 17:58:34 +0530 Subject: [PATCH 43/79] FIX: Rewrite phrase generation logic with note only optionally failing a generator --- src/bin/string-search-heaptrack.rs | 39 +++++++++++++++++++----------- src/bin/string-search.rs | 23 ++++++++++-------- src/utils/generate.rs | 35 ++++++++++++++------------- 3 files changed, 56 insertions(+), 41 deletions(-) diff --git a/src/bin/string-search-heaptrack.rs b/src/bin/string-search-heaptrack.rs index 0bedfd3..59a00e1 100644 --- a/src/bin/string-search-heaptrack.rs +++ b/src/bin/string-search-heaptrack.rs @@ -13,23 +13,34 @@ extern crate tutorials; #[tokio::main] pub async fn main() -> Result<(), Error> { - static PHRASE_COUNT: i32 = 50_000_000; + static PHRASE_COUNT: i32 = 50; let rng_phrases = phrases(PHRASE_COUNT); - let items: HashMap<&str, f64> = rng_phrases - .iter() - .map(|phrase| (phrase.as_str(), rng_amount())) - .collect(); + // let items: HashMap<&str, f64> = rng_phrases + // .iter() + // .map(|phrase| (phrase.as_str(), rng_amount())) + // .collect(); - let hm_keys = &items.clone().into_keys().collect::>(); + // let hm_keys = &items.clone().into_keys().collect::>(); - let mut entries = Entries::new(); - entries.add_many(black_box(items)); + // let mut entries = Entries::new(); + // entries.add_many(black_box(items)); + + // // Success, the buyer gets an instant match! + // let _ = black_box( + // entries + // .search(hm_keys[0], 120.00) + // .expect("Unable to search through entries!"), + // ); + + if let Ok(phrases) = rng_phrases { + println!( + "Phrases: {}", + phrases + .iter() + .map(|el| { format!("\"{}\", ", el) }) + .collect::() + ); + } - // Success, the buyer gets an instant match! - let _ = black_box( - entries - .search(hm_keys[0], 120.00) - .expect("Unable to search through entries!"), - ); Ok(()) } diff --git a/src/bin/string-search.rs b/src/bin/string-search.rs index dbd50af..7672cc0 100644 --- a/src/bin/string-search.rs +++ b/src/bin/string-search.rs @@ -51,18 +51,21 @@ pub mod tests { fn match_existing_available_item() { static PHRASE_COUNT: i32 = 10_000; let rng_phrases = phrases(PHRASE_COUNT); - let items: HashMap<&str, f64> = rng_phrases - .iter() - .map(|phrase| (phrase.as_str(), rng_amount())) - .collect(); - let hm_keys = &items.clone().into_keys().collect::>(); - let mut entries = Entries::new(); - entries.add_many(black_box(items)); - assert!(entries.len() > 0); + if let Ok(phrases) = rng_phrases { + let items: HashMap<&str, f64> = phrases + .iter() + .map(|phrase| (phrase.as_str(), rng_amount())) + .collect(); + let hm_keys = &items.clone().into_keys().collect::>(); - // Success, the buyer gets an instant match! - let _ = black_box(entries.search(hm_keys[0], 120.00).unwrap()); + let mut entries = Entries::new(); + entries.add_many(black_box(items)); + assert!(entries.len() > 0); + + // Success, the buyer gets an instant match! + let _ = black_box(entries.search(hm_keys[0], 120.00).unwrap()); + } } #[test] diff --git a/src/utils/generate.rs b/src/utils/generate.rs index bf4835c..89ac65b 100644 --- a/src/utils/generate.rs +++ b/src/utils/generate.rs @@ -19,27 +19,28 @@ //! COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER //! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN //! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +use anyhow::Error; use rand::distributions::{Alphanumeric, DistString}; use rand::{thread_rng, Rng}; -pub fn phrases(count: i32) -> Vec { - let mut rng = thread_rng(); - let max_word_characters = rng.gen_range(0..8); +pub fn phrases(count: i32) -> Result, Error> { + let rng_word_count = 8; + let rng_word_length = 10; + let rng_word_length_min = 3; - let data: Vec = (0..count) - .into_iter() + (0..count) + .map(|_| (1..thread_rng().gen_range(1..rng_word_count))) .map(|_| { - let phrase: Vec = (0..max_word_characters) - .into_iter() - .map(|_| -> String { - Alphanumeric - .sample_string(&mut rand::thread_rng(), max_word_characters as usize) - }) - .collect(); - - phrase.join(" ") + Alphanumeric.sample_string( + &mut thread_rng(), + thread_rng().gen_range(rng_word_length_min..rng_word_length) as usize, + ) }) - .collect(); - - data + // NOTE: Optionally, while this example does not generate any empty words, if we wanted to filter through such a scenario, it can be done using a filter to generate an `Option`. + // .map(|word| Some(word).filter(|word| word.len() == 0)) + // .map(|word| { + // word.ok_or(anyhow::anyhow!("Word length is 0!".to_string())); + // }) + .map(|word| Ok(word)) + .collect() } From 227a816bd4adc54dfdcc7690ca3e8e51f058773d Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sun, 25 Aug 2024 18:03:43 +0530 Subject: [PATCH 44/79] FIX: further cleanup for previous change --- src/bin/string-search-heaptrack.rs | 42 ++++++++++++++---------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/src/bin/string-search-heaptrack.rs b/src/bin/string-search-heaptrack.rs index 59a00e1..6305488 100644 --- a/src/bin/string-search-heaptrack.rs +++ b/src/bin/string-search-heaptrack.rs @@ -13,33 +13,31 @@ extern crate tutorials; #[tokio::main] pub async fn main() -> Result<(), Error> { - static PHRASE_COUNT: i32 = 50; + static PHRASE_COUNT: i32 = 10_000_000; let rng_phrases = phrases(PHRASE_COUNT); - // let items: HashMap<&str, f64> = rng_phrases - // .iter() - // .map(|phrase| (phrase.as_str(), rng_amount())) - // .collect(); - - // let hm_keys = &items.clone().into_keys().collect::>(); + if let Ok(phrases) = rng_phrases { + let items: HashMap<&str, f64> = phrases + .iter() + .map(|phrase| (phrase.as_str(), rng_amount())) + .collect(); - // let mut entries = Entries::new(); - // entries.add_many(black_box(items)); + let hm_keys = &items.clone().into_keys().collect::>(); - // // Success, the buyer gets an instant match! - // let _ = black_box( - // entries - // .search(hm_keys[0], 120.00) - // .expect("Unable to search through entries!"), - // ); + let mut entries = Entries::new(); + entries.add_many(black_box(items)); - if let Ok(phrases) = rng_phrases { - println!( - "Phrases: {}", - phrases - .iter() - .map(|el| { format!("\"{}\", ", el) }) - .collect::() + // Success, the buyer gets an instant match! + let _ = black_box( + entries + .search(hm_keys[0], 120.00) + .expect("Unable to search through entries!"), ); + + // let phrase_text = phrases + // .iter() + // .map(|el| format!("\"{}\", ", el)) + // .collect::(); + println!("Phrases generated: {}", &phrases.len()); } Ok(()) From 1f55a4cb71cb2c450142da84041ac54acdd96daa Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sun, 25 Aug 2024 18:11:47 +0530 Subject: [PATCH 45/79] FIX: further cleanup for previous change (again) --- src/utils/generate.rs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/utils/generate.rs b/src/utils/generate.rs index 89ac65b..d157dfa 100644 --- a/src/utils/generate.rs +++ b/src/utils/generate.rs @@ -19,7 +19,7 @@ //! COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER //! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN //! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -use anyhow::Error; +use anyhow::{Context, Error}; use rand::distributions::{Alphanumeric, DistString}; use rand::{thread_rng, Rng}; @@ -30,17 +30,11 @@ pub fn phrases(count: i32) -> Result, Error> { (0..count) .map(|_| (1..thread_rng().gen_range(1..rng_word_count))) - .map(|_| { - Alphanumeric.sample_string( - &mut thread_rng(), - thread_rng().gen_range(rng_word_length_min..rng_word_length) as usize, - ) - }) + .map(|_| thread_rng().gen_range(rng_word_length_min..rng_word_length) as usize) + .map(|length| Alphanumeric.sample_string(&mut thread_rng(), length)) // NOTE: Optionally, while this example does not generate any empty words, if we wanted to filter through such a scenario, it can be done using a filter to generate an `Option`. - // .map(|word| Some(word).filter(|word| word.len() == 0)) - // .map(|word| { - // word.ok_or(anyhow::anyhow!("Word length is 0!".to_string())); - // }) + // .map(|word| Some(word).filter(|word| word.len() != 0)) + // .map(|word| word.context(anyhow::anyhow!("Word length is 0!"))) .map(|word| Ok(word)) .collect() } From 04b44e72427654b85a1bb3d045271dac3ac93eaf Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sun, 25 Aug 2024 21:58:59 +0530 Subject: [PATCH 46/79] FIX: Make sure `Entries::search()` is exercised - Remove unnecessary clone when getting the `HashMap` keys. --- benchmarking/string-search/README.md | 17 +++++++++++++++++ src/bin/string-search-heaptrack.rs | 20 +++++++++++--------- src/bin/string-search.rs | 2 +- 3 files changed, 29 insertions(+), 10 deletions(-) create mode 100644 benchmarking/string-search/README.md diff --git a/benchmarking/string-search/README.md b/benchmarking/string-search/README.md new file mode 100644 index 0000000..38985b5 --- /dev/null +++ b/benchmarking/string-search/README.md @@ -0,0 +1,17 @@ +# string-search + +## flamegraph + +Run as follows, also works on Mac. + +``` +cargo flamegraph --root --flamechart --verbose --bin string-search-heaptrack +``` + + + +## heaptrack + +Run `./run_heaptrack.sh`. Tested on Debian 12. + +Connect to a remote host using `ssh -X` to allow forwarding `heaptrack_gui`. diff --git a/src/bin/string-search-heaptrack.rs b/src/bin/string-search-heaptrack.rs index 6305488..db34205 100644 --- a/src/bin/string-search-heaptrack.rs +++ b/src/bin/string-search-heaptrack.rs @@ -13,25 +13,27 @@ extern crate tutorials; #[tokio::main] pub async fn main() -> Result<(), Error> { - static PHRASE_COUNT: i32 = 10_000_000; + static PHRASE_COUNT: i32 = 500_000; + static INVOCATION_COUNT: i32 = 10_000; let rng_phrases = phrases(PHRASE_COUNT); if let Ok(phrases) = rng_phrases { let items: HashMap<&str, f64> = phrases .iter() .map(|phrase| (phrase.as_str(), rng_amount())) .collect(); - - let hm_keys = &items.clone().into_keys().collect::>(); + let hm_keys = &items.keys().map(|el| el.to_owned()).collect::>(); let mut entries = Entries::new(); entries.add_many(black_box(items)); - // Success, the buyer gets an instant match! - let _ = black_box( - entries - .search(hm_keys[0], 120.00) - .expect("Unable to search through entries!"), - ); + for _loop_idx in 0..INVOCATION_COUNT { + // Success, the buyer gets an instant match! + let _ = black_box( + entries + .search(hm_keys[0], 120.00) + .expect("Unable to search through entries!"), + ); + } // let phrase_text = phrases // .iter() diff --git a/src/bin/string-search.rs b/src/bin/string-search.rs index 7672cc0..ee6644d 100644 --- a/src/bin/string-search.rs +++ b/src/bin/string-search.rs @@ -57,7 +57,7 @@ pub mod tests { .iter() .map(|phrase| (phrase.as_str(), rng_amount())) .collect(); - let hm_keys = &items.clone().into_keys().collect::>(); + let hm_keys = &items.keys().map(|el| el.to_owned()).collect::>(); let mut entries = Entries::new(); entries.add_many(black_box(items)); From 79a9a2a78fa34f814c667073b1fe89398f84c965 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sun, 25 Aug 2024 22:08:42 +0530 Subject: [PATCH 47/79] FIX: Remove remnant map() that has no effect, and unnecessary now --- src/utils/generate.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/utils/generate.rs b/src/utils/generate.rs index d157dfa..263350f 100644 --- a/src/utils/generate.rs +++ b/src/utils/generate.rs @@ -24,12 +24,10 @@ use rand::distributions::{Alphanumeric, DistString}; use rand::{thread_rng, Rng}; pub fn phrases(count: i32) -> Result, Error> { - let rng_word_count = 8; let rng_word_length = 10; let rng_word_length_min = 3; (0..count) - .map(|_| (1..thread_rng().gen_range(1..rng_word_count))) .map(|_| thread_rng().gen_range(rng_word_length_min..rng_word_length) as usize) .map(|length| Alphanumeric.sample_string(&mut thread_rng(), length)) // NOTE: Optionally, while this example does not generate any empty words, if we wanted to filter through such a scenario, it can be done using a filter to generate an `Option`. From c0c32b98c26d02c2c1af71c41d16d43c441b667a Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Mon, 26 Aug 2024 09:44:26 +0530 Subject: [PATCH 48/79] Further cleanup, move tests into lib - Add shell script to run tests --- run_heaptrack.sh => run-heaptrack.sh | 0 src/bin/string-search-heaptrack.rs | 20 +++-- src/bin/string-search.rs | 108 -------------------------- src/string_search/core.rs | 109 +++++++++++++++++++++++++-- src/utils/generate.rs | 1 + test-string-search.sh | 6 ++ 6 files changed, 121 insertions(+), 123 deletions(-) rename run_heaptrack.sh => run-heaptrack.sh (100%) delete mode 100644 src/bin/string-search.rs create mode 100755 test-string-search.sh diff --git a/run_heaptrack.sh b/run-heaptrack.sh similarity index 100% rename from run_heaptrack.sh rename to run-heaptrack.sh diff --git a/src/bin/string-search-heaptrack.rs b/src/bin/string-search-heaptrack.rs index db34205..2ac2c1b 100644 --- a/src/bin/string-search-heaptrack.rs +++ b/src/bin/string-search-heaptrack.rs @@ -11,10 +11,13 @@ pub fn rng_amount() -> f64 { } extern crate tutorials; -#[tokio::main] -pub async fn main() -> Result<(), Error> { - static PHRASE_COUNT: i32 = 500_000; - static INVOCATION_COUNT: i32 = 10_000; +pub fn main() -> Result<(), Error> { + // static PHRASE_COUNT: i32 = 100_000; + // static INVOCATION_COUNT: i32 = 50_000; + + static PHRASE_COUNT: i32 = 20; + static INVOCATION_COUNT: i32 = 5; + let rng_phrases = phrases(PHRASE_COUNT); if let Ok(phrases) = rng_phrases { let items: HashMap<&str, f64> = phrases @@ -35,10 +38,11 @@ pub async fn main() -> Result<(), Error> { ); } - // let phrase_text = phrases - // .iter() - // .map(|el| format!("\"{}\", ", el)) - // .collect::(); + let phrase_text = phrases + .iter() + .map(|el| format!("\"{}\", ", el)) + .collect::(); + dbg!(&phrase_text); println!("Phrases generated: {}", &phrases.len()); } diff --git a/src/bin/string-search.rs b/src/bin/string-search.rs deleted file mode 100644 index ee6644d..0000000 --- a/src/bin/string-search.rs +++ /dev/null @@ -1,108 +0,0 @@ -//! Challenge: -//! -//! - Assume two users are using a site like eBay, but the concept is that user Bob can post an item for sale via a text input field and offer a sell price. -//! - The bidding action is performed purely in memory, so ignore I/O and other complexities. -//! - Think in terms of C++ and pointers. This is a quiz on simple data structures. -//! - Optimise for Big O performance. -//! -//! Design notes -//! -//! - User input is assumed to be all lower case, alphanumerals only, with single spaces. The assumption is that this binary operates on clean data. Validating and sanitising input is a further refinement. -//! - TBD -//! -//! Optimisations: -//! -//! Create small and big benchmarks: -//! -//! - Use https://github.com/bheisler/criterion.rs -//! - Use https://github.com/nvzqz/divan -//! -//! Running Tests, Benchmarking and Profiling -//! -//! - Tests: `cargo t --bin string-search` -//! - Flamegraph: `cargo flamegraph --root --flamechart --verbose --bin string-search-heaptrack` -//! - Heaptrack: TBD -//! -// #![allow(unused_imports, dead_code, unused_variables)] - -use anyhow::Error; - -extern crate tutorials; - -#[tokio::main] -pub async fn main() -> Result<(), Error> { - Ok(()) -} - -#[cfg(test)] -pub mod tests { - use crate::tutorials::string_search::core::*; - use crate::tutorials::utils::generate::phrases; - use rand::{thread_rng, Rng}; - use std::collections::HashMap; - use std::hint::black_box; - - pub fn rng_amount() -> f64 { - let mut rng = thread_rng(); - rng.gen_range(19.0..1.0e2) - } - - #[test] - fn match_existing_available_item() { - static PHRASE_COUNT: i32 = 10_000; - let rng_phrases = phrases(PHRASE_COUNT); - - if let Ok(phrases) = rng_phrases { - let items: HashMap<&str, f64> = phrases - .iter() - .map(|phrase| (phrase.as_str(), rng_amount())) - .collect(); - let hm_keys = &items.keys().map(|el| el.to_owned()).collect::>(); - - let mut entries = Entries::new(); - entries.add_many(black_box(items)); - assert!(entries.len() > 0); - - // Success, the buyer gets an instant match! - let _ = black_box(entries.search(hm_keys[0], 120.00).unwrap()); - } - } - - #[test] - fn panic_expect_error_for_low_bid() { - let items = HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); - let mut entries = Entries::new(); - entries.add_many(black_box(items)); - - if let Err(err) = entries.search("banana", 8.23) { - assert_eq!( - err.to_string(), - String::from("Item banana costs more than your offer of $8.23") - ) - } - } - - #[should_panic] - #[test] - fn handle_mismatching_category() { - let items = HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); - let mut entries = Entries::new(); - entries.add_many(black_box(items)); - - entries.search("fruit", 20.00).unwrap(); - } - - #[test] - fn panic_mismatching_category_partial_text() { - let items = HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); - let mut entries = Entries::new(); - entries.add_many(black_box(items)); - - if let Err(err) = entries.search("red appl", 8.23) { - assert_eq!( - err.to_string(), - String::from("Item red appl is not available!") - ) - } - } -} diff --git a/src/string_search/core.rs b/src/string_search/core.rs index 299f1a3..a4548b7 100644 --- a/src/string_search/core.rs +++ b/src/string_search/core.rs @@ -1,6 +1,32 @@ +//! Challenge: +//! +//! - Assume two users are using a site like eBay, but the concept is that user Bob can post an item for sale via a text input field and offer a sell price. +//! - The bidding action is performed purely in memory, so ignore I/O and other complexities. +//! - Think in terms of C++ and pointers. This is a quiz on simple data structures. +//! - Optimise for Big O performance. +//! +//! Design notes: +//! +//! - User input is assumed to be all lower case, alphanumerals only, with single spaces. The assumption is that this binary operates on clean data. Validating and sanitising input is a further refinement. +//! - TBD +//! +//! Optimisations: +//! +//! Create small and big benchmarks: +//! +//! - Use https://github.com/bheisler/criterion.rs +//! - Use https://github.com/nvzqz/divan +//! +//! Benchmarking and Profiling: +//! +//! Refer to `benchmarking/README.md` +//! +//! Tests: +//! +//! Run `test-string-search.sh` +//! use anyhow::anyhow; use anyhow::Error; -use rand::seq::IteratorRandom; use std::collections::HashMap; pub struct Entries(pub HashMap); @@ -16,10 +42,7 @@ impl Entries { pub fn add(&mut self, info: (String, f64)) { let (text, amount) = info; - let mut existing_data = self.0.clone(); - existing_data.insert(text, amount); - - self.0 = existing_data; + self.0.insert(text, amount); } pub fn add_many<'a>(&mut self, items: impl IntoIterator) { @@ -29,8 +52,7 @@ impl Entries { } pub fn search(&mut self, input: &str, amount: f64) -> Result<(String, f64, f64), Error> { - let text = input.to_string(); - let needle = text; + let needle = input.to_string(); let haystack: Vec = self.0.iter().map(|el| el.0.to_string()).collect(); log::debug!("Number of items to search: {}", self.0.len()); @@ -59,3 +81,76 @@ pub mod benchmarking { haystack.iter().any(|x| x == &needle) } } + +#[cfg(test)] +pub mod tests { + use super::Entries; + use crate::utils::generate::phrases; + use rand::{thread_rng, Rng}; + use std::collections::HashMap; + use std::hint::black_box; + + pub fn rng_amount() -> f64 { + let mut rng = thread_rng(); + rng.gen_range(19.0..1.0e2) + } + + #[test] + fn match_existing_available_item() { + static PHRASE_COUNT: i32 = 10_000; + let rng_phrases = phrases(PHRASE_COUNT); + + if let Ok(phrases) = rng_phrases { + let items: HashMap<&str, f64> = phrases + .iter() + .map(|phrase| (phrase.as_str(), rng_amount())) + .collect(); + let hm_keys = &items.keys().map(|el| el.to_owned()).collect::>(); + + let mut entries = Entries::new(); + entries.add_many(black_box(items)); + assert!(entries.len() > 0); + + // Success, the buyer gets an instant match! + let _ = black_box(entries.search(hm_keys[0], 120.00).unwrap()); + } + } + + #[test] + fn panic_expect_error_for_low_bid() { + let items = HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); + let mut entries = Entries::new(); + entries.add_many(black_box(items)); + + if let Err(err) = entries.search("banana", 8.23) { + assert_eq!( + err.to_string(), + String::from("Item banana costs more than your offer of $8.23") + ) + } + } + + #[should_panic] + #[test] + fn handle_mismatching_category() { + let items = HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); + let mut entries = Entries::new(); + entries.add_many(black_box(items)); + + entries.search("fruit", 20.00).unwrap(); + } + + #[test] + fn panic_mismatching_category_partial_text() { + let items = HashMap::from([("red apple", 20.0), ("ferrari", 32.1), ("banana", 12.99)]); + let mut entries = Entries::new(); + entries.add_many(black_box(items)); + + if let Err(err) = entries.search("red appl", 8.23) { + assert_eq!( + err.to_string(), + String::from("Item red appl is not available!") + ) + } + } +} diff --git a/src/utils/generate.rs b/src/utils/generate.rs index 263350f..8042b25 100644 --- a/src/utils/generate.rs +++ b/src/utils/generate.rs @@ -19,6 +19,7 @@ //! COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER //! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN //! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +#[allow(unused_imports)] use anyhow::{Context, Error}; use rand::distributions::{Alphanumeric, DistString}; use rand::{thread_rng, Rng}; diff --git a/test-string-search.sh b/test-string-search.sh new file mode 100755 index 0000000..b657327 --- /dev/null +++ b/test-string-search.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -e + +cargo b --bin string-search-heaptrack --release +cargo t --lib \ No newline at end of file From cba4920a603fe2d402c2a1b17f1b0c5b9c910e73 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Mon, 26 Aug 2024 09:52:52 +0530 Subject: [PATCH 49/79] WIP tweak invocation count for flamegraph --- src/bin/string-search-heaptrack.rs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/bin/string-search-heaptrack.rs b/src/bin/string-search-heaptrack.rs index 2ac2c1b..beb9f07 100644 --- a/src/bin/string-search-heaptrack.rs +++ b/src/bin/string-search-heaptrack.rs @@ -12,11 +12,8 @@ pub fn rng_amount() -> f64 { extern crate tutorials; pub fn main() -> Result<(), Error> { - // static PHRASE_COUNT: i32 = 100_000; - // static INVOCATION_COUNT: i32 = 50_000; - - static PHRASE_COUNT: i32 = 20; - static INVOCATION_COUNT: i32 = 5; + static PHRASE_COUNT: i32 = 10_000; + static INVOCATION_COUNT: i32 = 20; let rng_phrases = phrases(PHRASE_COUNT); if let Ok(phrases) = rng_phrases { @@ -38,11 +35,11 @@ pub fn main() -> Result<(), Error> { ); } - let phrase_text = phrases - .iter() - .map(|el| format!("\"{}\", ", el)) - .collect::(); - dbg!(&phrase_text); + // let phrase_text = phrases + // .iter() + // .map(|el| format!("\"{}\", ", el)) + // .collect::(); + // dbg!(&phrase_text); println!("Phrases generated: {}", &phrases.len()); } From 9056922cbc2d919ba83384296be66d71a8da2d71 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Tue, 27 Aug 2024 13:39:17 +0530 Subject: [PATCH 50/79] FIX: Do not alloc unnecessary Vec when searching HashMap --- benchmarking/string-search/README.md | 2 ++ src/bin/string-search-heaptrack.rs | 2 +- src/string_search/core.rs | 3 +-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/benchmarking/string-search/README.md b/benchmarking/string-search/README.md index 38985b5..5515249 100644 --- a/benchmarking/string-search/README.md +++ b/benchmarking/string-search/README.md @@ -8,6 +8,8 @@ Run as follows, also works on Mac. cargo flamegraph --root --flamechart --verbose --bin string-search-heaptrack ``` +This example runs 1 Million invocations of `Entries::search()` + ## heaptrack diff --git a/src/bin/string-search-heaptrack.rs b/src/bin/string-search-heaptrack.rs index beb9f07..722ca7b 100644 --- a/src/bin/string-search-heaptrack.rs +++ b/src/bin/string-search-heaptrack.rs @@ -13,7 +13,7 @@ extern crate tutorials; pub fn main() -> Result<(), Error> { static PHRASE_COUNT: i32 = 10_000; - static INVOCATION_COUNT: i32 = 20; + static INVOCATION_COUNT: i32 = 1_000_000; let rng_phrases = phrases(PHRASE_COUNT); if let Ok(phrases) = rng_phrases { diff --git a/src/string_search/core.rs b/src/string_search/core.rs index a4548b7..88e59a9 100644 --- a/src/string_search/core.rs +++ b/src/string_search/core.rs @@ -53,10 +53,9 @@ impl Entries { pub fn search(&mut self, input: &str, amount: f64) -> Result<(String, f64, f64), Error> { let needle = input.to_string(); - let haystack: Vec = self.0.iter().map(|el| el.0.to_string()).collect(); log::debug!("Number of items to search: {}", self.0.len()); - if haystack.contains(&needle) { + if self.0.contains_key(&needle) { if let Some(cost) = self.0.get(&needle) { if amount >= *cost { Ok((needle, amount, *cost)) From 8af53574e487a1b108bfa8cd449fdaac24552021 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Tue, 27 Aug 2024 13:43:23 +0530 Subject: [PATCH 51/79] Add flamegraph --- .gitignore | 2 +- benchmarking/string-search/flamegraph.svg | 491 ++++++++++++++++++++++ 2 files changed, 492 insertions(+), 1 deletion(-) create mode 100644 benchmarking/string-search/flamegraph.svg diff --git a/.gitignore b/.gitignore index f48622d..7ebcb40 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,4 @@ /.vscode Cargo.lock -flamegraph.svg \ No newline at end of file +/flamegraph.svg \ No newline at end of file diff --git a/benchmarking/string-search/flamegraph.svg b/benchmarking/string-search/flamegraph.svg new file mode 100644 index 0000000..19b5554 --- /dev/null +++ b/benchmarking/string-search/flamegraph.svg @@ -0,0 +1,491 @@ +Flame Graph Reset ZoomSearch string-search-heaptrack`tutorials::string_search::core::Entries::search (2 samples, 3.77%)stri..string-search-heaptrack`core::hash::BuildHasher::hash_one (8 samples, 15.09%)string-search-heaptrack..string-search-heaptrack`<core::hash::sip::Hasher<S> as core::hash::Hasher>::write (4 samples, 7.55%)string-sea..string-search-heaptrack`tutorials::string_search::core::Entries::search (16 samples, 30.19%)string-search-heaptrack`tutorials::string_search:..string-search-heaptrack`__rdl_alloc (1 samples, 1.89%)s..string-search-heaptrack`DYLD-STUB$$memcpy (1 samples, 1.89%)s..string-search-heaptrack`<std::collections::hash::map::HashMap<K,V,S> as core::iter::traits::collect::FromIterator<(K,V)>>::from_iter (1 samples, 1.89%)s..string-search-heaptrack`hashbrown::map::HashMap<K,V,S,A>::insert (1 samples, 1.89%)s..libsystem_platform.dylib`_platform_memmove (5 samples, 9.43%)libsystem_pla..libsystem_platform.dylib`_platform_memcmp (11 samples, 20.75%)libsystem_platform.dylib`_platfor..libsystem_malloc.dylib`nanov2_malloc (6 samples, 11.32%)libsystem_malloc...libsystem_malloc.dylib`_nanov2_free (3 samples, 5.66%)libsyst..libsystem_malloc.dylib`_malloc_zone_malloc (2 samples, 3.77%)libs..string-search-heaptrack`string_search_heaptrack::main (47 samples, 88.68%)string-search-heaptrack`string_search_heaptrack::mainstring-search-heaptrack`DYLD-STUB$$free (2 samples, 3.77%)stri..string-search-heaptrack`main (52 samples, 98.11%)string-search-heaptrack`mainstring-search-heaptrack`std::rt::lang_start_internal (52 samples, 98.11%)string-search-heaptrack`std::rt::lang_start_internalstring-search-heaptrack`std::rt::lang_start::_{{closure}} (52 samples, 98.11%)string-search-heaptrack`std::rt::lang_start::_{{closure}}string-search-heaptrack`std::sys::backtrace::__rust_begin_short_backtrace (52 samples, 98.11%)string-search-heaptrack`std::sys::backtrace::__rust_begin_short_backtracelibsystem_malloc.dylib`_free (1 samples, 1.89%)l..all (53 samples, 100%)dyld`start (53 samples, 100.00%)dyld`startlibdyld.dylib`dyld4::LibSystemHelpers::getenv (1 samples, 1.89%)l..libsystem_kernel.dylib`__exit (1 samples, 1.89%)l.. \ No newline at end of file From 8476c2a70ab7a9a8296255846160b9a918227e93 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Tue, 27 Aug 2024 13:56:38 +0530 Subject: [PATCH 52/79] FIX: Clean up string_search/core --- benchmarking/string-search/flamegraph.svg | 4 +-- src/string_search/core.rs | 26 ++++++------------- src/string_search/run-flamegraph.sh | 7 +++++ .../string_search/run-heaptrack.sh | 2 +- test-string-search.sh | 6 ----- 5 files changed, 18 insertions(+), 27 deletions(-) create mode 100755 src/string_search/run-flamegraph.sh rename run-heaptrack.sh => src/string_search/run-heaptrack.sh (57%) delete mode 100755 test-string-search.sh diff --git a/benchmarking/string-search/flamegraph.svg b/benchmarking/string-search/flamegraph.svg index 19b5554..495c868 100644 --- a/benchmarking/string-search/flamegraph.svg +++ b/benchmarking/string-search/flamegraph.svg @@ -1,4 +1,4 @@ - \ No newline at end of file diff --git a/src/string_search/core.rs b/src/string_search/core.rs index 88e59a9..5b4704d 100644 --- a/src/string_search/core.rs +++ b/src/string_search/core.rs @@ -55,19 +55,15 @@ impl Entries { let needle = input.to_string(); log::debug!("Number of items to search: {}", self.0.len()); - if self.0.contains_key(&needle) { - if let Some(cost) = self.0.get(&needle) { - if amount >= *cost { - Ok((needle, amount, *cost)) - } else { - return Err(anyhow!( - "Item {} costs more than your offer of ${}", - &needle, - &amount - )); - } + if let Some(cost) = self.0.get(&needle) { + if amount >= *cost { + Ok((needle, amount, *cost)) } else { - unreachable!() + return Err(anyhow!( + "Item {} costs more than your offer of ${}", + &needle, + &amount + )); } } else { return Err(anyhow!("Item {} is not available!", &needle)); @@ -75,12 +71,6 @@ impl Entries { } } -pub mod benchmarking { - pub fn contains(haystack: &[String], needle: String) -> bool { - haystack.iter().any(|x| x == &needle) - } -} - #[cfg(test)] pub mod tests { use super::Entries; diff --git a/src/string_search/run-flamegraph.sh b/src/string_search/run-flamegraph.sh new file mode 100755 index 0000000..7c8ecc0 --- /dev/null +++ b/src/string_search/run-flamegraph.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +cargo b --bin string-search-heaptrack --release +cargo t --lib +cargo flamegraph --root --flamechart --verbose --bin string-search-heaptrack \ No newline at end of file diff --git a/run-heaptrack.sh b/src/string_search/run-heaptrack.sh similarity index 57% rename from run-heaptrack.sh rename to src/string_search/run-heaptrack.sh index cca6197..267aa3c 100755 --- a/run-heaptrack.sh +++ b/src/string_search/run-heaptrack.sh @@ -3,4 +3,4 @@ set -e cargo build --bin string-search-heaptrack --release -heaptrack target/release/string-search-heaptrack \ No newline at end of file +heaptrack ../../target/release/string-search-heaptrack \ No newline at end of file diff --git a/test-string-search.sh b/test-string-search.sh deleted file mode 100755 index b657327..0000000 --- a/test-string-search.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -e - -cargo b --bin string-search-heaptrack --release -cargo t --lib \ No newline at end of file From 70f7eb65b9c30acdbd569d7e65c6476f8dcc5142 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Tue, 27 Aug 2024 13:59:15 +0530 Subject: [PATCH 53/79] FIX: use relative path for running heaptrack --- src/string_search/run-heaptrack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/string_search/run-heaptrack.sh b/src/string_search/run-heaptrack.sh index 267aa3c..cca6197 100755 --- a/src/string_search/run-heaptrack.sh +++ b/src/string_search/run-heaptrack.sh @@ -3,4 +3,4 @@ set -e cargo build --bin string-search-heaptrack --release -heaptrack ../../target/release/string-search-heaptrack \ No newline at end of file +heaptrack target/release/string-search-heaptrack \ No newline at end of file From 6f493607ba2f516622ab0d1ad17c652c31856252 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Tue, 27 Aug 2024 14:03:57 +0530 Subject: [PATCH 54/79] Readability clean up --- src/string_search/core.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/string_search/core.rs b/src/string_search/core.rs index 5b4704d..fa2c520 100644 --- a/src/string_search/core.rs +++ b/src/string_search/core.rs @@ -51,13 +51,12 @@ impl Entries { self.0 = new; } - pub fn search(&mut self, input: &str, amount: f64) -> Result<(String, f64, f64), Error> { - let needle = input.to_string(); + pub fn search(&mut self, needle: &str, amount: f64) -> Result<(String, f64, f64), Error> { log::debug!("Number of items to search: {}", self.0.len()); - if let Some(cost) = self.0.get(&needle) { + if let Some(cost) = self.0.get(needle) { if amount >= *cost { - Ok((needle, amount, *cost)) + Ok((needle.to_string(), amount, *cost)) } else { return Err(anyhow!( "Item {} costs more than your offer of ${}", From df0702cb157e1d38b8b8a3806f4d34f5629ac2e0 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Tue, 27 Aug 2024 14:12:31 +0530 Subject: [PATCH 55/79] FIX: Entries::search() returns cost of found needle only --- benchmarking/string-search/flamegraph.svg | 4 ++-- src/string_search/core.rs | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/benchmarking/string-search/flamegraph.svg b/benchmarking/string-search/flamegraph.svg index 495c868..d8aa5fd 100644 --- a/benchmarking/string-search/flamegraph.svg +++ b/benchmarking/string-search/flamegraph.svg @@ -1,4 +1,4 @@ - \ No newline at end of file diff --git a/src/string_search/core.rs b/src/string_search/core.rs index fa2c520..d1e134f 100644 --- a/src/string_search/core.rs +++ b/src/string_search/core.rs @@ -51,17 +51,17 @@ impl Entries { self.0 = new; } - pub fn search(&mut self, needle: &str, amount: f64) -> Result<(String, f64, f64), Error> { + pub fn search(&mut self, needle: &str, offered_amount: f64) -> Result { log::debug!("Number of items to search: {}", self.0.len()); if let Some(cost) = self.0.get(needle) { - if amount >= *cost { - Ok((needle.to_string(), amount, *cost)) + if offered_amount >= *cost { + Ok(*cost) } else { return Err(anyhow!( "Item {} costs more than your offer of ${}", &needle, - &amount + &offered_amount )); } } else { From 9a5a14fc7d8230fcdd0f4c64ae9c50cdbcd34530 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Tue, 27 Aug 2024 16:55:48 +0530 Subject: [PATCH 56/79] Add `phrases_randomized` --- Cargo.toml | 4 ++++ src/bin/phrases-randomized.rs | 29 +++++++++++++++++++++++++++++ src/utils/generate.rs | 16 ++++++++++++++++ 3 files changed, 49 insertions(+) create mode 100644 src/bin/phrases-randomized.rs diff --git a/Cargo.toml b/Cargo.toml index 70a5cf3..37dc3f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,10 @@ path = "src/bin/string-search.rs" name = "string-search-heaptrack" path = "src/bin/string-search-heaptrack.rs" +[[bin]] +name = "phrases-randomized" +path = "src/bin/phrases-randomized.rs" + [dependencies] anyhow = "1.0.66" futures = "0.3.19" diff --git a/src/bin/phrases-randomized.rs b/src/bin/phrases-randomized.rs new file mode 100644 index 0000000..f67e08f --- /dev/null +++ b/src/bin/phrases-randomized.rs @@ -0,0 +1,29 @@ +use crate::tutorials::string_search::core::*; +use anyhow::Error; +use rand::{thread_rng, Rng}; +use std::collections::HashMap; +use std::hint::black_box; +use tutorials::utils::generate::phrases_randomized; + +extern crate tutorials; + +pub fn main() -> Result<(), Error> { + static PHRASE_COUNT: i32 = 20; + // static INVOCATION_COUNT: i32 = 1_000; + + let rng_phrases = phrases_randomized(PHRASE_COUNT.try_into()?, 3); + if let Ok(phrases) = rng_phrases { + // for _loop_idx in 0..INVOCATION_COUNT { + // let _ = black_box(()); + // } + + let phrase_text = phrases + .iter() + .map(|el| format!("\"{}\", ", el)) + .collect::(); + // dbg!(&phrase_text); + println!("Phrases generated [{}]: {}", &phrases.len(), &phrase_text); + } + + Ok(()) +} diff --git a/src/utils/generate.rs b/src/utils/generate.rs index 8042b25..72ab30f 100644 --- a/src/utils/generate.rs +++ b/src/utils/generate.rs @@ -23,6 +23,7 @@ use anyhow::{Context, Error}; use rand::distributions::{Alphanumeric, DistString}; use rand::{thread_rng, Rng}; +use tokio::time::error::Elapsed; pub fn phrases(count: i32) -> Result, Error> { let rng_word_length = 10; @@ -37,3 +38,18 @@ pub fn phrases(count: i32) -> Result, Error> { .map(|word| Ok(word)) .collect() } + +pub fn phrases_randomized(count: usize, words_per_phrase: usize) -> Result, Error> { + let rng_word_length = 10; + let rng_word_length_min = 3; + + Ok((0..count) + .map(|_| { + (0..words_per_phrase) + .map(|_| thread_rng().gen_range(rng_word_length_min..rng_word_length) as usize) + .map(|length| Alphanumeric.sample_string(&mut thread_rng(), length)) + .collect::>() + .join(" ") + }) + .collect()) +} From 0b5a059a671f3224b339057656b7da4201e9e702 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Wed, 4 Sep 2024 09:47:29 +0530 Subject: [PATCH 57/79] Add example on a swappable storage backend --- .gitignore | 3 +- .../pluggable-storage/.gitignore | 1 + .../pluggable-storage/Cargo.toml | 14 ++ .../pluggable-storage/rust-toolchain.toml | 2 + .../pluggable-storage/src/backend.rs | 12 ++ .../pluggable-storage/src/backend/disk.rs | 85 +++++++++++ .../pluggable-storage/src/backend/s3.rs | 92 ++++++++++++ .../pluggable-storage/src/bin/disk-backend.rs | 15 ++ .../pluggable-storage/src/bin/s3-backend.rs | 21 +++ .../pluggable-storage/src/lib.rs | 11 ++ .../trait-objects-downcast/Cargo.toml | 9 ++ .../trait-objects-downcast/scatch.rs | 38 +++++ .../trait-objects-downcast/src/main.rs | 137 ++++++++++++++++++ .../Cargo.toml | 9 ++ .../scatch.rs | 38 +++++ .../src/main.rs | 130 +++++++++++++++++ 16 files changed, 616 insertions(+), 1 deletion(-) create mode 100644 examples/traits-dyn-generics/pluggable-storage/.gitignore create mode 100644 examples/traits-dyn-generics/pluggable-storage/Cargo.toml create mode 100644 examples/traits-dyn-generics/pluggable-storage/rust-toolchain.toml create mode 100644 examples/traits-dyn-generics/pluggable-storage/src/backend.rs create mode 100644 examples/traits-dyn-generics/pluggable-storage/src/backend/disk.rs create mode 100644 examples/traits-dyn-generics/pluggable-storage/src/backend/s3.rs create mode 100644 examples/traits-dyn-generics/pluggable-storage/src/bin/disk-backend.rs create mode 100644 examples/traits-dyn-generics/pluggable-storage/src/bin/s3-backend.rs create mode 100644 examples/traits-dyn-generics/pluggable-storage/src/lib.rs create mode 100644 examples/traits-dyn-generics/trait-objects-downcast/Cargo.toml create mode 100644 examples/traits-dyn-generics/trait-objects-downcast/scatch.rs create mode 100644 examples/traits-dyn-generics/trait-objects-downcast/src/main.rs create mode 100644 examples/traits-dyn-generics/trait-objects-without-downcast-generics/Cargo.toml create mode 100644 examples/traits-dyn-generics/trait-objects-without-downcast-generics/scatch.rs create mode 100644 examples/traits-dyn-generics/trait-objects-without-downcast-generics/src/main.rs diff --git a/.gitignore b/.gitignore index 7ebcb40..d3ae36e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /target +/examples/**/target/* /.vscode Cargo.lock -/flamegraph.svg \ No newline at end of file +/flamegraph.svg diff --git a/examples/traits-dyn-generics/pluggable-storage/.gitignore b/examples/traits-dyn-generics/pluggable-storage/.gitignore new file mode 100644 index 0000000..c41cc9e --- /dev/null +++ b/examples/traits-dyn-generics/pluggable-storage/.gitignore @@ -0,0 +1 @@ +/target \ No newline at end of file diff --git a/examples/traits-dyn-generics/pluggable-storage/Cargo.toml b/examples/traits-dyn-generics/pluggable-storage/Cargo.toml new file mode 100644 index 0000000..e0ea049 --- /dev/null +++ b/examples/traits-dyn-generics/pluggable-storage/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "pluggable-storage" +version = "0.1.0" +authors = ["Michael de Silva impl Future> + Send; +} diff --git a/examples/traits-dyn-generics/pluggable-storage/src/backend/disk.rs b/examples/traits-dyn-generics/pluggable-storage/src/backend/disk.rs new file mode 100644 index 0000000..702f8ee --- /dev/null +++ b/examples/traits-dyn-generics/pluggable-storage/src/backend/disk.rs @@ -0,0 +1,85 @@ +use anyhow::{Context, Result}; +use std::fmt::Debug; +use std::ops::{Deref, DerefMut}; +use std::path::PathBuf; + +use super::Backend; + +#[derive(Debug)] +pub struct FileBackend { + path: PathBuf, +} + +impl FileBackend { + fn new(s: &str) -> Self { + Self { path: s.into() } + } +} + +pub trait FileBackendTrait { + fn new(s: &str) -> Self; +} + +impl FileBackendTrait for FileBackend { + // NOTE: This is a provided method via the Trait `FileBackendTrait` + fn new(s: &str) -> Self { + // NOTE: this is an associated method on `FileBackend` + Self::new(s) + } +} + +#[derive(Debug)] +pub struct FileBackendBuilder(T); + +// Example on implemeting Deref for an inner type. +impl Deref for FileBackendBuilder { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for FileBackendBuilder { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl FileBackendBuilder +where + T: FileBackendTrait, +{ + pub fn new(path: &str) -> Self { + Self(T::new(path)) + } + + pub fn build(self) -> T { + self.0 + } +} + +impl Backend for FileBackend { + async fn persist(&mut self, data: &[u8]) -> Result<()> { + if let Err(err) = tokio::fs::write(&self.path, data).await { + return Err(err).with_context(|| format!("Failed to persist to {:?}", self.path)); + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[should_panic] + #[tokio::test] + async fn panic_when_writing_to_missing_directory() { + let data = String::from("If my calculations are correct, when this baby hits eighty-eight miles per hour... you're gonna see some serious s**t."); + let backend_builder: FileBackendBuilder = FileBackendBuilder::new("./missing/output.json"); + let backend = &mut backend_builder.build(); + + backend.persist(data.as_bytes()).await.unwrap(); + } +} diff --git a/examples/traits-dyn-generics/pluggable-storage/src/backend/s3.rs b/examples/traits-dyn-generics/pluggable-storage/src/backend/s3.rs new file mode 100644 index 0000000..0aef031 --- /dev/null +++ b/examples/traits-dyn-generics/pluggable-storage/src/backend/s3.rs @@ -0,0 +1,92 @@ +#[allow(unused_imports)] +use anyhow::{Context, Result}; +use derive_builder::Builder; +use std::fmt::Debug; +use std::ops::{Deref, DerefMut}; + +use super::Backend; + +#[derive(Default, Builder, Debug)] +#[builder(setter(into))] +pub struct S3Config { + bucket: String, + file_name: String, +} + +#[allow(dead_code)] +#[derive(Debug)] +pub struct S3Backend { + config: S3Config, +} + +impl S3Backend { + fn new(config: S3Config) -> Self { + Self { config } + } + + pub fn bucket(&self) -> String { + self.config.bucket.to_string() + } + + pub fn file_name(&self) -> String { + self.config.file_name.to_string() + } +} + +pub trait S3BackendTrait { + fn new(config: S3Config) -> Self; +} + +impl S3BackendTrait for S3Backend { + // NOTE: This is a provided method via the Trait `S3BackendTrait` + fn new(config: S3Config) -> Self { + // NOTE: this is an associated method on `S3Backend` + Self::new(config) + } +} + +#[derive(Debug)] +pub struct S3BackendBuilder(T); + +// Example on implemeting Deref for an inner type. +impl Deref for S3BackendBuilder { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for S3BackendBuilder { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl S3BackendBuilder +where + T: S3BackendTrait, +{ + pub fn new(config: S3Config) -> Self { + Self(T::new(config)) + } + + pub fn build(self) -> T { + self.0 + } +} + +impl Backend for S3Backend { + async fn persist(&mut self, data: &[u8]) -> Result<()> { + let bucket_name = &self.bucket(); + let file_name = &self.file_name(); + + unimplemented!(); + + // if let Err(err) = _ { + // return Err(err).with_context(|| format!("Failed to persist to {:?}", self.path)); + // } + + Ok(()) + } +} diff --git a/examples/traits-dyn-generics/pluggable-storage/src/bin/disk-backend.rs b/examples/traits-dyn-generics/pluggable-storage/src/bin/disk-backend.rs new file mode 100644 index 0000000..7e9895e --- /dev/null +++ b/examples/traits-dyn-generics/pluggable-storage/src/bin/disk-backend.rs @@ -0,0 +1,15 @@ +use pluggable_storage::prelude::*; + +extern crate pluggable_storage; + +#[tokio::main] +async fn main() -> Result<()> { + // This will be replaced with a source of data using Serde + let data = String::default(); + let backend_builder: FileBackendBuilder = FileBackendBuilder::new("./output/output.json"); + let backend = &mut backend_builder.build(); + + backend.persist(data.as_bytes()).await?; + + Ok(()) +} diff --git a/examples/traits-dyn-generics/pluggable-storage/src/bin/s3-backend.rs b/examples/traits-dyn-generics/pluggable-storage/src/bin/s3-backend.rs new file mode 100644 index 0000000..b0dc6d2 --- /dev/null +++ b/examples/traits-dyn-generics/pluggable-storage/src/bin/s3-backend.rs @@ -0,0 +1,21 @@ +use pluggable_storage::prelude::*; + +extern crate pluggable_storage; + +#[tokio::main] +async fn main() -> Result<()> { + // This will be replaced with a source of data using Serde + let data = String::default(); + + let config = S3ConfigBuilder::default() + // block + .bucket("pluggable_storage_demo") + .file_name("output.json") + .build()?; + let backend_builder: S3BackendBuilder = S3BackendBuilder::new(config); + let backend = &mut backend_builder.build(); + + backend.persist(data.as_bytes()).await?; + + Ok(()) +} diff --git a/examples/traits-dyn-generics/pluggable-storage/src/lib.rs b/examples/traits-dyn-generics/pluggable-storage/src/lib.rs new file mode 100644 index 0000000..9571154 --- /dev/null +++ b/examples/traits-dyn-generics/pluggable-storage/src/lib.rs @@ -0,0 +1,11 @@ +pub(crate) mod backend; + +/// Re-exports +pub mod prelude { + pub use crate::backend::{ + disk::{FileBackend, FileBackendBuilder}, + s3::{S3Backend, S3BackendBuilder, S3Config, S3ConfigBuilder}, + Backend, + }; + pub use anyhow::{Context, Result}; +} diff --git a/examples/traits-dyn-generics/trait-objects-downcast/Cargo.toml b/examples/traits-dyn-generics/trait-objects-downcast/Cargo.toml new file mode 100644 index 0000000..c6567d6 --- /dev/null +++ b/examples/traits-dyn-generics/trait-objects-downcast/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "trait-objects" +version = "0.1.0" +authors = ["Michael de Silva (value: &T) { + let value_any = value as &dyn Any; + + // Try to convert our value its concrete type + match value_any.downcast_ref::() { + Some(as_string) => { + println!("String ({}): {}", as_string.len(), as_string); + } + None => { + println!("{value:?} is not a String"); + } + } + + match value_any.downcast_ref::() { + Some(as_i32) => { + println!("i32: {}", as_i32); + } + None => { + println!("{value:?} is not an i32"); + } + } + } + + pub mod box_deref { + use super::*; + + fn example() { + let boxed: Box = Box::new(3_i32); + // You're more likely to want this: + let actual_id = (&*boxed).type_id(); + // ... than this: + let boxed_id = boxed.type_id(); + + assert_eq!(actual_id, TypeId::of::()); + assert_eq!(boxed_id, TypeId::of::>()); + } + } + \ No newline at end of file diff --git a/examples/traits-dyn-generics/trait-objects-downcast/src/main.rs b/examples/traits-dyn-generics/trait-objects-downcast/src/main.rs new file mode 100644 index 0000000..55d287f --- /dev/null +++ b/examples/traits-dyn-generics/trait-objects-downcast/src/main.rs @@ -0,0 +1,137 @@ +use downcast_rs::{impl_downcast, Downcast}; +use rand; +use rand::seq::SliceRandom; +use rand::Rng; +use std::any::{Any, TypeId}; +use std::fmt::Debug; +use std::fmt::Display; +use std::ops::{Deref, DerefMut}; + +#[derive(Debug)] +struct Dwarf {} + +impl Display for Dwarf { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "I may be small but the ladies don't complain!") + } +} + +#[derive(Debug)] +struct Elf {} + +#[derive(Debug)] +struct Human {} + +#[derive(Debug)] +enum Thing { + Sword, + Trinket, +} + +trait Enchanter: std::fmt::Debug { + fn competency(&self) -> f64; + + fn enchant(&self, thing: &mut Thing) { + let probability_of_success = self.competency(); + let spell_is_successful = rand::thread_rng().gen_bool(probability_of_success); // <1> + + print!("{:?} mutters incoherently. ", self); + if spell_is_successful { + println!("The {:?} glows brightly.", thing); + } else { + println!( + "The {:?} fizzes, \ + then turns into a worthless trinket.", + thing + ); + *thing = Thing::Trinket {}; + } + } +} + +impl Enchanter for Dwarf { + fn competency(&self) -> f64 { + 0.5 // <2> + } +} +impl Enchanter for Elf { + fn competency(&self) -> f64 { + 0.95 // <3> + } +} +impl Enchanter for Human { + fn competency(&self) -> f64 { + 0.8 // <4> + } +} + +/// Holds the log value, stored on the heap. +#[derive(Debug)] +struct LogRecord { + pub text: String, +} + +impl LogRecord { + fn new() -> Self { + Self { + text: String::default(), + } + } +} + +trait Recordable: std::fmt::Debug + Downcast { + fn set_text(&mut self, _: String) {} + + fn text(&self) -> String; +} +impl_downcast!(Recordable); + +impl Recordable for LogRecord { + fn set_text(&mut self, value: String) { + self.text = value; + } + + fn text(&self) -> String { + self.text.to_string() + } +} + +// This is a pointless downcast, as the same can be achieved with generics. +fn log_enchanted<'a, T: Any + Debug>(value: &T, l: &'a mut impl Recordable) -> &'a dyn Recordable { + let value_any = value as &dyn Any; + + // Try to convert our value its concrete type + match value_any.downcast_ref::() { + Some(_) => { + l.set_text("Default text within the recorder".to_string()); + + l + } + _ => l, + } +} + +fn main() { + let mut it = Thing::Sword; + + let d = Dwarf {}; + let e = Elf {}; + let h = Human {}; + + let party: Vec<&dyn Enchanter> = vec![&d, &h, &e]; // <5> + + // This is a contrived example to try and downcase from a leaked Box, to its concrete type without breaking Safety. + let d = Dwarf {}; + let mut l = LogRecord::new(); + let r = log_enchanted(&d, &mut l); + assert_eq!("Default text within the recorder".to_string(), r.text()); + + match r.downcast_ref::() { + Some(record) => println!("Record: {}", record.text), + None => println!("Downcast failed"), + } + + let spellcaster = party.choose(&mut rand::thread_rng()).unwrap(); + + spellcaster.enchant(&mut it); +} diff --git a/examples/traits-dyn-generics/trait-objects-without-downcast-generics/Cargo.toml b/examples/traits-dyn-generics/trait-objects-without-downcast-generics/Cargo.toml new file mode 100644 index 0000000..c6567d6 --- /dev/null +++ b/examples/traits-dyn-generics/trait-objects-without-downcast-generics/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "trait-objects" +version = "0.1.0" +authors = ["Michael de Silva (value: &T) { + let value_any = value as &dyn Any; + + // Try to convert our value its concrete type + match value_any.downcast_ref::() { + Some(as_string) => { + println!("String ({}): {}", as_string.len(), as_string); + } + None => { + println!("{value:?} is not a String"); + } + } + + match value_any.downcast_ref::() { + Some(as_i32) => { + println!("i32: {}", as_i32); + } + None => { + println!("{value:?} is not an i32"); + } + } + } + + pub mod box_deref { + use super::*; + + fn example() { + let boxed: Box = Box::new(3_i32); + // You're more likely to want this: + let actual_id = (&*boxed).type_id(); + // ... than this: + let boxed_id = boxed.type_id(); + + assert_eq!(actual_id, TypeId::of::()); + assert_eq!(boxed_id, TypeId::of::>()); + } + } + \ No newline at end of file diff --git a/examples/traits-dyn-generics/trait-objects-without-downcast-generics/src/main.rs b/examples/traits-dyn-generics/trait-objects-without-downcast-generics/src/main.rs new file mode 100644 index 0000000..a467fc8 --- /dev/null +++ b/examples/traits-dyn-generics/trait-objects-without-downcast-generics/src/main.rs @@ -0,0 +1,130 @@ +use downcast_rs::{impl_downcast, Downcast}; +use rand; +use rand::seq::SliceRandom; +use rand::Rng; +use std::any::{Any, TypeId}; +use std::fmt::Debug; +use std::fmt::Display; +use std::ops::{Deref, DerefMut}; + +#[derive(Debug)] +struct Dwarf {} + +impl Display for Dwarf { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "I may be small but the ladies don't complain!") + } +} + +#[derive(Debug)] +struct Elf {} + +#[derive(Debug)] +struct Human {} + +#[derive(Debug)] +enum Thing { + Sword, + Trinket, +} + +trait Enchanter: std::fmt::Debug { + fn competency(&self) -> f64; + + fn enchant(&self, thing: &mut Thing) { + let probability_of_success = self.competency(); + let spell_is_successful = rand::thread_rng().gen_bool(probability_of_success); // <1> + + print!("{:?} mutters incoherently. ", self); + if spell_is_successful { + println!("The {:?} glows brightly.", thing); + } else { + println!( + "The {:?} fizzes, \ + then turns into a worthless trinket.", + thing + ); + *thing = Thing::Trinket {}; + } + } +} + +impl Enchanter for Dwarf { + fn competency(&self) -> f64 { + 0.5 // <2> + } +} +impl Enchanter for Elf { + fn competency(&self) -> f64 { + 0.95 // <3> + } +} +impl Enchanter for Human { + fn competency(&self) -> f64 { + 0.8 // <4> + } +} + +/// Holds the log value, stored on the heap. +#[derive(Debug)] +struct LogRecord { + pub text: String, +} + +impl LogRecord { + fn new() -> Self { + Self { + text: String::default(), + } + } +} + +trait Recordable: Debug { + fn text(&self) -> String; + + fn set_text(&mut self, _: String); +} + +impl Recordable for LogRecord { + fn set_text(&mut self, value: String) { + self.text = value; + } + + fn text(&self) -> String { + self.text.to_string() + } +} + +fn log_enchanted<'a, T: Any + Debug, U: Recordable>(value: &T, l: &'a mut U) -> &'a U { + let value_any = value as &dyn Any; + + // Try to convert our value its concrete type + match value_any.downcast_ref::() { + Some(_) => { + l.set_text("Default text within the recorder".to_string()); + + l + } + _ => l, + } +} + +fn main() { + let mut it = Thing::Sword; + + let d = Dwarf {}; + let e = Elf {}; + let h = Human {}; + + let party: Vec<&dyn Enchanter> = vec![&d, &h, &e]; // <5> + + // This is a contrived example to try and downcase from a leaked Box, to its concrete type without breaking Safety. + let d = Dwarf {}; + let mut l = LogRecord::new(); + let r: &LogRecord = log_enchanted(&d, &mut l); + assert_eq!("Default text within the recorder".to_string(), r.text()); + + let spellcaster = party.choose(&mut rand::thread_rng()).unwrap(); + + spellcaster.enchant(&mut it); +} From f9f7e54296f6ebd50ed017e80717514bfc964e17 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Wed, 4 Sep 2024 09:52:38 +0530 Subject: [PATCH 58/79] Remove unused imports --- .../pluggable-storage/src/backend.rs | 6 +- .../pluggable-storage/src/backend/s3.rs | 82 +++++++++---------- 2 files changed, 43 insertions(+), 45 deletions(-) diff --git a/examples/traits-dyn-generics/pluggable-storage/src/backend.rs b/examples/traits-dyn-generics/pluggable-storage/src/backend.rs index 7015d6b..93474b7 100644 --- a/examples/traits-dyn-generics/pluggable-storage/src/backend.rs +++ b/examples/traits-dyn-generics/pluggable-storage/src/backend.rs @@ -1,12 +1,10 @@ -use anyhow::{Context, Result}; +use anyhow::Result; use std::fmt::Debug; use std::future::Future; -use std::ops::{Deref, DerefMut}; -use std::path::PathBuf; pub(crate) mod disk; pub(crate) mod s3; pub trait Backend: Debug { - fn persist(&mut self, data: &[u8]) -> impl Future> + Send; + fn persist(&mut self, data: &[u8]) -> impl Future> + Send; } diff --git a/examples/traits-dyn-generics/pluggable-storage/src/backend/s3.rs b/examples/traits-dyn-generics/pluggable-storage/src/backend/s3.rs index 0aef031..dfca030 100644 --- a/examples/traits-dyn-generics/pluggable-storage/src/backend/s3.rs +++ b/examples/traits-dyn-generics/pluggable-storage/src/backend/s3.rs @@ -9,40 +9,40 @@ use super::Backend; #[derive(Default, Builder, Debug)] #[builder(setter(into))] pub struct S3Config { - bucket: String, - file_name: String, + bucket: String, + file_name: String, } #[allow(dead_code)] #[derive(Debug)] pub struct S3Backend { - config: S3Config, + config: S3Config, } impl S3Backend { - fn new(config: S3Config) -> Self { - Self { config } - } + fn new(config: S3Config) -> Self { + Self { config } + } - pub fn bucket(&self) -> String { - self.config.bucket.to_string() - } + pub fn bucket(&self) -> String { + self.config.bucket.to_string() + } - pub fn file_name(&self) -> String { - self.config.file_name.to_string() - } + pub fn file_name(&self) -> String { + self.config.file_name.to_string() + } } pub trait S3BackendTrait { - fn new(config: S3Config) -> Self; + fn new(config: S3Config) -> Self; } impl S3BackendTrait for S3Backend { - // NOTE: This is a provided method via the Trait `S3BackendTrait` - fn new(config: S3Config) -> Self { - // NOTE: this is an associated method on `S3Backend` - Self::new(config) - } + // NOTE: This is a provided method via the Trait `S3BackendTrait` + fn new(config: S3Config) -> Self { + // NOTE: this is an associated method on `S3Backend` + Self::new(config) + } } #[derive(Debug)] @@ -50,43 +50,43 @@ pub struct S3BackendBuilder(T); // Example on implemeting Deref for an inner type. impl Deref for S3BackendBuilder { - type Target = T; + type Target = T; - fn deref(&self) -> &Self::Target { - &self.0 - } + fn deref(&self) -> &Self::Target { + &self.0 + } } impl DerefMut for S3BackendBuilder { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } } impl S3BackendBuilder where - T: S3BackendTrait, + T: S3BackendTrait, { - pub fn new(config: S3Config) -> Self { - Self(T::new(config)) - } + pub fn new(config: S3Config) -> Self { + Self(T::new(config)) + } - pub fn build(self) -> T { - self.0 - } + pub fn build(self) -> T { + self.0 + } } impl Backend for S3Backend { - async fn persist(&mut self, data: &[u8]) -> Result<()> { - let bucket_name = &self.bucket(); - let file_name = &self.file_name(); + async fn persist(&mut self, data: &[u8]) -> Result<()> { + let bucket_name = &self.bucket(); + let file_name = &self.file_name(); - unimplemented!(); + unimplemented!(); - // if let Err(err) = _ { - // return Err(err).with_context(|| format!("Failed to persist to {:?}", self.path)); - // } + // if let Err(err) = _ { + // return Err(err).with_context(|| format!("Failed to persist to {:?}", self.path)); + // } - Ok(()) - } + // Ok(()) + } } From 51c8077ead9a1b1ee38b0a8be1b336b7f830e070 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Wed, 4 Sep 2024 20:29:07 +0530 Subject: [PATCH 59/79] FIXME: LimitReader expects only the type that it contains, it can't support being created from any possible type implementing ReaderTrait --- examples/limit-reader/Cargo.toml | 12 + .../limit-reader/src/bin/reader-example.rs | 6 + examples/limit-reader/src/lib.rs | 228 ++++++++++++++++++ 3 files changed, 246 insertions(+) create mode 100644 examples/limit-reader/Cargo.toml create mode 100644 examples/limit-reader/src/bin/reader-example.rs create mode 100644 examples/limit-reader/src/lib.rs diff --git a/examples/limit-reader/Cargo.toml b/examples/limit-reader/Cargo.toml new file mode 100644 index 0000000..13b0da0 --- /dev/null +++ b/examples/limit-reader/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "limit-reader" +version = "0.1.0" +authors = ["Michael de Silva anyhow::Result<()> { + // let buf: [u8; BUF_SIZE] = [0; BUF_SIZE]; + // let x = Object::::read("./source.txt".into(), buf).unwrap(); + + Ok(()) +} diff --git a/examples/limit-reader/src/lib.rs b/examples/limit-reader/src/lib.rs new file mode 100644 index 0000000..f2347f3 --- /dev/null +++ b/examples/limit-reader/src/lib.rs @@ -0,0 +1,228 @@ +#![allow(unused_imports, dead_code, unused_variables)] + +use anyhow::{Context, Result}; +use flate2::read::ZlibDecoder; +use flate2::write::ZlibEncoder; +use flate2::Compression; +use readable::LimitReader; +use readable::Readable; +use std::any; +use std::ffi::CStr; +use std::fmt; +use std::fmt::format; +use std::fs; +use std::fs::File; +use std::io; +use std::io::prelude::*; +use std::io::BufReader; +use std::path::Path; +use std::path::PathBuf; + +const BUF_SIZE: usize = 1024; + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Kind { + Blob, +} + +impl fmt::Display for Kind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Kind::Blob => write!(f, "blob"), + } + } +} + +pub(crate) struct Object { + pub(crate) kind: Kind, + pub(crate) expected_size: u64, + pub(crate) reader: R, +} + +pub trait ReaderTrait: std::io::Read {} +struct MyBufReader(BufReader); + +impl std::io::Read for MyBufReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.0.read(buf) + } +} + +impl ReaderTrait for MyBufReader {} + +impl Object +where + R: Readable + std::io::Read, +{ + pub(crate) fn read(source: PathBuf, mut buf: [u8; BUF_SIZE]) -> Result { + let f = std::fs::File::open(source).context("Unable to open provided path")?; + // let z = ZlibDecoder::new(f); + let z = MyBufReader(BufReader::new(f)); + + let mut reader = R::new(z); + + // NOTE old impl. Replacing this with generic type `R` + // let mut reader = LimitReader { + // reader: z, + // limit: LIMIT_READER as usize, + // }; + + let try_read = reader.perform_read(&mut buf); + match try_read { + Ok(value) => Ok(value), + Err(err) => { + let detail = err.to_string(); + return Err(err) + .context(format!("LimitReader failed to read the thing: {}", detail)); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use readable::LimitReader; + use std::fs::File; + use std::io::Write; + use tempfile::tempdir; + + #[test] + fn limit_reader_should_error() { + let dir = tempdir().unwrap(); + + let file_path = dir.path().join("test-source-data.txt"); + let mut file = File::create(&file_path).unwrap(); + writeln!(file, "Mike was here. Briefly.").unwrap(); + + let buf: [u8; BUF_SIZE] = [0; BUF_SIZE]; + if let Err(err) = Object::>::read(file_path, buf) { + assert_eq!( + "LimitReader failed to read the thing: too many bytes", + err.to_string() + ) + } + + drop(file); + dir.close().unwrap(); + } +} + +pub(crate) mod readable { + use super::*; + + pub trait Readable { + fn new(r: impl ReaderTrait) -> Self; + + fn perform_read(&mut self, buf: &mut [u8]) -> io::Result; + } + + impl Readable for LimitReader + where + R: ReaderTrait, + { + // NOTE: This is a provided method via the Trait `Readable` + fn new(r: impl ReaderTrait) -> Self { + // NOTE: this is an associated method on `LimitReader` + Self::new(r) + } + + // FIXME: + // But LimitReader expects only the type that it contains, it can't support being created from any possible type implementing ReaderTrait + // + // error[E0308]: mismatched types + // --> src/lib.rs:127:23 + // | + // 120 | impl Readable for LimitReader + // | - expected type parameter + // ... + // 125 | fn new(r: impl ReaderTrait) -> Self { + // | ---------------- found type parameter + // 126 | // NOTE: this is an associated method on `LimitReader` + // 127 | Self::new(r) + // | --------- ^ expected type parameter `R`, found type parameter `impl ReaderTrait` + // | | + // | arguments to this function are incorrect + // | + // = note: expected type parameter `R` + // found type parameter `impl ReaderTrait` + // = note: a type parameter was expected, but a different one was found; you might be missing a type parameter or trait bound + // = note: for more information, visit https://doc.rust-lang.org/book/ch10-02-traits.html#traits-as-parameters + // note: associated function defined here + // --> src/lib.rs:147:12 + // | + // 147 | fn new(r: R) -> Self { + // | ^^^ ---- + + fn perform_read(&mut self, buf: &mut [u8]) -> io::Result { + self.read(buf) + } + } + + pub(crate) struct LimitReader + where + R: ReaderTrait, + { + pub reader: R, + pub limit: usize, + } + + impl LimitReader + where + R: ReaderTrait, + { + fn new(r: R) -> Self { + const LIMIT_READER: u64 = 8_u64; + + Self { + reader: r, + limit: LIMIT_READER as usize, + } + + // FIXME: ideas? + // error[E0308]: mismatched types + // --> src/lib.rs:150:25 + // | + // 142 | impl LimitReader + // | - expected type parameter + // ... + // 146 | fn new(r: impl ReaderTrait) -> Self { + // | ---------------- found type parameter + // ... + // 150 | reader: r, + // | ^ expected type parameter `R`, found type parameter `impl ReaderTrait` + // | + // = note: expected type parameter `R` + // found type parameter `impl ReaderTrait` + // = note: a type parameter was expected, but a different one was found; you might be missing a type parameter or trait bound + // = note: for more information, visit https://doc.rust-lang.org/book/ch10-02-traits.html#traits-as-parameters + } + } + + impl Read for LimitReader + where + R: ReaderTrait, + { + fn read(&mut self, mut buf: &mut [u8]) -> io::Result { + if buf.len() > self.limit {} + buf = &mut buf[..self.limit + 1]; + let n = self.reader.read(buf)?; + if n > self.limit { + return Err(io::Error::new(io::ErrorKind::Other, "too many bytes")); + } + self.limit -= n; + Ok(n) + } + } + + impl BufRead for LimitReader + where + R: ReaderTrait, + { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + unimplemented!("LimitReader should never call `fill_buf`") + } + + fn consume(&mut self, amt: usize) {} + } +} From 5e8271cf491c83f337d5a4e40ea3fd584de8f823 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Wed, 4 Sep 2024 20:56:03 +0530 Subject: [PATCH 60/79] Apply crude fix by removing `R` type dependency on Object; needs cleanup. Add prelude --- .../limit-reader/src/bin/reader-example.rs | 10 +- examples/limit-reader/src/lib.rs | 158 ++---------------- examples/limit-reader/src/readable.rs | 76 +++++++++ 3 files changed, 101 insertions(+), 143 deletions(-) create mode 100644 examples/limit-reader/src/readable.rs diff --git a/examples/limit-reader/src/bin/reader-example.rs b/examples/limit-reader/src/bin/reader-example.rs index e793844..312a808 100644 --- a/examples/limit-reader/src/bin/reader-example.rs +++ b/examples/limit-reader/src/bin/reader-example.rs @@ -1,6 +1,12 @@ +use limit_reader::prelude::*; + +extern crate limit_reader; + +const BUF_SIZE: usize = 1024; + fn main() -> anyhow::Result<()> { - // let buf: [u8; BUF_SIZE] = [0; BUF_SIZE]; - // let x = Object::::read("./source.txt".into(), buf).unwrap(); + let buf: [u8; BUF_SIZE] = [0; BUF_SIZE]; + let _read_size = Object::read("./source.txt".into(), buf).unwrap(); Ok(()) } diff --git a/examples/limit-reader/src/lib.rs b/examples/limit-reader/src/lib.rs index f2347f3..7e89d60 100644 --- a/examples/limit-reader/src/lib.rs +++ b/examples/limit-reader/src/lib.rs @@ -5,6 +5,7 @@ use flate2::read::ZlibDecoder; use flate2::write::ZlibEncoder; use flate2::Compression; use readable::LimitReader; +use readable::MyBufReader; use readable::Readable; use std::any; use std::ffi::CStr; @@ -20,6 +21,14 @@ use std::path::PathBuf; const BUF_SIZE: usize = 1024; +pub(crate) mod readable; + +/// Re-exports +pub mod prelude { + pub use crate::Object; + pub use anyhow::{Context, Result}; +} + #[derive(Debug, PartialEq, Eq)] pub(crate) enum Kind { Blob, @@ -33,39 +42,25 @@ impl fmt::Display for Kind { } } -pub(crate) struct Object { +pub struct Object { pub(crate) kind: Kind, pub(crate) expected_size: u64, pub(crate) reader: R, } -pub trait ReaderTrait: std::io::Read {} -struct MyBufReader(BufReader); - -impl std::io::Read for MyBufReader { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - self.0.read(buf) +impl fmt::Display for Object { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "object") } } -impl ReaderTrait for MyBufReader {} - -impl Object -where - R: Readable + std::io::Read, -{ - pub(crate) fn read(source: PathBuf, mut buf: [u8; BUF_SIZE]) -> Result { +impl Object<()> { + pub fn read(source: PathBuf, mut buf: [u8; BUF_SIZE]) -> Result { let f = std::fs::File::open(source).context("Unable to open provided path")?; // let z = ZlibDecoder::new(f); let z = MyBufReader(BufReader::new(f)); - let mut reader = R::new(z); - - // NOTE old impl. Replacing this with generic type `R` - // let mut reader = LimitReader { - // reader: z, - // limit: LIMIT_READER as usize, - // }; + let mut reader = LimitReader::new(z); let try_read = reader.perform_read(&mut buf); match try_read { @@ -96,7 +91,7 @@ mod tests { writeln!(file, "Mike was here. Briefly.").unwrap(); let buf: [u8; BUF_SIZE] = [0; BUF_SIZE]; - if let Err(err) = Object::>::read(file_path, buf) { + if let Err(err) = Object::<()>::read(file_path, buf) { assert_eq!( "LimitReader failed to read the thing: too many bytes", err.to_string() @@ -107,122 +102,3 @@ mod tests { dir.close().unwrap(); } } - -pub(crate) mod readable { - use super::*; - - pub trait Readable { - fn new(r: impl ReaderTrait) -> Self; - - fn perform_read(&mut self, buf: &mut [u8]) -> io::Result; - } - - impl Readable for LimitReader - where - R: ReaderTrait, - { - // NOTE: This is a provided method via the Trait `Readable` - fn new(r: impl ReaderTrait) -> Self { - // NOTE: this is an associated method on `LimitReader` - Self::new(r) - } - - // FIXME: - // But LimitReader expects only the type that it contains, it can't support being created from any possible type implementing ReaderTrait - // - // error[E0308]: mismatched types - // --> src/lib.rs:127:23 - // | - // 120 | impl Readable for LimitReader - // | - expected type parameter - // ... - // 125 | fn new(r: impl ReaderTrait) -> Self { - // | ---------------- found type parameter - // 126 | // NOTE: this is an associated method on `LimitReader` - // 127 | Self::new(r) - // | --------- ^ expected type parameter `R`, found type parameter `impl ReaderTrait` - // | | - // | arguments to this function are incorrect - // | - // = note: expected type parameter `R` - // found type parameter `impl ReaderTrait` - // = note: a type parameter was expected, but a different one was found; you might be missing a type parameter or trait bound - // = note: for more information, visit https://doc.rust-lang.org/book/ch10-02-traits.html#traits-as-parameters - // note: associated function defined here - // --> src/lib.rs:147:12 - // | - // 147 | fn new(r: R) -> Self { - // | ^^^ ---- - - fn perform_read(&mut self, buf: &mut [u8]) -> io::Result { - self.read(buf) - } - } - - pub(crate) struct LimitReader - where - R: ReaderTrait, - { - pub reader: R, - pub limit: usize, - } - - impl LimitReader - where - R: ReaderTrait, - { - fn new(r: R) -> Self { - const LIMIT_READER: u64 = 8_u64; - - Self { - reader: r, - limit: LIMIT_READER as usize, - } - - // FIXME: ideas? - // error[E0308]: mismatched types - // --> src/lib.rs:150:25 - // | - // 142 | impl LimitReader - // | - expected type parameter - // ... - // 146 | fn new(r: impl ReaderTrait) -> Self { - // | ---------------- found type parameter - // ... - // 150 | reader: r, - // | ^ expected type parameter `R`, found type parameter `impl ReaderTrait` - // | - // = note: expected type parameter `R` - // found type parameter `impl ReaderTrait` - // = note: a type parameter was expected, but a different one was found; you might be missing a type parameter or trait bound - // = note: for more information, visit https://doc.rust-lang.org/book/ch10-02-traits.html#traits-as-parameters - } - } - - impl Read for LimitReader - where - R: ReaderTrait, - { - fn read(&mut self, mut buf: &mut [u8]) -> io::Result { - if buf.len() > self.limit {} - buf = &mut buf[..self.limit + 1]; - let n = self.reader.read(buf)?; - if n > self.limit { - return Err(io::Error::new(io::ErrorKind::Other, "too many bytes")); - } - self.limit -= n; - Ok(n) - } - } - - impl BufRead for LimitReader - where - R: ReaderTrait, - { - fn fill_buf(&mut self) -> io::Result<&[u8]> { - unimplemented!("LimitReader should never call `fill_buf`") - } - - fn consume(&mut self, amt: usize) {} - } -} diff --git a/examples/limit-reader/src/readable.rs b/examples/limit-reader/src/readable.rs new file mode 100644 index 0000000..8d6040d --- /dev/null +++ b/examples/limit-reader/src/readable.rs @@ -0,0 +1,76 @@ +use super::*; + +pub trait ReaderTrait: std::io::Read {} +pub struct MyBufReader(pub BufReader); + +impl std::io::Read for MyBufReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.0.read(buf) + } +} + +impl ReaderTrait for MyBufReader {} + +pub trait Readable { + // fn new(r: impl ReaderTrait) -> Self; + + fn perform_read(&mut self, buf: &mut [u8]) -> io::Result; +} + +impl Readable for LimitReader +where + R: ReaderTrait, +{ + fn perform_read(&mut self, buf: &mut [u8]) -> io::Result { + self.read(buf) + } +} + +pub(crate) struct LimitReader +where + R: ReaderTrait, +{ + pub reader: R, + pub limit: usize, +} + +impl LimitReader +where + R: ReaderTrait, +{ + pub fn new(r: R) -> Self { + const LIMIT_READER: u64 = 8_u64; + + Self { + reader: r, + limit: LIMIT_READER as usize, + } + } +} + +impl Read for LimitReader +where + R: ReaderTrait, +{ + fn read(&mut self, mut buf: &mut [u8]) -> io::Result { + if buf.len() > self.limit {} + buf = &mut buf[..self.limit + 1]; + let n = self.reader.read(buf)?; + if n > self.limit { + return Err(io::Error::new(io::ErrorKind::Other, "too many bytes")); + } + self.limit -= n; + Ok(n) + } +} + +impl BufRead for LimitReader +where + R: ReaderTrait, +{ + fn fill_buf(&mut self) -> io::Result<&[u8]> { + unimplemented!("LimitReader should never call `fill_buf`") + } + + fn consume(&mut self, amt: usize) {} +} From bd281312ace7ee778593b426d1e11c0f550de712 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Thu, 5 Sep 2024 13:08:35 +0530 Subject: [PATCH 61/79] Finish writing basic tests; Object needs a better name --- .../limit-reader/src/bin/reader-example.rs | 6 +- examples/limit-reader/src/lib.rs | 191 +++++++++++++++--- examples/limit-reader/src/readable.rs | 22 +- 3 files changed, 169 insertions(+), 50 deletions(-) diff --git a/examples/limit-reader/src/bin/reader-example.rs b/examples/limit-reader/src/bin/reader-example.rs index 312a808..75a9c84 100644 --- a/examples/limit-reader/src/bin/reader-example.rs +++ b/examples/limit-reader/src/bin/reader-example.rs @@ -2,11 +2,9 @@ use limit_reader::prelude::*; extern crate limit_reader; -const BUF_SIZE: usize = 1024; - fn main() -> anyhow::Result<()> { - let buf: [u8; BUF_SIZE] = [0; BUF_SIZE]; - let _read_size = Object::read("./source.txt".into(), buf).unwrap(); + let mut object = Object::new(); + let _read_size = object.read("./source.txt".into(), false)?; Ok(()) } diff --git a/examples/limit-reader/src/lib.rs b/examples/limit-reader/src/lib.rs index 7e89d60..16b6f73 100644 --- a/examples/limit-reader/src/lib.rs +++ b/examples/limit-reader/src/lib.rs @@ -1,4 +1,6 @@ +//! Exposes [Object] which is a limit reader, that protects against zip-bombs and other nefarious activities. #![allow(unused_imports, dead_code, unused_variables)] +#![warn(missing_docs)] use anyhow::{Context, Result}; use flate2::read::ZlibDecoder; @@ -19,8 +21,6 @@ use std::io::BufReader; use std::path::Path; use std::path::PathBuf; -const BUF_SIZE: usize = 1024; - pub(crate) mod readable; /// Re-exports @@ -29,40 +29,49 @@ pub mod prelude { pub use anyhow::{Context, Result}; } -#[derive(Debug, PartialEq, Eq)] -pub(crate) enum Kind { - Blob, +/// [Object] record struct which holds a buffer for the limit reader. +pub struct Object { + buf: [u8; Self::DEFAULT_BUF_SIZE], + expected_size: u64, } -impl fmt::Display for Kind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Kind::Blob => write!(f, "blob"), +// Holds a `LimitReader` with a default buffer of size `Object::DEFAULT_BUF_SIZE` +impl Object { + /// Default buffer size for the internal `LimitReader` + pub const DEFAULT_BUF_SIZE: usize = 1024; + + /// Create a new instance of [Object] + pub fn new() -> Self { + Self { + buf: [0; Self::DEFAULT_BUF_SIZE], + expected_size: (Self::DEFAULT_BUF_SIZE - 1) as u64, } } -} - -pub struct Object { - pub(crate) kind: Kind, - pub(crate) expected_size: u64, - pub(crate) reader: R, -} -impl fmt::Display for Object { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "object") + /// Increase the allowed limit on the `LimitReader` + pub fn limit(&mut self, limit: u64) { + self.expected_size = limit; } -} -impl Object<()> { - pub fn read(source: PathBuf, mut buf: [u8; BUF_SIZE]) -> Result { + /// Read from provided source file. If the source data is already Zlib compressed, optionally decode the data stream before reading it through a limit-reader. + pub fn read(&mut self, source: PathBuf, decode_zlib: bool) -> Result { let f = std::fs::File::open(source).context("Unable to open provided path")?; - // let z = ZlibDecoder::new(f); - let z = MyBufReader(BufReader::new(f)); + if decode_zlib { + let z = ZlibDecoder::new(f); + let buf_reader = MyBufReader(z); + let reader = LimitReader::new(buf_reader, self.expected_size as usize); + + self.try_read(reader) + } else { + let buf_reader = MyBufReader(BufReader::new(f)); + let reader = LimitReader::new(buf_reader, self.expected_size as usize); - let mut reader = LimitReader::new(z); + self.try_read(reader) + } + } - let try_read = reader.perform_read(&mut buf); + fn try_read(&mut self, mut reader: impl Readable) -> Result { + let try_read = reader.perform_read(&mut self.buf); match try_read { Ok(value) => Ok(value), Err(err) => { @@ -77,25 +86,143 @@ impl Object<()> { #[cfg(test)] mod tests { use super::*; + use flate2::read; use readable::LimitReader; use std::fs::File; use std::io::Write; use tempfile::tempdir; + #[test] + fn limit_reader_works() { + let dir = tempdir().unwrap(); + + let text = "Mike was here. Briefly."; + let file_path = dir.path().join("test_output.txt"); + let mut file = File::create(&file_path).unwrap(); + writeln!(file, "{}", &text).unwrap(); + + let mut object = Object::new(); + + match object.read(file_path, false) { + Ok(read_size) => { + assert!(read_size == 24); + } + Err(err) => unreachable!(), + } + + let persisted_text = String::from_utf8(object.buf[..24].to_vec()).unwrap(); + assert_eq!(persisted_text, format!("{}\n", &text).to_string()); + + drop(file); + dir.close().unwrap(); + } + #[test] fn limit_reader_should_error() { let dir = tempdir().unwrap(); - let file_path = dir.path().join("test-source-data.txt"); + let text = "Mike was here. Briefly."; + let file_path = dir.path().join("test_output.txt"); let mut file = File::create(&file_path).unwrap(); - writeln!(file, "Mike was here. Briefly.").unwrap(); + writeln!(file, "{}", &text).unwrap(); + + let mut object = Object::new(); + object.limit(8); - let buf: [u8; BUF_SIZE] = [0; BUF_SIZE]; - if let Err(err) = Object::<()>::read(file_path, buf) { - assert_eq!( + match object.read(file_path, false) { + Ok(read_size) => { + assert!(read_size == 24); + } + Err(err) => { + assert_eq!( + "LimitReader failed to read the thing: too many bytes", + err.to_string() + ); + } + } + + drop(file); + dir.close().unwrap(); + } + + #[test] + fn limit_reader_with_decode_zlib() { + let dir = tempdir().unwrap(); + + let text = "Mike was here. Briefly."; + let mut e = ZlibEncoder::new(Vec::new(), Compression::default()); + e.write_all(text.as_bytes()).unwrap(); + let compressed = e.finish().unwrap(); + + let file_path = dir.path().join("test_output.txt"); + let mut file = File::create(&file_path).unwrap(); + file.write_all(&compressed).unwrap(); + + let mut object = Object::new(); + match object.read(file_path, true) { + Ok(read_size) => { + let persisted_text = String::from_utf8(object.buf[..read_size].to_vec()).unwrap(); + assert_eq!(persisted_text, format!("{}", &text).to_string()); + } + Err(err) => unreachable!(), + }; + + drop(file); + dir.close().unwrap(); + } + + #[test] + fn limit_reader_with_decode_zlib_should_error() { + let dir = tempdir().unwrap(); + + let text = "Mike was here. Briefly."; + let mut e = ZlibEncoder::new(Vec::new(), Compression::default()); + e.write_all(text.as_bytes()).unwrap(); + let compressed = e.finish().unwrap(); + + let file_path = dir.path().join("test_output.txt"); + let mut file = File::create(&file_path).unwrap(); + file.write_all(&compressed).unwrap(); + + let mut object = Object::new(); + + // NOTE: This should error due to exceeding our limit. + object.limit(8); + + match object.read(file_path, true) { + Ok(read_size) => { + let persisted_text = String::from_utf8(object.buf[..read_size].to_vec()).unwrap(); + assert_eq!(persisted_text, format!("{}", &text).to_string()); + } + Err(err) => assert_eq!( "LimitReader failed to read the thing: too many bytes", err.to_string() - ) + ), + }; + + drop(file); + dir.close().unwrap(); + } + + #[test] + fn limit_reader_decode_zlib_error_on_corrupt_deflate_stream() { + let dir = tempdir().unwrap(); + + let text = "Mike was here. Briefly."; + let file_path = dir.path().join("test_output.txt"); + let mut file = File::create(&file_path).unwrap(); + writeln!(file, "{}", &text).unwrap(); + + let mut object = Object::new(); + + match object.read(file_path, true) { + Ok(read_size) => { + assert!(read_size == 24); + } + Err(err) => assert_eq!( + "LimitReader failed to read the thing: corrupt deflate stream", + err.to_string() + ), } drop(file); diff --git a/examples/limit-reader/src/readable.rs b/examples/limit-reader/src/readable.rs index 8d6040d..96f842c 100644 --- a/examples/limit-reader/src/readable.rs +++ b/examples/limit-reader/src/readable.rs @@ -1,19 +1,17 @@ use super::*; pub trait ReaderTrait: std::io::Read {} -pub struct MyBufReader(pub BufReader); +pub struct MyBufReader(pub Z); -impl std::io::Read for MyBufReader { +impl Read for MyBufReader { fn read(&mut self, buf: &mut [u8]) -> io::Result { self.0.read(buf) } } -impl ReaderTrait for MyBufReader {} +impl ReaderTrait for MyBufReader {} pub trait Readable { - // fn new(r: impl ReaderTrait) -> Self; - fn perform_read(&mut self, buf: &mut [u8]) -> io::Result; } @@ -30,21 +28,16 @@ pub(crate) struct LimitReader where R: ReaderTrait, { - pub reader: R, - pub limit: usize, + reader: R, + limit: usize, } impl LimitReader where R: ReaderTrait, { - pub fn new(r: R) -> Self { - const LIMIT_READER: u64 = 8_u64; - - Self { - reader: r, - limit: LIMIT_READER as usize, - } + pub fn new(r: R, limit: usize) -> Self { + Self { reader: r, limit } } } @@ -60,6 +53,7 @@ where return Err(io::Error::new(io::ErrorKind::Other, "too many bytes")); } self.limit -= n; + Ok(n) } } From 411524586c362f929d9bb4a7ff1eda0c6a8506cc Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Thu, 5 Sep 2024 13:20:17 +0530 Subject: [PATCH 62/79] Cleanup and rename Object to LimitReader --- .../limit-reader/src/bin/reader-example.rs | 4 +- examples/limit-reader/src/lib.rs | 74 +++++++++---------- examples/limit-reader/src/readable.rs | 27 +++---- 3 files changed, 48 insertions(+), 57 deletions(-) diff --git a/examples/limit-reader/src/bin/reader-example.rs b/examples/limit-reader/src/bin/reader-example.rs index 75a9c84..d08562a 100644 --- a/examples/limit-reader/src/bin/reader-example.rs +++ b/examples/limit-reader/src/bin/reader-example.rs @@ -3,8 +3,8 @@ use limit_reader::prelude::*; extern crate limit_reader; fn main() -> anyhow::Result<()> { - let mut object = Object::new(); - let _read_size = object.read("./source.txt".into(), false)?; + let mut limit_reader = LimitReader::new(); + let _read_size = limit_reader.read("./source.txt".into(), false)?; Ok(()) } diff --git a/examples/limit-reader/src/lib.rs b/examples/limit-reader/src/lib.rs index 16b6f73..d72d645 100644 --- a/examples/limit-reader/src/lib.rs +++ b/examples/limit-reader/src/lib.rs @@ -1,46 +1,38 @@ -//! Exposes [Object] which is a limit reader, that protects against zip-bombs and other nefarious activities. -#![allow(unused_imports, dead_code, unused_variables)] +//! Exposes [LimitReader] which is a limit reader, that protects against zip-bombs and other nefarious activities. #![warn(missing_docs)] use anyhow::{Context, Result}; use flate2::read::ZlibDecoder; -use flate2::write::ZlibEncoder; -use flate2::Compression; -use readable::LimitReader; +use readable::LimitReaderPrivate; use readable::MyBufReader; use readable::Readable; -use std::any; -use std::ffi::CStr; -use std::fmt; -use std::fmt::format; -use std::fs; -use std::fs::File; use std::io; use std::io::prelude::*; use std::io::BufReader; -use std::path::Path; use std::path::PathBuf; pub(crate) mod readable; /// Re-exports pub mod prelude { - pub use crate::Object; + pub use crate::LimitReader; pub use anyhow::{Context, Result}; } -/// [Object] record struct which holds a buffer for the limit reader. -pub struct Object { +/// The [LimitReader] reads into `buf` which is held within the record struct. +/// +/// * `expected_size`: This is the default limit placed on the [LimitReader] +pub struct LimitReader { buf: [u8; Self::DEFAULT_BUF_SIZE], expected_size: u64, } -// Holds a `LimitReader` with a default buffer of size `Object::DEFAULT_BUF_SIZE` -impl Object { +// Holds a `LimitReader` with a default buffer of size `LimitReader::DEFAULT_BUF_SIZE` +impl LimitReader { /// Default buffer size for the internal `LimitReader` pub const DEFAULT_BUF_SIZE: usize = 1024; - /// Create a new instance of [Object] + /// Create a new instance of [LimitReader] pub fn new() -> Self { Self { buf: [0; Self::DEFAULT_BUF_SIZE], @@ -59,12 +51,12 @@ impl Object { if decode_zlib { let z = ZlibDecoder::new(f); let buf_reader = MyBufReader(z); - let reader = LimitReader::new(buf_reader, self.expected_size as usize); + let reader = LimitReaderPrivate::new(buf_reader, self.expected_size as usize); self.try_read(reader) } else { let buf_reader = MyBufReader(BufReader::new(f)); - let reader = LimitReader::new(buf_reader, self.expected_size as usize); + let reader = LimitReaderPrivate::new(buf_reader, self.expected_size as usize); self.try_read(reader) } @@ -85,9 +77,9 @@ impl Object { #[cfg(test)] mod tests { - use super::*; - use flate2::read; - use readable::LimitReader; + use crate::LimitReader; + use flate2::write::ZlibEncoder; + use flate2::Compression; use std::fs::File; use std::io::Write; use tempfile::tempdir; @@ -101,16 +93,16 @@ mod tests { let mut file = File::create(&file_path).unwrap(); writeln!(file, "{}", &text).unwrap(); - let mut object = Object::new(); + let mut limit_reader = LimitReader::new(); - match object.read(file_path, false) { + match limit_reader.read(file_path, false) { Ok(read_size) => { assert!(read_size == 24); } - Err(err) => unreachable!(), + Err(_) => unreachable!(), } - let persisted_text = String::from_utf8(object.buf[..24].to_vec()).unwrap(); + let persisted_text = String::from_utf8(limit_reader.buf[..24].to_vec()).unwrap(); assert_eq!(persisted_text, format!("{}\n", &text).to_string()); drop(file); @@ -126,10 +118,10 @@ mod tests { let mut file = File::create(&file_path).unwrap(); writeln!(file, "{}", &text).unwrap(); - let mut object = Object::new(); - object.limit(8); + let mut limit_reader = LimitReader::new(); + limit_reader.limit(8); - match object.read(file_path, false) { + match limit_reader.read(file_path, false) { Ok(read_size) => { assert!(read_size == 24); } @@ -158,13 +150,14 @@ mod tests { let mut file = File::create(&file_path).unwrap(); file.write_all(&compressed).unwrap(); - let mut object = Object::new(); - match object.read(file_path, true) { + let mut limit_reader = LimitReader::new(); + match limit_reader.read(file_path, true) { Ok(read_size) => { - let persisted_text = String::from_utf8(object.buf[..read_size].to_vec()).unwrap(); + let persisted_text = + String::from_utf8(limit_reader.buf[..read_size].to_vec()).unwrap(); assert_eq!(persisted_text, format!("{}", &text).to_string()); } - Err(err) => unreachable!(), + Err(_) => unreachable!(), }; drop(file); @@ -184,14 +177,15 @@ mod tests { let mut file = File::create(&file_path).unwrap(); file.write_all(&compressed).unwrap(); - let mut object = Object::new(); + let mut limit_reader = LimitReader::new(); // NOTE: This should error due to exceeding our limit. - object.limit(8); + limit_reader.limit(8); - match object.read(file_path, true) { + match limit_reader.read(file_path, true) { Ok(read_size) => { - let persisted_text = String::from_utf8(object.buf[..read_size].to_vec()).unwrap(); + let persisted_text = + String::from_utf8(limit_reader.buf[..read_size].to_vec()).unwrap(); assert_eq!(persisted_text, format!("{}", &text).to_string()); } Err(err) => assert_eq!( @@ -213,9 +207,9 @@ mod tests { let mut file = File::create(&file_path).unwrap(); writeln!(file, "{}", &text).unwrap(); - let mut object = Object::new(); + let mut limit_reader = LimitReader::new(); - match object.read(file_path, true) { + match limit_reader.read(file_path, true) { Ok(read_size) => { assert!(read_size == 24); } diff --git a/examples/limit-reader/src/readable.rs b/examples/limit-reader/src/readable.rs index 96f842c..0cfc5a4 100644 --- a/examples/limit-reader/src/readable.rs +++ b/examples/limit-reader/src/readable.rs @@ -1,6 +1,5 @@ use super::*; -pub trait ReaderTrait: std::io::Read {} pub struct MyBufReader(pub Z); impl Read for MyBufReader { @@ -9,41 +8,39 @@ impl Read for MyBufReader { } } -impl ReaderTrait for MyBufReader {} - pub trait Readable { fn perform_read(&mut self, buf: &mut [u8]) -> io::Result; } -impl Readable for LimitReader +impl Readable for LimitReaderPrivate where - R: ReaderTrait, + R: Read, { fn perform_read(&mut self, buf: &mut [u8]) -> io::Result { self.read(buf) } } -pub(crate) struct LimitReader +pub(crate) struct LimitReaderPrivate where - R: ReaderTrait, + R: Read, { reader: R, limit: usize, } -impl LimitReader +impl LimitReaderPrivate where - R: ReaderTrait, + R: Read, { pub fn new(r: R, limit: usize) -> Self { Self { reader: r, limit } } } -impl Read for LimitReader +impl Read for LimitReaderPrivate where - R: ReaderTrait, + R: Read, { fn read(&mut self, mut buf: &mut [u8]) -> io::Result { if buf.len() > self.limit {} @@ -58,13 +55,13 @@ where } } -impl BufRead for LimitReader +impl BufRead for LimitReaderPrivate where - R: ReaderTrait, + R: Read, { fn fill_buf(&mut self) -> io::Result<&[u8]> { - unimplemented!("LimitReader should never call `fill_buf`") + unimplemented!("LimitReaderPrivate should never call `fill_buf`") } - fn consume(&mut self, amt: usize) {} + fn consume(&mut self, _: usize) {} } From a1567b2b763f0e7fed8e05dcfd040397ee04310a Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Fri, 25 Oct 2024 20:11:13 +0530 Subject: [PATCH 63/79] Add example on trait objects and `Sized` --- Cargo.toml | 4 - examples/limit-reader/Cargo.toml | 12 - examples/limit-reader/README.md | 3 + .../limit-reader/src/bin/reader-example.rs | 10 - examples/limit-reader/src/lib.rs | 225 ------------------ examples/limit-reader/src/readable.rs | 67 ------ .../pluggable-storage/src/backend/disk.rs | 81 +++---- rust-toolchain.toml | 2 +- src/bin/maybeuninit.rs | 56 +++++ src/bin/trait-objects-1.rs | 63 +++++ src/traits.rs | 2 +- 11 files changed, 165 insertions(+), 360 deletions(-) delete mode 100644 examples/limit-reader/Cargo.toml create mode 100644 examples/limit-reader/README.md delete mode 100644 examples/limit-reader/src/bin/reader-example.rs delete mode 100644 examples/limit-reader/src/lib.rs delete mode 100644 examples/limit-reader/src/readable.rs create mode 100644 src/bin/maybeuninit.rs create mode 100644 src/bin/trait-objects-1.rs diff --git a/Cargo.toml b/Cargo.toml index 37dc3f7..52acca0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,10 +9,6 @@ publish = false name = "tokio-panic-handling" path = "src/bin/tokio-panic-handling.rs" -[[bin]] -name = "string-search" -path = "src/bin/string-search.rs" - [[bin]] name = "string-search-heaptrack" path = "src/bin/string-search-heaptrack.rs" diff --git a/examples/limit-reader/Cargo.toml b/examples/limit-reader/Cargo.toml deleted file mode 100644 index 13b0da0..0000000 --- a/examples/limit-reader/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[package] -name = "limit-reader" -version = "0.1.0" -authors = ["Michael de Silva anyhow::Result<()> { - let mut limit_reader = LimitReader::new(); - let _read_size = limit_reader.read("./source.txt".into(), false)?; - - Ok(()) -} diff --git a/examples/limit-reader/src/lib.rs b/examples/limit-reader/src/lib.rs deleted file mode 100644 index d72d645..0000000 --- a/examples/limit-reader/src/lib.rs +++ /dev/null @@ -1,225 +0,0 @@ -//! Exposes [LimitReader] which is a limit reader, that protects against zip-bombs and other nefarious activities. -#![warn(missing_docs)] - -use anyhow::{Context, Result}; -use flate2::read::ZlibDecoder; -use readable::LimitReaderPrivate; -use readable::MyBufReader; -use readable::Readable; -use std::io; -use std::io::prelude::*; -use std::io::BufReader; -use std::path::PathBuf; - -pub(crate) mod readable; - -/// Re-exports -pub mod prelude { - pub use crate::LimitReader; - pub use anyhow::{Context, Result}; -} - -/// The [LimitReader] reads into `buf` which is held within the record struct. -/// -/// * `expected_size`: This is the default limit placed on the [LimitReader] -pub struct LimitReader { - buf: [u8; Self::DEFAULT_BUF_SIZE], - expected_size: u64, -} - -// Holds a `LimitReader` with a default buffer of size `LimitReader::DEFAULT_BUF_SIZE` -impl LimitReader { - /// Default buffer size for the internal `LimitReader` - pub const DEFAULT_BUF_SIZE: usize = 1024; - - /// Create a new instance of [LimitReader] - pub fn new() -> Self { - Self { - buf: [0; Self::DEFAULT_BUF_SIZE], - expected_size: (Self::DEFAULT_BUF_SIZE - 1) as u64, - } - } - - /// Increase the allowed limit on the `LimitReader` - pub fn limit(&mut self, limit: u64) { - self.expected_size = limit; - } - - /// Read from provided source file. If the source data is already Zlib compressed, optionally decode the data stream before reading it through a limit-reader. - pub fn read(&mut self, source: PathBuf, decode_zlib: bool) -> Result { - let f = std::fs::File::open(source).context("Unable to open provided path")?; - if decode_zlib { - let z = ZlibDecoder::new(f); - let buf_reader = MyBufReader(z); - let reader = LimitReaderPrivate::new(buf_reader, self.expected_size as usize); - - self.try_read(reader) - } else { - let buf_reader = MyBufReader(BufReader::new(f)); - let reader = LimitReaderPrivate::new(buf_reader, self.expected_size as usize); - - self.try_read(reader) - } - } - - fn try_read(&mut self, mut reader: impl Readable) -> Result { - let try_read = reader.perform_read(&mut self.buf); - match try_read { - Ok(value) => Ok(value), - Err(err) => { - let detail = err.to_string(); - return Err(err) - .context(format!("LimitReader failed to read the thing: {}", detail)); - } - } - } -} - -#[cfg(test)] -mod tests { - use crate::LimitReader; - use flate2::write::ZlibEncoder; - use flate2::Compression; - use std::fs::File; - use std::io::Write; - use tempfile::tempdir; - - #[test] - fn limit_reader_works() { - let dir = tempdir().unwrap(); - - let text = "Mike was here. Briefly."; - let file_path = dir.path().join("test_output.txt"); - let mut file = File::create(&file_path).unwrap(); - writeln!(file, "{}", &text).unwrap(); - - let mut limit_reader = LimitReader::new(); - - match limit_reader.read(file_path, false) { - Ok(read_size) => { - assert!(read_size == 24); - } - Err(_) => unreachable!(), - } - - let persisted_text = String::from_utf8(limit_reader.buf[..24].to_vec()).unwrap(); - assert_eq!(persisted_text, format!("{}\n", &text).to_string()); - - drop(file); - dir.close().unwrap(); - } - - #[test] - fn limit_reader_should_error() { - let dir = tempdir().unwrap(); - - let text = "Mike was here. Briefly."; - let file_path = dir.path().join("test_output.txt"); - let mut file = File::create(&file_path).unwrap(); - writeln!(file, "{}", &text).unwrap(); - - let mut limit_reader = LimitReader::new(); - limit_reader.limit(8); - - match limit_reader.read(file_path, false) { - Ok(read_size) => { - assert!(read_size == 24); - } - Err(err) => { - assert_eq!( - "LimitReader failed to read the thing: too many bytes", - err.to_string() - ); - } - } - - drop(file); - dir.close().unwrap(); - } - - #[test] - fn limit_reader_with_decode_zlib() { - let dir = tempdir().unwrap(); - - let text = "Mike was here. Briefly."; - let mut e = ZlibEncoder::new(Vec::new(), Compression::default()); - e.write_all(text.as_bytes()).unwrap(); - let compressed = e.finish().unwrap(); - - let file_path = dir.path().join("test_output.txt"); - let mut file = File::create(&file_path).unwrap(); - file.write_all(&compressed).unwrap(); - - let mut limit_reader = LimitReader::new(); - match limit_reader.read(file_path, true) { - Ok(read_size) => { - let persisted_text = - String::from_utf8(limit_reader.buf[..read_size].to_vec()).unwrap(); - assert_eq!(persisted_text, format!("{}", &text).to_string()); - } - Err(_) => unreachable!(), - }; - - drop(file); - dir.close().unwrap(); - } - - #[test] - fn limit_reader_with_decode_zlib_should_error() { - let dir = tempdir().unwrap(); - - let text = "Mike was here. Briefly."; - let mut e = ZlibEncoder::new(Vec::new(), Compression::default()); - e.write_all(text.as_bytes()).unwrap(); - let compressed = e.finish().unwrap(); - - let file_path = dir.path().join("test_output.txt"); - let mut file = File::create(&file_path).unwrap(); - file.write_all(&compressed).unwrap(); - - let mut limit_reader = LimitReader::new(); - - // NOTE: This should error due to exceeding our limit. - limit_reader.limit(8); - - match limit_reader.read(file_path, true) { - Ok(read_size) => { - let persisted_text = - String::from_utf8(limit_reader.buf[..read_size].to_vec()).unwrap(); - assert_eq!(persisted_text, format!("{}", &text).to_string()); - } - Err(err) => assert_eq!( - "LimitReader failed to read the thing: too many bytes", - err.to_string() - ), - }; - - drop(file); - dir.close().unwrap(); - } - - #[test] - fn limit_reader_decode_zlib_error_on_corrupt_deflate_stream() { - let dir = tempdir().unwrap(); - - let text = "Mike was here. Briefly."; - let file_path = dir.path().join("test_output.txt"); - let mut file = File::create(&file_path).unwrap(); - writeln!(file, "{}", &text).unwrap(); - - let mut limit_reader = LimitReader::new(); - - match limit_reader.read(file_path, true) { - Ok(read_size) => { - assert!(read_size == 24); - } - Err(err) => assert_eq!( - "LimitReader failed to read the thing: corrupt deflate stream", - err.to_string() - ), - } - - drop(file); - dir.close().unwrap(); - } -} diff --git a/examples/limit-reader/src/readable.rs b/examples/limit-reader/src/readable.rs deleted file mode 100644 index 0cfc5a4..0000000 --- a/examples/limit-reader/src/readable.rs +++ /dev/null @@ -1,67 +0,0 @@ -use super::*; - -pub struct MyBufReader(pub Z); - -impl Read for MyBufReader { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - self.0.read(buf) - } -} - -pub trait Readable { - fn perform_read(&mut self, buf: &mut [u8]) -> io::Result; -} - -impl Readable for LimitReaderPrivate -where - R: Read, -{ - fn perform_read(&mut self, buf: &mut [u8]) -> io::Result { - self.read(buf) - } -} - -pub(crate) struct LimitReaderPrivate -where - R: Read, -{ - reader: R, - limit: usize, -} - -impl LimitReaderPrivate -where - R: Read, -{ - pub fn new(r: R, limit: usize) -> Self { - Self { reader: r, limit } - } -} - -impl Read for LimitReaderPrivate -where - R: Read, -{ - fn read(&mut self, mut buf: &mut [u8]) -> io::Result { - if buf.len() > self.limit {} - buf = &mut buf[..self.limit + 1]; - let n = self.reader.read(buf)?; - if n > self.limit { - return Err(io::Error::new(io::ErrorKind::Other, "too many bytes")); - } - self.limit -= n; - - Ok(n) - } -} - -impl BufRead for LimitReaderPrivate -where - R: Read, -{ - fn fill_buf(&mut self) -> io::Result<&[u8]> { - unimplemented!("LimitReaderPrivate should never call `fill_buf`") - } - - fn consume(&mut self, _: usize) {} -} diff --git a/examples/traits-dyn-generics/pluggable-storage/src/backend/disk.rs b/examples/traits-dyn-generics/pluggable-storage/src/backend/disk.rs index 702f8ee..9464327 100644 --- a/examples/traits-dyn-generics/pluggable-storage/src/backend/disk.rs +++ b/examples/traits-dyn-generics/pluggable-storage/src/backend/disk.rs @@ -7,79 +7,80 @@ use super::Backend; #[derive(Debug)] pub struct FileBackend { - path: PathBuf, + path: PathBuf, } impl FileBackend { - fn new(s: &str) -> Self { - Self { path: s.into() } - } + fn new(s: &str) -> Self { + Self { path: s.into() } + } } pub trait FileBackendTrait { - fn new(s: &str) -> Self; + fn new(s: &str) -> Self; } impl FileBackendTrait for FileBackend { - // NOTE: This is a provided method via the Trait `FileBackendTrait` - fn new(s: &str) -> Self { - // NOTE: this is an associated method on `FileBackend` - Self::new(s) - } + // NOTE: This is a provided method via the Trait `FileBackendTrait` + fn new(s: &str) -> Self { + // NOTE: this is an associated method on `FileBackend` + Self::new(s) + } } #[derive(Debug)] pub struct FileBackendBuilder(T); -// Example on implemeting Deref for an inner type. +// Deref on inner type. impl Deref for FileBackendBuilder { - type Target = T; + type Target = T; - fn deref(&self) -> &Self::Target { - &self.0 - } + fn deref(&self) -> &Self::Target { + &self.0 + } } impl DerefMut for FileBackendBuilder { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } } impl FileBackendBuilder where - T: FileBackendTrait, + T: FileBackendTrait, { - pub fn new(path: &str) -> Self { - Self(T::new(path)) - } + pub fn new(path: &str) -> Self { + Self(T::new(path)) + } - pub fn build(self) -> T { - self.0 - } + pub fn build(self) -> T { + self.0 + } } impl Backend for FileBackend { - async fn persist(&mut self, data: &[u8]) -> Result<()> { - if let Err(err) = tokio::fs::write(&self.path, data).await { - return Err(err).with_context(|| format!("Failed to persist to {:?}", self.path)); - } + async fn persist(&mut self, data: &[u8]) -> Result<()> { + if let Err(err) = tokio::fs::write(&self.path, data).await { + return Err(err).with_context(|| format!("Failed to persist to {:?}", self.path)); + } - Ok(()) - } + Ok(()) + } } #[cfg(test)] mod tests { - use super::*; + use super::*; - #[should_panic] - #[tokio::test] - async fn panic_when_writing_to_missing_directory() { - let data = String::from("If my calculations are correct, when this baby hits eighty-eight miles per hour... you're gonna see some serious s**t."); - let backend_builder: FileBackendBuilder = FileBackendBuilder::new("./missing/output.json"); - let backend = &mut backend_builder.build(); + #[should_panic] + #[tokio::test] + async fn panic_when_writing_to_missing_directory() { + let data = String::from("If my calculations are correct, when this baby hits eighty-eight miles per hour... you're gonna see some serious s**t."); + let backend_builder: FileBackendBuilder = + FileBackendBuilder::new("./missing/output.json"); + let backend = &mut backend_builder.build(); - backend.persist(data.as_bytes()).await.unwrap(); - } + backend.persist(data.as_bytes()).await.unwrap(); + } } diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 66eee23..271800c 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "beta" \ No newline at end of file +channel = "nightly" \ No newline at end of file diff --git a/src/bin/maybeuninit.rs b/src/bin/maybeuninit.rs new file mode 100644 index 0000000..1466566 --- /dev/null +++ b/src/bin/maybeuninit.rs @@ -0,0 +1,56 @@ +#![feature(maybe_uninit_write_slice)] +#![feature(maybe_uninit_slice)] + +use anyhow::Error; + +extern crate tutorials; + +pub fn main() -> Result<(), Error> { + Ok(()) +} + +pub(crate) mod mem { + use std::fmt; + use std::mem::MaybeUninit; + use std::str; + + /// Used for slow path in `Display` implementations when alignment is required. + pub struct DisplayBuffer { + buf: [MaybeUninit; SIZE], + len: usize, + } + + impl DisplayBuffer { + #[inline] + pub const fn new() -> Self { + Self { + buf: [MaybeUninit::uninit(); SIZE], + len: 0, + } + } + + #[inline] + pub fn as_str(&self) -> &str { + // SAFETY: `buf` is only written to by the `fmt::Write::write_str` implementation + // which writes a valid UTF-8 string to `buf` and correctly sets `len`. + unsafe { + let s = MaybeUninit::slice_assume_init_ref(&self.buf[..self.len]); + str::from_utf8_unchecked(s) + } + } + } + + impl fmt::Write for DisplayBuffer { + fn write_str(&mut self, s: &str) -> fmt::Result { + let bytes = s.as_bytes(); + + if let Some(buf) = self.buf.get_mut(self.len..(self.len + bytes.len())) { + MaybeUninit::copy_from_slice(buf, bytes); + self.len += bytes.len(); + Ok(()) + } else { + Err(fmt::Error) + } + } + } +} diff --git a/src/bin/trait-objects-1.rs b/src/bin/trait-objects-1.rs new file mode 100644 index 0000000..e0138b6 --- /dev/null +++ b/src/bin/trait-objects-1.rs @@ -0,0 +1,63 @@ +#![allow(unused_variables)] +#![allow(dead_code)] + +use anyhow::Error; + +extern crate tutorials; + +fn invoke_trait_object(x: &mut dyn Foo) { + x.dont_need_sized(); +} + +trait Foo { + fn has_generic(&self, gen: A); + fn dont_need_sized(&self); + // fn need_sized(self) -> Self; + // where + // Self: Sized; +} + +trait Clonable: Clone { + fn is_clonable(&self) -> bool; +} + +// Blanket impl on any T that impls the provided traits. +impl Clonable for T +where + T: std::fmt::Display + std::clone::Clone, +{ + fn is_clonable(&self) -> bool { + true + } +} + +struct S; + +impl Foo for S +where + A: Clonable + std::fmt::Display, +{ + fn has_generic(&self, gen: A) { + let a = gen.is_clonable(); + let x = gen.clone(); + let x_ptr = &x as *const A; + let x_points_at = unsafe { (*x_ptr).clone() }; + + println!("this is x: {}", x_points_at); + } + + fn dont_need_sized(&self) {} + + // fn need_sized(self) -> Self { + // Self {} + // } +} + +pub fn main() -> Result<(), Error> { + let mut s = S {}; + let x = invoke_trait_object::(&mut s); + + s.has_generic("foo".to_string()); + + Ok(()) +} diff --git a/src/traits.rs b/src/traits.rs index f10fe25..affddfa 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -99,7 +99,7 @@ pub fn runner() -> Result<()> { lesson_1_add_trait_bound_to_parameter(); lesson_2(); lesson_3::run()?; - // lesson_4::run();``````````````````````````````````` + // lesson_4::run(); // lesson_5::run(); // lesson6::run(); From cd4c4560d083684c0ab972c6affeb7c45bafbfb7 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Fri, 25 Oct 2024 21:03:50 +0530 Subject: [PATCH 64/79] Add example to demonstrate need for use of `?Sized` bound --- Cargo.toml | 6 ++- src/bin/sized-1.rs | 108 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 src/bin/sized-1.rs diff --git a/Cargo.toml b/Cargo.toml index 52acca0..72c454f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,8 +53,12 @@ tower-http = { version = "0.4.0", features = ["trace", "cors", "catch-panic"] } tower-layer = "0.3.1" tower-service = "0.3.1" +# Logging support +tracing = "0.1.30" +tracing-subscriber = { version = "0.3.8", features = ["env-filter"] } + [dev-dependencies] test-log = "0.2.10" [profile.release] -debug = true \ No newline at end of file +debug = true diff --git a/src/bin/sized-1.rs b/src/bin/sized-1.rs new file mode 100644 index 0000000..21d3eae --- /dev/null +++ b/src/bin/sized-1.rs @@ -0,0 +1,108 @@ +#![allow(unused_variables)] +#![allow(dead_code)] + +use std::fmt::Debug; + +use anyhow::Error; + +extern crate tutorials; + +pub trait Storable: Debug { + fn can_return_string(&self) -> String { + let caller_type = std::any::type_name::(); + + format!("Hello, this text is on the heap! {}", caller_type) + } + + fn need_sized(self) -> Self + where + Self: Sized, + { + self + } +} + +#[derive(Debug)] +struct Holder<'a>(&'a dyn Storable); + +#[derive(Debug)] +struct Carton {} + +#[derive(Debug)] +struct Crate {} + +impl Storable for Carton {} +impl Storable for Crate {} + +// NOTE: This example demonstrates that we can call a method on a trait object, but notice that we +// have to use the `?Sized` bound on the type parameter `S` in the function signature. +// +// 38 | pub fn do_something(s: &S) -> &S +// | ^ required by the implicit `Sized` requirement on this type parameter in `do_something` +// help: consider relaxing the implicit `Sized` restriction +// | +// 38 | pub fn do_something(s: &S) -> &S +// | ++++++++ +pub fn do_something(s: &S) -> &S +where + S: ?Sized, +{ + let caller_type = std::any::type_name::(); + tracing::info!("This is {:?}", caller_type); + + s +} + +pub fn main() -> Result<(), Error> { + // Set the RUST_LOG, if it hasn't been explicitly defined + if std::env::var_os("RUST_LOG").is_none() { + std::env::set_var( + "RUST_LOG", + "tutorials=info,tower_http=trace,tokio=trace,runtime=trace", + ) + } + tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .with_file(true) + .with_line_number(true) + .init(); + + let item = &Carton {}; + let holder1 = Holder(item); + tracing::info!("holder1: {:?}", holder1); + tracing::info!( + "holder1: can_return_string(): {:?}", + holder1.0.can_return_string() + ); + let item = Crate {}; + let holder2 = Holder(&item); + tracing::info!("holder2: {:?}", holder2); + tracing::info!( + "holder2: can_return_string(): {:?}", + holder2.0.can_return_string() + ); + + tracing::info!( + "need_sized() can be called directly on {:?}", + &item.need_sized(), + ); + + // NOTE: However, we cannot call this on a trait object due to the Sized requirement + // + // dbg!(holder2.0.need_sized()); + // error: the `need_sized` method cannot be invoked on a trait object + // --> src/bin/sized-1.rs:71:20 + // | + // 19 | Self: Sized, + // | ----- this has a `Sized` requirement + // ... + // 71 | dbg!(holder2.0.need_sized()); + // | ^^^^^^^^^^ + + let item = Crate {}; + let holder = Holder(&item); + let s = do_something(holder.0); + dbg!(s); + + Ok(()) +} From 5425af93c6228fb3ec6126bf2956f68b48f6637c Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Fri, 25 Oct 2024 21:25:08 +0530 Subject: [PATCH 65/79] Add example for dereferencing a ptr using unsafe --- src/bin/pointers-1.rs | 52 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 src/bin/pointers-1.rs diff --git a/src/bin/pointers-1.rs b/src/bin/pointers-1.rs new file mode 100644 index 0000000..a7d5b21 --- /dev/null +++ b/src/bin/pointers-1.rs @@ -0,0 +1,52 @@ +#![allow(unused_variables)] +#![allow(dead_code)] + +use std::fmt::Debug; + +use anyhow::Error; + +extern crate tutorials; + +pub fn main() -> Result<(), Error> { + // Set the RUST_LOG, if it hasn't been explicitly defined + if std::env::var_os("RUST_LOG").is_none() { + std::env::set_var( + "RUST_LOG", + "tutorials=info,tower_http=trace,tokio=trace,runtime=trace", + ) + } + tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .with_file(true) + .with_line_number(true) + .init(); + + let data = Box::new(0); + let data_ptr = data.as_ref() as *const i32; + tracing::info!( + "Address of the Boxed object on the heap represented by data_ptr: {:p} / {:?}", + data_ptr, + data + ); + + let box_deref = unsafe { *data_ptr }; + tracing::info!( + "We can defreference the pointer to the Boxed object on the heap: {:?}", + box_deref + ); + + // Example with a String object + let str_point_memory = String::from("Text on the Heap"); + let str_ref = &str_point_memory; + tracing::info!( + "Address of the object on the heap represented by str_point_memory: {:p}", + str_point_memory.as_ptr(), + ); + tracing::info!( + "Address of str_point_memory on the stack {:p} / {:?}", + str_ref, + str_ref + ); + + Ok(()) +} From 63105c8cbefc0f10601c8860e732b0dae95342b0 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Thu, 21 Nov 2024 12:42:19 +0530 Subject: [PATCH 66/79] Add example using pinning --- src/bin/pin-weird-1.rs | 67 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 src/bin/pin-weird-1.rs diff --git a/src/bin/pin-weird-1.rs b/src/bin/pin-weird-1.rs new file mode 100644 index 0000000..c48857d --- /dev/null +++ b/src/bin/pin-weird-1.rs @@ -0,0 +1,67 @@ +#![allow(unused_variables)] +#![allow(dead_code)] + +use anyhow::Error; +use std::fmt::Debug; +use std::pin::Pin; +use std::ptr; + +extern crate tutorials; + +// NOTE: Warning: avoid the pin! macro unless you have a very good understanding of what it does. This macro is sometimes over-eagerly suggested by the compiler as the “solution” to pinning-related errors (as you may already have noticed in the compiler error listed in the previous chapter). However, this macro is only applicable under specific conditions and often not the right tool for the job. +// https://sander.saares.eu/2024/11/06/why-is-stdpinpin-so-weird/ + +pub struct BagOfApples { + count: usize, + + // In the example code, this self-reference is mostly useless. + // This is just to keep the example code simple – the emphasis is + // on the effects of pinning, not why a type may be designed to need it. + self_reference: *mut BagOfApples, + + _require_pin: std::marker::PhantomPinned, +} + +impl BagOfApples { + pub fn new() -> Self { + BagOfApples { + count: 0, + // We cannot set this here because we have not yet + // created the BagOfApples – there is nothing to reference. + self_reference: ptr::null_mut(), + _require_pin: std::marker::PhantomPinned, + } + } + + /// Call this after creating a BagOfApples to initialize the instance. + pub fn initialize(mut self: Pin<&mut Self>) { + // SAFETY: BagOfApples requires pinning and we do not allow + // the obtained pointer to be exposed outside this type, so + // we know it always points to a valid value and therefore it + // is safe to store the pointer. We have also reviewed the code of + // this function to ensure that we do not move the BagOfApples + // instance via the reference we obtain from here nor via the pointer. + let self_mut = unsafe { self.as_mut().get_unchecked_mut() }; + + self_mut.self_reference = self_mut; + } + + pub fn count(&self) -> usize { + assert!( + !self.self_reference.is_null(), + "BagOfApples is not initialized" + ); + + // SAFETY: Simple read-only access to the count field, which + // is safe. We do it via the pointer for example purposes. + unsafe { (*self.self_reference).count } + } +} + +pub fn main() -> Result<(), Error> { + let mut bag = Box::pin(BagOfApples::new()); + bag.as_mut().initialize(); + println!("Apple count: {}", bag.count()); + + Ok(()) +} From 5f800ce123687a02457d9b7dfa3a6c1486de7654 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Thu, 21 Nov 2024 12:43:48 +0530 Subject: [PATCH 67/79] Add previous example on using Box --- src/bin/str-1.rs | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 src/bin/str-1.rs diff --git a/src/bin/str-1.rs b/src/bin/str-1.rs new file mode 100644 index 0000000..fe57ad1 --- /dev/null +++ b/src/bin/str-1.rs @@ -0,0 +1,42 @@ +#![allow(unused_variables)] +#![allow(dead_code)] + +use std::fmt::Debug; + +use anyhow::Error; + +extern crate tutorials; + +pub fn main() -> Result<(), Error> { + // Set the RUST_LOG, if it hasn't been explicitly defined + if std::env::var_os("RUST_LOG").is_none() { + std::env::set_var( + "RUST_LOG", + "tutorials=info,tower_http=trace,tokio=trace,runtime=trace", + ) + } + tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .with_file(true) + .with_line_number(true) + .init(); + + let mut box_string: Box = String::from("hello").into_boxed_str(); + box_string.make_ascii_uppercase(); + tracing::info!("{:?}", box_string); + + let mut vec = Vec::with_capacity(10); + vec.extend([1, 2, 3]); + + assert!(vec.capacity() >= 10); + + // NOTE: This erases the capacity of the vector + let slice = vec.into_boxed_slice(); + + let x = slice.iter().map(|el| *el).collect::>(); + dbg!(x); + + assert_eq!(slice.into_vec().capacity(), 3); + + Ok(()) +} From db62a9b266630e4db8ae213ea14ffc3e80d42874 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Fri, 22 Nov 2024 13:23:49 +0530 Subject: [PATCH 68/79] Add more examples on pinning --- Cargo.toml | 1 + src/bin/pin-2.rs | 53 ++++++++++++++++++++++++++ src/bin/pin-nested-future.rs | 73 ++++++++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+) create mode 100644 src/bin/pin-2.rs create mode 100644 src/bin/pin-nested-future.rs diff --git a/Cargo.toml b/Cargo.toml index 72c454f..bb3fab9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,6 +43,7 @@ aes-gcm-siv = "^0.11" aes = "^0.8" heapless = "^0.8" rand = "0.8.5" +pin-project = "1.1.7" axum = { version = "0.6.18", features = ["headers", "tower-log"] } axum-extra = "0.1.2" diff --git a/src/bin/pin-2.rs b/src/bin/pin-2.rs new file mode 100644 index 0000000..1693efa --- /dev/null +++ b/src/bin/pin-2.rs @@ -0,0 +1,53 @@ +#![allow(unused_variables)] +#![allow(dead_code)] + +use anyhow::Error; + +#[derive(Debug)] +struct Test { + a: String, + b: *const String, +} + +impl Test { + fn new(txt: &str) -> Self { + Test { + a: String::from(txt), + b: std::ptr::null(), + } + } + + fn init(&mut self) { + let self_ref: *const String = &self.a; + self.b = self_ref; + } + + fn a(&self) -> &str { + &self.a + } + + fn b(&self) -> &String { + assert!( + !self.b.is_null(), + "Test::b called without Test::init being called first" + ); + unsafe { &*(self.b) } + } +} + +// https://rust-lang.github.io/async-book/04_pinning/01_chapter.html#pinning-in-detail +// Demonstrates the issue of self-referential types without the use of pinning leading to UB. +pub fn main() -> Result<(), Error> { + let mut test1 = Test::new("test1"); + test1.init(); + let mut test2 = Test::new("test2"); + test2.init(); + + println!("a: {}, b: {}", test1.a(), test1.b()); + std::mem::swap(&mut test1, &mut test2); + test1.a = "I've totally changed now!".to_string(); + + println!("a: {}, b: {}", test2.a(), test2.b()); + + Ok(()) +} diff --git a/src/bin/pin-nested-future.rs b/src/bin/pin-nested-future.rs new file mode 100644 index 0000000..121c128 --- /dev/null +++ b/src/bin/pin-nested-future.rs @@ -0,0 +1,73 @@ +#![allow(unused_variables)] +#![allow(dead_code)] + +use anyhow::Error; +use pin_project; + +use std::{ + future::Future, + pin::Pin, + task::{Context, Poll}, + time::{Duration, Instant}, +}; + +#[pin_project::pin_project] +pub struct TimedWrapper { + start: Option, + #[pin] + future: Fut, +} + +impl TimedWrapper { + pub fn new(future: Fut) -> Self { + Self { + future, + start: None, + } + } +} + +impl Future for TimedWrapper { + type Output = (Fut::Output, Duration); + + fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { + let mut this = self.project(); + // Call the inner poll, measuring how long it took. + let start = this.start.get_or_insert_with(Instant::now); + let inner_poll = this.future.as_mut().poll(cx); + let elapsed = start.elapsed(); + + match inner_poll { + // The inner future needs more time, so this future needs more time too + Poll::Pending => Poll::Pending, + // Success! + Poll::Ready(output) => Poll::Ready((output, elapsed)), + } + } +} + +// Article: https://blog.cloudflare.com/pin-and-unpin-in-rust/ +// Github: https://github.com/adamchalmers/nested-future-example/blob/master/src/main.rs +#[tokio::main] +async fn main() { + let (resp, time) = TimedWrapper::new(fake_reqwest()).await; + println!( + "Got a HTTP {} in {}ms", + resp.unwrap().status(), + time.as_millis() + ) +} + +// Stubbing out use of `reqwest` with a dummy, just to get this to compile. +struct FakeRequest {} + +impl FakeRequest { + fn status(&self) -> String { + String::from("") + } +} +async fn fake_reqwest() -> Result { + let f = FakeRequest {}; + + Ok(f) +} From 12b078570018b879c58a7ddbee7b5448ee3e0834 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Fri, 22 Nov 2024 19:50:28 +0530 Subject: [PATCH 69/79] Add example on using `pin_project` --- src/bin/async-1.rs | 34 +++++++++++++ src/bin/pin-async-to-heap.rs | 47 ++++++++++++++++++ src/bin/pin-project.rs | 92 ++++++++++++++++++++++++++++++++++++ 3 files changed, 173 insertions(+) create mode 100644 src/bin/async-1.rs create mode 100644 src/bin/pin-async-to-heap.rs create mode 100644 src/bin/pin-project.rs diff --git a/src/bin/async-1.rs b/src/bin/async-1.rs new file mode 100644 index 0000000..8d31abe --- /dev/null +++ b/src/bin/async-1.rs @@ -0,0 +1,34 @@ +#![allow(unused_variables)] +#![allow(dead_code)] + +use anyhow::Error; +use std::pin::Pin; + +async fn blocks(item: Item) -> Result<(), Error> { + let my_string = "foo".to_string(); + + let future_one = async { + // ... + println!("{my_string}"); + }; + + let future_two = async { + // ... + println!("{my_string}"); + }; + + // Run both futures to completion, printing "foo" twice: + let ((), ()) = futures::join!(future_one, future_two); + + Ok(()) +} + +struct Item {} + +#[tokio::main] +async fn main() { + let item = Item {}; + // let pinned = Box::pin(item); + + let _ = blocks(item).await; +} diff --git a/src/bin/pin-async-to-heap.rs b/src/bin/pin-async-to-heap.rs new file mode 100644 index 0000000..f1abf20 --- /dev/null +++ b/src/bin/pin-async-to-heap.rs @@ -0,0 +1,47 @@ +#![allow(unused_variables)] +#![allow(dead_code)] + +use anyhow::Error; +use pin_project; +use std::marker::PhantomPinned; +use std::pin::Pin; +use std::ptr; + +#[derive(Debug)] +struct Test { + a: String, + b: *const String, + _marker: PhantomPinned, +} + +impl Test { + fn new(txt: &str) -> Pin> { + let t = Test { + a: String::from(txt), + b: std::ptr::null(), + _marker: PhantomPinned, + }; + let mut boxed = Box::pin(t); + let self_ptr: *const String = &boxed.a; + unsafe { boxed.as_mut().get_unchecked_mut().b = self_ptr }; + + boxed + } + + fn a(self: Pin<&Self>) -> &str { + &self.get_ref().a + } + + fn b(self: Pin<&Self>) -> &String { + unsafe { &*(self.b) } + } +} + +#[tokio::main] +pub async fn main() { + let test1 = Test::new("test1"); + let test2 = Test::new("test2"); + + println!("a: {}, b: {}", test1.as_ref().a(), test1.as_ref().b()); + println!("a: {}, b: {}", test2.as_ref().a(), test2.as_ref().b()); +} diff --git a/src/bin/pin-project.rs b/src/bin/pin-project.rs new file mode 100644 index 0000000..56a4873 --- /dev/null +++ b/src/bin/pin-project.rs @@ -0,0 +1,92 @@ +#![allow(unused_variables)] +#![allow(dead_code)] + +use anyhow::Error; +use pin_project::pin_project; +use std::marker::PhantomPinned; +use std::pin::Pin; +use std::ptr; + +#[derive(Default)] +struct BagOfApples { + _require_pin: std::marker::PhantomPinned, +} + +impl BagOfApples { + fn sell_one(self: Pin<&mut Self>) { + println!("Sold an apple!"); + } +} + +#[derive(Default)] +struct BagOfOranges { + _require_pin: std::marker::PhantomPinned, +} + +impl BagOfOranges { + fn sell_one(self: Pin<&mut Self>) { + println!("Sold an orange!"); + } +} + +#[derive(Default)] +struct BagOfBananas { + _require_pin: std::marker::PhantomPinned, +} + +impl BagOfBananas { + fn sell_one(self: Pin<&mut Self>) { + println!("Sold a banana!"); + } +} + +// ### Projected variant. +#[pin_project] +struct FruitStand { + #[pin] + apples: BagOfApples, + #[pin] + oranges: BagOfOranges, + #[pin] + bananas: BagOfBananas, + + total_sold: usize, +} + +impl FruitStand { + fn sell_one_of_each(mut self: Pin<&mut Self>) { + let self_projected = self.as_mut().project(); + + self_projected.apples.sell_one(); + self_projected.oranges.sell_one(); + self_projected.bananas.sell_one(); + *self_projected.total_sold += 3; + } + + fn new() -> Pin> { + let this = Self { + apples: BagOfApples { + _require_pin: std::marker::PhantomPinned, + }, + oranges: BagOfOranges { + _require_pin: std::marker::PhantomPinned, + }, + bananas: BagOfBananas { + _require_pin: std::marker::PhantomPinned, + }, + total_sold: 0, + }; + + Box::pin(this) + } +} + +// `pin_project` on struct fields +// Article: https://sander.saares.eu/2024/11/06/why-is-stdpinpin-so-weird/ +// Github: https://github.com/sandersaares/pins-in-rust/blob/main/examples/05_project.rs +#[tokio::main] +pub async fn main() { + let money_jar = &mut FruitStand::new(); + + money_jar.as_mut().sell_one_of_each(); +} From 9aac87f26832cdcacc8a2de53defdd97ff78f6f4 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sun, 1 Dec 2024 12:38:51 +0530 Subject: [PATCH 70/79] Formatting --- .../src/main.rs | 126 +++++++++--------- 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/examples/traits-dyn-generics/trait-objects-without-downcast-generics/src/main.rs b/examples/traits-dyn-generics/trait-objects-without-downcast-generics/src/main.rs index a467fc8..774d5ef 100644 --- a/examples/traits-dyn-generics/trait-objects-without-downcast-generics/src/main.rs +++ b/examples/traits-dyn-generics/trait-objects-without-downcast-generics/src/main.rs @@ -11,9 +11,9 @@ use std::ops::{Deref, DerefMut}; struct Dwarf {} impl Display for Dwarf { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "I may be small but the ladies don't complain!") - } + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "I may be small but the ladies don't complain!") + } } #[derive(Debug)] @@ -24,107 +24,107 @@ struct Human {} #[derive(Debug)] enum Thing { - Sword, - Trinket, + Sword, + Trinket, } trait Enchanter: std::fmt::Debug { - fn competency(&self) -> f64; - - fn enchant(&self, thing: &mut Thing) { - let probability_of_success = self.competency(); - let spell_is_successful = rand::thread_rng().gen_bool(probability_of_success); // <1> - - print!("{:?} mutters incoherently. ", self); - if spell_is_successful { - println!("The {:?} glows brightly.", thing); - } else { - println!( - "The {:?} fizzes, \ + fn competency(&self) -> f64; + + fn enchant(&self, thing: &mut Thing) { + let probability_of_success = self.competency(); + let spell_is_successful = rand::thread_rng().gen_bool(probability_of_success); // <1> + + print!("{:?} mutters incoherently. ", self); + if spell_is_successful { + println!("The {:?} glows brightly.", thing); + } else { + println!( + "The {:?} fizzes, \ then turns into a worthless trinket.", - thing - ); - *thing = Thing::Trinket {}; + thing + ); + *thing = Thing::Trinket {}; + } } - } } impl Enchanter for Dwarf { - fn competency(&self) -> f64 { - 0.5 // <2> - } + fn competency(&self) -> f64 { + 0.5 // <2> + } } impl Enchanter for Elf { - fn competency(&self) -> f64 { - 0.95 // <3> - } + fn competency(&self) -> f64 { + 0.95 // <3> + } } impl Enchanter for Human { - fn competency(&self) -> f64 { - 0.8 // <4> - } + fn competency(&self) -> f64 { + 0.8 // <4> + } } /// Holds the log value, stored on the heap. #[derive(Debug)] struct LogRecord { - pub text: String, + pub text: String, } impl LogRecord { - fn new() -> Self { - Self { - text: String::default(), + fn new() -> Self { + Self { + text: String::default(), + } } - } } trait Recordable: Debug { - fn text(&self) -> String; + fn text(&self) -> String; - fn set_text(&mut self, _: String); + fn set_text(&mut self, _: String); } impl Recordable for LogRecord { - fn set_text(&mut self, value: String) { - self.text = value; - } + fn set_text(&mut self, value: String) { + self.text = value; + } - fn text(&self) -> String { - self.text.to_string() - } + fn text(&self) -> String { + self.text.to_string() + } } fn log_enchanted<'a, T: Any + Debug, U: Recordable>(value: &T, l: &'a mut U) -> &'a U { - let value_any = value as &dyn Any; + let value_any = value as &dyn Any; - // Try to convert our value its concrete type - match value_any.downcast_ref::() { - Some(_) => { - l.set_text("Default text within the recorder".to_string()); + // Try to convert our value its concrete type + match value_any.downcast_ref::() { + Some(_) => { + l.set_text("Default text within the recorder".to_string()); - l + l + } + _ => l, } - _ => l, - } } fn main() { - let mut it = Thing::Sword; + let mut it = Thing::Sword; - let d = Dwarf {}; - let e = Elf {}; - let h = Human {}; + let d = Dwarf {}; + let e = Elf {}; + let h = Human {}; - let party: Vec<&dyn Enchanter> = vec![&d, &h, &e]; // <5> + let party: Vec<&dyn Enchanter> = vec![&d, &h, &e]; // <5> - // This is a contrived example to try and downcase from a leaked Box, to its concrete type without breaking Safety. - let d = Dwarf {}; - let mut l = LogRecord::new(); - let r: &LogRecord = log_enchanted(&d, &mut l); - assert_eq!("Default text within the recorder".to_string(), r.text()); + // This is a contrived example to try and downcase from a leaked Box, to its concrete type without breaking Safety. + let d = Dwarf {}; + let mut l = LogRecord::new(); + let r: &LogRecord = log_enchanted(&d, &mut l); + assert_eq!("Default text within the recorder".to_string(), r.text()); - let spellcaster = party.choose(&mut rand::thread_rng()).unwrap(); + let spellcaster = party.choose(&mut rand::thread_rng()).unwrap(); - spellcaster.enchant(&mut it); + spellcaster.enchant(&mut it); } From 61e32d31986e55cec65c35242160cbab9ee05e6e Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Mon, 2 Dec 2024 13:57:39 +0530 Subject: [PATCH 71/79] Add example on using Newtype to adapt to trait bounds on generics --- Cargo.toml | 6 ++++ src/bin/newtype-axum-1.rs | 59 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 src/bin/newtype-axum-1.rs diff --git a/Cargo.toml b/Cargo.toml index bb3fab9..6e6b693 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,10 @@ heapless = "^0.8" rand = "0.8.5" pin-project = "1.1.7" +# Misc +crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] } +kanal = "0.1.0-pre8" + axum = { version = "0.6.18", features = ["headers", "tower-log"] } axum-extra = "0.1.2" @@ -53,10 +57,12 @@ tower = { version = "0.4.13", features = ["limit", "load-shed", "filter", "util" tower-http = { version = "0.4.0", features = ["trace", "cors", "catch-panic"] } tower-layer = "0.3.1" tower-service = "0.3.1" +tower-http_0_2_4 = { package = "tower-http", version = "0.2.4" , features = ["cors"] } # Logging support tracing = "0.1.30" tracing-subscriber = { version = "0.3.8", features = ["env-filter"] } +nalgebra = "0.33.2" [dev-dependencies] test-log = "0.2.10" diff --git a/src/bin/newtype-axum-1.rs b/src/bin/newtype-axum-1.rs new file mode 100644 index 0000000..acc41f3 --- /dev/null +++ b/src/bin/newtype-axum-1.rs @@ -0,0 +1,59 @@ +#![allow(unused_variables)] +#![allow(dead_code)] + +use anyhow::Error; +use hyper::header::HeaderValue; +use std::fmt; +use std::ops; +use tower_http_0_2_4::cors::{CorsLayer, Origin}; + +struct CorsOrigins<'a>(pub(crate) &'a Vec); + +impl IntoIterator for CorsOrigins<'_> { + type Item = HeaderValue; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + let mut collection: Vec = vec![]; + let _result: Vec = self + .0 + .iter() + .map(|x| { + collection.push(HeaderValue::from_str(x).unwrap()); + x.to_string() + }) + .collect(); + + collection.into_iter() + } +} + +impl<'a> fmt::Display for CorsOrigins<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.iter().fold(Ok(()), |result, origin| { + result.and_then(|_| writeln!(f, "{}", origin)) + }) + } +} + +impl ops::Deref for CorsOrigins<'_> { + type Target = Vec; + + fn deref(&self) -> &Self::Target { + self.0 + } +} + +#[tokio::main] +async fn main() { + let production_origins = vec!["https://example.com".parse().unwrap()]; + + tracing::info!( + "[ OK ]: CORS access enabled! {}", + CorsOrigins(&production_origins) + ); + + // In this example we create a Newtype to implement `IntoIterator`. This is because Axum's `Origin::list()` function takes a generic parameter with a trait bound of `IntoIterator`. + // We therefore implement the `IntoIterator` trait on our Newtype so that it can be passed as an argument to the `Origin::list()` function. + CorsLayer::new().allow_origin(Origin::list(CorsOrigins(&production_origins))); +} From fd2d4e36153d1612ff30c92692411ec29a4d7c30 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sun, 15 Dec 2024 22:05:36 +0530 Subject: [PATCH 72/79] Add unfinished examples --- Cargo.toml | 1 + src/bin/thread-1.rs | 25 +++++ src/bin/tokio-spawn-1.rs | 8 ++ src/bin/vector-1.rs | 212 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 246 insertions(+) create mode 100644 src/bin/thread-1.rs create mode 100644 src/bin/tokio-spawn-1.rs create mode 100644 src/bin/vector-1.rs diff --git a/Cargo.toml b/Cargo.toml index 6e6b693..c3d0117 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,7 @@ heapless = "^0.8" rand = "0.8.5" pin-project = "1.1.7" + # Misc crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] } kanal = "0.1.0-pre8" diff --git a/src/bin/thread-1.rs b/src/bin/thread-1.rs new file mode 100644 index 0000000..c7692f7 --- /dev/null +++ b/src/bin/thread-1.rs @@ -0,0 +1,25 @@ +#![allow(unused_variables)] +#![allow(dead_code)] + +use anyhow::Error; + +const NUM_CPUS: u8 = 8; + +pub fn run_sim() { + let mut handles = Vec::with_capacity(NUM_CPUS as usize); + + let (tx, rx) = kanal::bounded::<()>(NUM_CPUS as usize); + + (0..NUM_CPUS - 1).for_each(|_| { + let handle = std::thread::spawn(move || loop { + + // + }); + + handles.push(handle); + }) +} + +pub fn main() -> Result<(), Error> { + Ok(()) +} diff --git a/src/bin/tokio-spawn-1.rs b/src/bin/tokio-spawn-1.rs new file mode 100644 index 0000000..f645212 --- /dev/null +++ b/src/bin/tokio-spawn-1.rs @@ -0,0 +1,8 @@ +#![allow(unused_variables)] +#![allow(dead_code)] + +use anyhow::Error; + +pub fn main() -> Result<(), Error> { + Ok(()) +} diff --git a/src/bin/vector-1.rs b/src/bin/vector-1.rs new file mode 100644 index 0000000..d29219a --- /dev/null +++ b/src/bin/vector-1.rs @@ -0,0 +1,212 @@ +#![allow(unused_variables)] +#![allow(dead_code)] + +use anyhow::Error; +use nalgebra::distance_squared; +use nalgebra::Point3; +use nalgebra::SimdComplexField; +use nalgebra::Vector3; +use rand::prelude::*; +use rand::Rng; + +pub trait NewtonianMechanics { + fn get_mass(&self) -> T; + + fn get_position(&self) -> Point3; + fn get_velocity(&self) -> Vector3; + fn set_position(&mut self, position: Point3); + fn set_velocity(&mut self, direction: Vector3); + + fn compute_force_vec(&self, other: &Self) -> Option>; +} + +#[derive(Clone, Debug)] +pub struct PointMass { + position: Point3, + velocity: Vector3, + mass: T, +} + +impl PointMass { + pub fn new(position: Point3, velocity: Vector3, mass: T) -> Self { + Self { + position, + velocity, + mass, + } + } +} + +const G: f32 = 6.67430e-5; + +macro_rules! impl_mechanics_for_pointmass { + ($t:ty) => { + impl NewtonianMechanics<$t> for PointMass<$t> { + fn get_mass(&self) -> $t { + self.mass + } + + fn get_position(&self) -> Point3<$t> { + self.position + } + fn get_velocity(&self) -> Vector3<$t> { + self.velocity + } + fn set_position(&mut self, position: Point3<$t>) { + self.position = position; + } + fn set_velocity(&mut self, velocity: Vector3<$t>) { + self.velocity = velocity; + } + + fn compute_force_vec(&self, other: &Self) -> Option> { + if self.get_position() == other.get_position() { + return None; + } + let dist2 = distance_squared(&self.get_position(), &other.get_position()); + + let force = <$t>::from(G) * self.get_mass() * other.get_mass() / dist2; + + let direction = other.get_position() - self.get_position(); + dbg!(direction); + + Some((direction * (force)).into()) + } + } + }; +} + +impl_mechanics_for_pointmass!(f32); +impl_mechanics_for_pointmass!(f64); + +pub type PointMassCollection = Vec>; + +#[derive(Clone)] +pub struct Population { + items: PointMassCollection, +} + +impl Population { + pub fn new(points: usize) -> Self { + Self { + items: Vec::with_capacity(points), + } + } + + pub fn add(&mut self, point_mass: PointMass) { + self.items.push(point_mass); + } + + pub fn get(&self) -> &PointMassCollection { + &self.items + } + + pub fn get_mut(&mut self) -> &mut PointMassCollection { + &mut self.items + } + + pub fn compute_next_positions(pop1: &mut Population, pop2: &mut Population, ns_per_frame: u64) { + let mut p1_items = pop1.get_mut(); + let p2_items = pop2.get_mut(); + + compute_next_positions(&mut p1_items, &p2_items, ns_per_frame); + } +} + +pub fn compute_next_positions( + next_set: &mut [impl NewtonianMechanics], + last_set: &[impl NewtonianMechanics], + ns_per_frame: u64, +) where + T: SimdComplexField, +{ + let time_const = ns_per_frame as f64 / 1e9; + + for (i, point) in last_set.iter().enumerate() { + let mut force = Vector3::new(T::zero(), T::zero(), T::zero()); + + for other in last_set.iter() { + if let Some(f) = point.compute_force_vec(other) { + force += f; + } else { + continue; + } + } + let acceleration: Vector3 = force / point.get_mass(); + let old_velocity: Vector3 = point.get_velocity(); + + let new_velocity = old_velocity.clone() + + (acceleration * T::from_simd_real(nalgebra::convert(time_const))); + + let new_position = point.get_position() + (old_velocity + new_velocity.clone()); + + next_set[i].set_velocity(new_velocity); + next_set[i].set_position(new_position); + } +} + +pub fn run(num_points: usize, max_init_speed: f32, max_mass: f32, spawn_radius: f32) { + let mut population = Population::new(num_points); + + let mut rng = rand::rngs::StdRng::from_entropy(); + + population.add(PointMass::new( + nalgebra::Point3::new( + rng.gen_range(-spawn_radius..spawn_radius), + rng.gen_range(-spawn_radius..spawn_radius), + rng.gen_range(-spawn_radius..spawn_radius), + ), + nalgebra::Vector3::new( + rng.gen_range(-max_init_speed..max_init_speed), + rng.gen_range(-max_init_speed..max_init_speed), + rng.gen_range(-max_init_speed..max_init_speed), + ), + rng.gen_range(1.0..max_mass), + )); + let mass = rand::random::() * max_mass; + + dbg!(population.get()); +} + +const NUM_ENTITIES: usize = 1; +const MAX_INIT_SPEED: f32 = 1000.0; +const MAX_MASS: f32 = 100000000.0; +const SPAWN_RADIUS: f32 = 10000.0; + +pub fn main() -> Result<(), Error> { + run(NUM_ENTITIES, MAX_INIT_SPEED, MAX_MASS, SPAWN_RADIUS); + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_compute_force_vec_pre_computed() { + let point1 = PointMass::new(Point3::new(0.0, 0.0, 0.0), Vector3::new(0.0, 0.0, 0.0), 1e4); + let point2 = PointMass::new(Point3::new(2.0, 3.0, 6.0), Vector3::new(0.0, 0.0, 0.0), 1e4); + + let force = point1.compute_force_vec(&point2).unwrap(); + assert_eq!( + force, + Vector3::new(272.4204060374474, 408.63060905617107, 817.2612181123421) + ); + } + + // FIXME: verify the formula here. + #[test] + fn test_compute_force_vec_unverified() { + let point1 = PointMass::new(Point3::new(0.0, 0.0, 0.0), Vector3::new(0.0, 0.0, 0.0), 1e4); + let point2 = PointMass::new(Point3::new(2.0, 3.0, 6.0), Vector3::new(0.0, 0.0, 0.0), 1e4); + + let force = point1.compute_force_vec(&point2).unwrap(); + assert!( + force + - Vector3::new(0.2857142857142857, 0.42857142857142855, 0.8571428571428571) * 1e8 + / 49.0 + < Vector3::new(1e-6, 1e-6, 1e-6) + ); + } +} From 8184f83596b96ef73412a3bcdd66e2a9e5975fbb Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sun, 22 Dec 2024 13:05:34 +0530 Subject: [PATCH 73/79] Add example on parsing with nom --- Cargo.toml | 1 + src/bin/nom-parse-hex-1.rs | 113 +++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 src/bin/nom-parse-hex-1.rs diff --git a/Cargo.toml b/Cargo.toml index c3d0117..7561150 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,6 +64,7 @@ tower-http_0_2_4 = { package = "tower-http", version = "0.2.4" , features = ["co tracing = "0.1.30" tracing-subscriber = { version = "0.3.8", features = ["env-filter"] } nalgebra = "0.33.2" +nom = "7.1.3" [dev-dependencies] test-log = "0.2.10" diff --git a/src/bin/nom-parse-hex-1.rs b/src/bin/nom-parse-hex-1.rs new file mode 100644 index 0000000..c178bc4 --- /dev/null +++ b/src/bin/nom-parse-hex-1.rs @@ -0,0 +1,113 @@ +#![allow(unused_variables)] +#![allow(dead_code)] + +use anyhow::Error; + +pub fn main() -> Result<(), Error> { + Ok(()) +} + +#[cfg(test)] +mod tests_3 { + use nom::{ + bytes::complete::{tag, take_while_m_n}, + combinator::map_res, + error::{context, convert_error}, + sequence::Tuple, + IResult, Parser, + }; + + pub fn parse_hex_seg(input: &str) -> IResult<&str, u8, nom::error::VerboseError<&str>> { + let parse_two_hex_digits_parser_fn = take_while_m_n( + // RA block + 2, + 2, + |it: char| it.is_ascii_hexdigit(), + ); + + let mut map_res_fn = map_res( + // RA block + parse_two_hex_digits_parser_fn, + |it| u8::from_str_radix(it, 16), + ); + + map_res_fn.parse(input) + } + + /// `nom` is used to parse the hex digits from string. Then [u8::from_str_radix] is + /// used to convert the hex string into a number. This can't fail, even though in the + /// function signature, that may return a [core::num::ParseIntError], which never + /// happens. Note the use of [nom::error::VerboseError] to get more detailed error + /// messages that are passed to [nom::error::convert_error]. + /// + /// Even if [core::num::ParseIntError] were to be thrown, it would be consumed, and + /// a higher level `nom` error would be returned for the `map_res` combinator. + pub fn parse_hex_seg_combined( + input: &str, + ) -> IResult<&str, u8, nom::error::VerboseError<&str>> { + map_res( + take_while_m_n(2, 2, |it: char| it.is_ascii_hexdigit()), + |it| u8::from_str_radix(it, 16), + ) + .parse(input) + } + + /// Note the use of [nom::error::VerboseError] to get more detailed error messages + /// that are passed to [nom::error::convert_error]. + pub fn root(input: &str) -> IResult<&str, (u8, u8, u8), nom::error::VerboseError<&str>> { + let (remainder, (_, red, green, blue)) = ( + context("start of hex color", tag("#")), + context("hex seg 1", parse_hex_seg), + context("hex seg 2", parse_hex_seg), + context("hex seg 3", parse_hex_seg), + ) + .parse(input)?; + + Ok((remainder, (red, green, blue))) + } + + #[test] + fn test_root_1() { + let input = "x#FF0000"; + let result = root(input); + println!("{:?}", result); + assert!(result.is_err()); + + match result { + Err(nom::Err::Error(e)) | Err(nom::Err::Failure(e)) => { + println!("Could not parse because ... {}", convert_error(input, e)); + } + _ => { /* do nothing for nom::Err::Incomplete(_) */ } + } + } + + #[test] + fn test_root_2() { + let input = "#FF_0000"; + let result = root(input); + println!("{:?}", result); + assert!(result.is_err()); + + match result { + Err(nom::Err::Error(e)) | Err(nom::Err::Failure(e)) => { + println!("Could not parse because ... {}", convert_error(input, e)); + } + _ => { /* do nothing for nom::Err::Incomplete(_) */ } + } + } + + #[test] + fn test_root_3() { + let input = "#FF00AA"; + let result = root(input); + println!("{:?}", result); + assert!(result.is_ok()); + + match result { + Err(nom::Err::Error(e)) | Err(nom::Err::Failure(e)) => { + println!("Could not parse because ... {}", convert_error(input, e)); + } + _ => { /* do nothing for nom::Err::Incomplete(_) */ } + } + } +} From 1fcb8ea4573b2367cad1378ea06da68fe6e550e2 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sun, 22 Dec 2024 13:39:04 +0530 Subject: [PATCH 74/79] Add super simple example on type-erasure of a writable backend --- Cargo.toml | 1 + src/bin/common-backend.rs | 90 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 src/bin/common-backend.rs diff --git a/Cargo.toml b/Cargo.toml index 7561150..8351be9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,6 +65,7 @@ tracing = "0.1.30" tracing-subscriber = { version = "0.3.8", features = ["env-filter"] } nalgebra = "0.33.2" nom = "7.1.3" +async-trait = "0.1.83" [dev-dependencies] test-log = "0.2.10" diff --git a/src/bin/common-backend.rs b/src/bin/common-backend.rs new file mode 100644 index 0000000..3081d2e --- /dev/null +++ b/src/bin/common-backend.rs @@ -0,0 +1,90 @@ +#![allow(unused_variables)] +#![allow(dead_code)] + +use anyhow::Error; +use async_trait::async_trait; +use std::fmt::Debug; +use std::io::Write; +use tokio::io::AsyncWriteExt; + +#[async_trait] +pub trait PersistantBackend: Debug { + async fn insert(&mut self, data: Vec) -> Result<(), Error>; +} + +pub mod backend { + use super::*; + use tokio::fs::File; + + #[derive(Debug)] + pub struct FileStore {} + + #[async_trait] + impl PersistantBackend for FileStore { + async fn insert(&mut self, data: Vec) -> Result<(), Error> { + let mut file = File::create("data.txt").await?; + let text = data + .iter() + .map(|b| b.to_string()) + .collect::>() + .join(","); + + file.write_all(text.as_bytes()).await?; + Ok(()) + } + } +} + +#[derive(Debug)] +pub struct PersistableBackend { + backend: Box, +} + +#[tokio::main] +async fn main() { + todo!() +} + +pub async fn insert_data(backend: &mut PersistableBackend, data: Vec) -> Result<(), Error> { + backend.backend.insert(data).await +} + +#[cfg(test)] +mod tests { + use super::*; + use nom::AsBytes; + use tokio::fs::File; + use tokio::io::AsyncReadExt; + + #[tokio::test] + async fn test_insert_data() { + let file_store = backend::FileStore {}; + let mut backend = PersistableBackend { + backend: Box::new(file_store), + }; + + let data = vec![1, 2, 3]; + insert_data(&mut backend, data.clone()) + .await + .expect("Failed to insert data"); + + let mut file = File::open("data.txt").await.expect("Failed to open file"); + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer) + .await + .expect("Failed to read file"); + + let text = data + .iter() + .map(|b| b.to_string()) + .collect::>() + .join(","); + + assert_eq!(text.as_bytes(), buffer); + + // Cleanup + tokio::fs::remove_file("data.txt") + .await + .expect("Failed to remove file"); + } +} From be96392beb34aaf235e4610a4ca40b75d5b1e3dd Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Wed, 19 Mar 2025 14:03:01 +0530 Subject: [PATCH 75/79] chore: add example with retry strategy using backon --- Cargo.toml | 10 ++-- src/bin/async-fut-reqwest.rs | 106 +++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 4 deletions(-) create mode 100644 src/bin/async-fut-reqwest.rs diff --git a/Cargo.toml b/Cargo.toml index 8351be9..794ed01 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,7 +44,10 @@ aes = "^0.8" heapless = "^0.8" rand = "0.8.5" pin-project = "1.1.7" - +nalgebra = "0.33.2" +nom = "7.1.3" +async-trait = "0.1.83" +reqwest = "0.12.15" # Misc crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] } @@ -63,9 +66,8 @@ tower-http_0_2_4 = { package = "tower-http", version = "0.2.4" , features = ["co # Logging support tracing = "0.1.30" tracing-subscriber = { version = "0.3.8", features = ["env-filter"] } -nalgebra = "0.33.2" -nom = "7.1.3" -async-trait = "0.1.83" +backon = "1.4.0" +bytes = "1.10.1" [dev-dependencies] test-log = "0.2.10" diff --git a/src/bin/async-fut-reqwest.rs b/src/bin/async-fut-reqwest.rs new file mode 100644 index 0000000..59d3322 --- /dev/null +++ b/src/bin/async-fut-reqwest.rs @@ -0,0 +1,106 @@ +use async_trait::async_trait; +use backon::{ExponentialBuilder, Retryable}; +use bytes::Bytes; +use helpers::with_retry; +use reqwest::{Error, Response}; +use retry_strategy::RetryStrategy; +use std::future::Future; + +// Newtype wrapper for reqwest::Client (async) +#[derive(Default)] +pub struct ReqwestClient(reqwest::Client); + +impl ReqwestClient { + pub fn new() -> Self { + Self(reqwest::Client::new()) + } + + pub fn client(&self) -> &reqwest::Client { + &self.0 + } + + pub fn client_mut(&mut self) -> &mut reqwest::Client { + &mut self.0 + } +} + +// Trait for the HTTP client +#[async_trait] +pub trait HttpClient { + async fn post(&self, url: &str, api_key: &str, body: &[u8]) -> Result; +} + +// Implement HttpClient for the newtype with exponential backoff +#[async_trait] +#[allow(clippy::redundant_closure)] +impl HttpClient for ReqwestClient { + // Alternatively, we could also pass `body: Bytes` and insice the async closure use `body.clone()`, as this is what reqwest expects. + async fn post(&self, url: &str, api_key: &str, body: &[u8]) -> Result { + let retry_strategy = RetryStrategy::default().get_config(); + + let operation = async || { + let body = Bytes::copy_from_slice(body); + + self.client() + .post(url) + .header("Authorization", format!("Bearer {}", api_key)) + .header("Content-Type", "application/json") + .body(body) + .send() + .await + }; + + // Perform retry operation + with_retry(operation, retry_strategy).await + } +} + +mod helpers { + use super::*; + + /// Perform the retry operation with the given closure and retry strategy + pub async fn with_retry( + closure: F, + retry_strategy: ExponentialBuilder, + ) -> Result + where + F: Fn() -> Fut, + Fut: Future>, + { + let resp: Result = closure.retry(retry_strategy).await; + + // TODO: add error handling + + resp + } +} + +mod retry_strategy { + use backon::ExponentialBuilder; + + const MAX_DELAY_MILLIS: u64 = 1000; + pub struct RetryStrategy(ExponentialBuilder); + + impl Default for RetryStrategy { + /// Default backoff strategy. This uses defaults with jitter. + fn default() -> Self { + let retry_strategy = ExponentialBuilder::default() + .with_factor(2_f32) + .with_max_times(3) + .with_max_delay(std::time::Duration::from_millis(MAX_DELAY_MILLIS)) + .with_jitter(); + + Self(retry_strategy) + } + } + + impl RetryStrategy { + /// Get the backoff configuration. + pub fn get_config(self) -> ExponentialBuilder { + self.0 + } + } +} + +#[tokio::main] +async fn main() {} From 6f992d23b4a3656ecc9c7816af8ab8e19dfa597c Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Thu, 20 Mar 2025 01:27:55 +0530 Subject: [PATCH 76/79] fix: typo --- src/bin/async-fut-reqwest.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/async-fut-reqwest.rs b/src/bin/async-fut-reqwest.rs index 59d3322..b85767a 100644 --- a/src/bin/async-fut-reqwest.rs +++ b/src/bin/async-fut-reqwest.rs @@ -34,7 +34,7 @@ pub trait HttpClient { #[async_trait] #[allow(clippy::redundant_closure)] impl HttpClient for ReqwestClient { - // Alternatively, we could also pass `body: Bytes` and insice the async closure use `body.clone()`, as this is what reqwest expects. + // Alternatively, we could also pass `body: Bytes` and inside the async closure use `body.clone()`, as this is what reqwest expects. async fn post(&self, url: &str, api_key: &str, body: &[u8]) -> Result { let retry_strategy = RetryStrategy::default().get_config(); From a498f1948366e2ba6a4c9faa93668bd0f2d88fe7 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sun, 23 Mar 2025 20:35:31 +0530 Subject: [PATCH 77/79] chore: Postgres multi-repository pattern with traits --- Cargo.toml | 1 + src/bin/db-abstraction1.rs | 85 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 src/bin/db-abstraction1.rs diff --git a/Cargo.toml b/Cargo.toml index 794ed01..a691d2b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,7 @@ tracing = "0.1.30" tracing-subscriber = { version = "0.3.8", features = ["env-filter"] } backon = "1.4.0" bytes = "1.10.1" +thiserror = "2.0.12" [dev-dependencies] test-log = "0.2.10" diff --git a/src/bin/db-abstraction1.rs b/src/bin/db-abstraction1.rs new file mode 100644 index 0000000..9022454 --- /dev/null +++ b/src/bin/db-abstraction1.rs @@ -0,0 +1,85 @@ +#![allow(unused_variables)] + +use anyhow::Error; +use std::future::Future; + +#[derive(Clone)] +pub struct PostgresDatabase { + ignored_users_repository: PostgresIgnoredUsersRepository, +} + +#[derive(Clone)] +struct PostgresIgnoredUsersRepository { + pool: (), +} + +pub struct IgnoredUser; + +#[derive(Debug, thiserror::Error)] +pub enum IgnoredUsersRepositoryError { + #[error(transparent)] + Unknown(#[from] anyhow::Error), +} + +pub trait IgnoredUsersRepository: Send + Sync { + fn create( + &self, + ignored_user_create: IgnoredUser, + ) -> impl Future> + Send; +} + +pub struct Dependencies<'a, R> { + pub repository: &'a R, +} + +pub struct Request; +pub struct Response; + +pub struct AddIgnoredUser<'a, R> { + repository: &'a R, +} + +impl<'a, R> AddIgnoredUser<'a, R> +where + R: IgnoredUsersRepository, +{ + pub fn new(dependencies: Dependencies<'a, R>) -> Self { + Self { + repository: dependencies.repository, + } + } + + pub async fn exec(&self, request: Request) -> Result { + // NOTE: this call works because the type of `self.repository` is `&R` and `R` implements `IgnoredUsersRepository` + IgnoredUsersRepository::create(self.repository, IgnoredUser {}).await?; + + Ok(Response) + } +} + +impl IgnoredUsersRepository for PostgresIgnoredUsersRepository { + async fn create( + &self, + ignored_user_create: IgnoredUser, + ) -> Result<(), IgnoredUsersRepositoryError> { + // Perform the actual database operation here + println!("Creating ignored user"); + + Ok(()) + } +} + +#[tokio::main] +pub async fn main() -> Result<(), Error> { + let repository = PostgresIgnoredUsersRepository { pool: () }; + + // This abstracts the use case from the DB operations that run on the repository + let use_case = AddIgnoredUser::new(Dependencies { + repository: &repository, + }); + + use_case.exec(Request {}).await?; + println!("Performed use case"); + + Ok(()) +} From ff8818962f15e066e112f524dc1548eac19bd424 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Sun, 13 Jul 2025 21:45:20 +0530 Subject: [PATCH 78/79] chore: update example --- src/bin/db-abstraction1.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/bin/db-abstraction1.rs b/src/bin/db-abstraction1.rs index 9022454..81eb6b8 100644 --- a/src/bin/db-abstraction1.rs +++ b/src/bin/db-abstraction1.rs @@ -74,9 +74,10 @@ pub async fn main() -> Result<(), Error> { let repository = PostgresIgnoredUsersRepository { pool: () }; // This abstracts the use case from the DB operations that run on the repository - let use_case = AddIgnoredUser::new(Dependencies { + let dependency = Dependencies { repository: &repository, - }); + }; + let use_case = AddIgnoredUser::new(dependency); use_case.exec(Request {}).await?; println!("Performed use case"); From d12807b626dbd12ba649cab2422b2440f653d7e3 Mon Sep 17 00:00:00 2001 From: Michael de Silva Date: Wed, 30 Jul 2025 23:34:15 +0530 Subject: [PATCH 79/79] chore: add example for walking validation errors using thiserror --- .gitignore | 1 + src/bin/thiserror-walk-all-errors.rs | 104 +++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 src/bin/thiserror-walk-all-errors.rs diff --git a/.gitignore b/.gitignore index d3ae36e..4314c0f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.idea/ /target /examples/**/target/* /.vscode diff --git a/src/bin/thiserror-walk-all-errors.rs b/src/bin/thiserror-walk-all-errors.rs new file mode 100644 index 0000000..c664e06 --- /dev/null +++ b/src/bin/thiserror-walk-all-errors.rs @@ -0,0 +1,104 @@ +use std::fmt::{Display, Formatter}; + +#[derive(Debug, thiserror::Error)] +pub enum ValidateOnePaperDocumentErrorKind { + #[error("another error")] + AnotherError, + #[error("you do not have the permissions to upload this document")] + Forbidden, +} + +#[derive(Debug, thiserror::Error)] +#[error("{document_kind}: {error_kind}")] +pub struct ValidateOnePaperDocumentError { + document_kind: PaperDocuments, + error_kind: ValidateOnePaperDocumentErrorKind, +} + +#[derive(Debug)] +pub struct ValidatePaperDocumentsError { + errors: Vec, +} + +impl Default for ValidatePaperDocumentsError { + fn default() -> Self { + Self { errors: Vec::new() } + } +} + +#[derive(Debug)] +pub enum PaperDocuments { + Office, + Ledger, +} + +impl Display for PaperDocuments { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + PaperDocuments::Office => f.write_str("Office"), + PaperDocuments::Ledger => f.write_str("Ledger"), + } + } +} + +async fn validate_office_documents<'a, I: Iterator>( + documents: I, +) -> Result<(), ValidatePaperDocumentsError> { + let mut validation = ValidatePaperDocumentsError::default(); + + for (document, kind) in documents { + if *document == 1 { + validation.errors.push(ValidateOnePaperDocumentError { + document_kind: PaperDocuments::Office, + error_kind: ValidateOnePaperDocumentErrorKind::Forbidden, + }); + }; + + if *document == 2 { + validation.errors.push(ValidateOnePaperDocumentError { + document_kind: PaperDocuments::Ledger, + error_kind: ValidateOnePaperDocumentErrorKind::AnotherError, + }); + }; + + continue; + } + + if !validation.errors.is_empty() { + return Err(validation); + } + + Ok(()) +} + +#[derive(Debug, thiserror::Error)] +pub enum CustomError { + #[error("custom error: {0}")] + ErrorMessage(String), +} + +impl From for CustomError { + fn from(value: ValidatePaperDocumentsError) -> Self { + let all_errors = value + .errors + .iter() + .map(ToString::to_string) + .collect::>() + .join(", "); + + Self::ErrorMessage(format!("paper documents validation error: {all_errors}")) + } +} + +#[tokio::main] +async fn main() -> Result<(), CustomError> { + let docs = vec![ + (0, PaperDocuments::Office), + (1, PaperDocuments::Ledger), + (2, PaperDocuments::Office), + ]; + + let r = validate_office_documents(docs.iter()).await?; + + Ok(()) +}