diff --git a/Cargo.lock b/Cargo.lock index 3e7f0028dc..610a6fbad2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" + [[package]] name = "addr2line" version = "0.17.0" @@ -37,6 +43,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + [[package]] name = "ambient-authority" version = "0.0.1" @@ -2185,14 +2197,52 @@ version = "6.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff" +[[package]] +name = "ouroboros" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f56a2b0aa5fc88687aaf63e85a7974422790ce3419a2e1a15870f8a55227822" +dependencies = [ + "aliasable", + "ouroboros_macro", +] + +[[package]] +name = "ouroboros_macro" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c40641e27d0eb38cae3dee081d920104d2db47a8e853c1a592ef68d33f5ebf4" +dependencies = [ + "Inflector", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "outbound-http" +version = "0.2.0" +dependencies = [ + "anyhow", + "bytes", + "http", + "reqwest", + "spin-app", + "spin-core", + "tracing", + "tracing-futures", + "url", + "wit-bindgen-wasmtime", +] + [[package]] name = "outbound-pg" version = "0.2.0" dependencies = [ "anyhow", "postgres", - "spin-engine", - "spin-manifest", + "spin-core", "tracing", "wit-bindgen-wasmtime", ] @@ -2202,24 +2252,14 @@ name = "outbound-redis" version = "0.2.0" dependencies = [ "anyhow", - "owning_ref", "redis", - "spin-engine", - "spin-manifest", + "spin-core", + "tokio", "tracing", "tracing-futures", "wit-bindgen-wasmtime", ] -[[package]] -name = "owning_ref" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff55baddef9e4ad00f88b6c743a2a8062d4c6ade126c2a528644b8e444d52ce" -dependencies = [ - "stable_deref_trait", -] - [[package]] name = "parking_lot" version = "0.11.2" @@ -2938,9 +2978,9 @@ dependencies = [ [[package]] name = "sanitize-filename" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf18934a12018228c5b55a6dae9df5d0641e3566b3630cb46cc55564068e7c2f" +checksum = "08c502bdb638f1396509467cb0580ef3b29aa2a45c5d43e5d84928241280296c" dependencies = [ "lazy_static", "regex", @@ -3260,6 +3300,19 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin-app" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "ouroboros", + "serde", + "serde_json", + "spin-core", + "thiserror", +] + [[package]] name = "spin-build" version = "0.2.0" @@ -3296,7 +3349,6 @@ dependencies = [ "lazy_static", "nix 0.24.2", "openssl", - "outbound-redis", "path-absolutize", "regex", "reqwest", @@ -3306,7 +3358,6 @@ dependencies = [ "sha2 0.10.3", "spin-build", "spin-config", - "spin-engine", "spin-http-engine", "spin-loader", "spin-manifest", @@ -3323,7 +3374,6 @@ dependencies = [ "url", "uuid", "vergen", - "wasi-outbound-http", "wasmtime", "which", ] @@ -3333,32 +3383,28 @@ name = "spin-config" version = "0.2.0" dependencies = [ "anyhow", - "serde", + "async-trait", + "dotenvy", + "once_cell", + "spin-app", + "spin-core", "thiserror", + "tokio", "toml", "wit-bindgen-wasmtime", ] [[package]] -name = "spin-engine" -version = "0.2.0" +name = "spin-core" +version = "0.1.0" dependencies = [ "anyhow", - "bytes", - "cap-std 0.24.4", - "dirs 4.0.0", - "sanitize-filename", - "spin-config", - "spin-manifest", - "tempfile", - "tokio", + "async-trait", "tracing", - "tracing-futures", "wasi-cap-std-sync", "wasi-common", "wasmtime", "wasmtime-wasi", - "wit-bindgen-wasmtime", ] [[package]] @@ -3383,7 +3429,8 @@ dependencies = [ "percent-encoding", "rustls-pemfile 0.3.0", "serde", - "spin-engine", + "spin-app", + "spin-core", "spin-manifest", "spin-testing", "spin-trigger", @@ -3392,12 +3439,8 @@ dependencies = [ "tokio-rustls", "tracing", "tracing-futures", - "tracing-subscriber", "url", - "wasi-cap-std-sync", - "wasi-common", "wasmtime", - "wasmtime-wasi", "wit-bindgen-wasmtime", ] @@ -3415,12 +3458,13 @@ dependencies = [ "glob", "itertools", "lazy_static", + "outbound-http", "path-absolutize", "regex", "reqwest", "serde", "sha2 0.10.3", - "spin-config", + "spin-app", "spin-manifest", "tempfile", "tokio", @@ -3428,9 +3472,7 @@ dependencies = [ "toml", "tracing", "tracing-futures", - "tracing-subscriber", "walkdir", - "wasi-outbound-http", ] [[package]] @@ -3493,14 +3535,14 @@ dependencies = [ "log", "redis", "serde", - "spin-engine", + "spin-app", + "spin-core", "spin-manifest", "spin-testing", "spin-trigger", "tokio", "tracing", "tracing-futures", - "tracing-subscriber", "wasi-common", "wasmtime", "wasmtime-wasi", @@ -3562,33 +3604,13 @@ dependencies = [ "anyhow", "http", "hyper", - "spin-engine", + "serde", + "serde_json", + "spin-app", + "spin-core", "spin-http-engine", - "spin-manifest", - "spin-trigger", -] - -[[package]] -name = "spin-timer" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "chrono", - "env_logger", - "futures", - "log", - "spin-engine", - "spin-manifest", "spin-trigger", - "tokio", - "tracing", - "tracing-futures", "tracing-subscriber", - "wasi-common", - "wasmtime", - "wasmtime-wasi", - "wit-bindgen-wasmtime", ] [[package]] @@ -3599,19 +3621,21 @@ dependencies = [ "async-trait", "clap 3.2.19", "ctrlc", - "dotenvy", + "dirs 4.0.0", "futures", "http", + "outbound-http", "outbound-pg", "outbound-redis", + "sanitize-filename", "serde", + "serde_json", + "spin-app", "spin-config", - "spin-engine", + "spin-core", "spin-loader", "spin-manifest", "tracing", - "wasi-outbound-http", - "wasmtime", ] [[package]] @@ -4297,21 +4321,21 @@ dependencies = [ ] [[package]] -name = "wasi-outbound-http" -version = "0.2.0" +name = "wasi-tokio" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab325bba31ae9286b8ebdc18d32a43d6471312c9bc4e477240be444e00ec4f4" dependencies = [ "anyhow", - "bytes", - "futures", - "http", - "reqwest", - "spin-engine", - "spin-manifest", + "cap-std 0.25.2", + "io-extras 0.15.0", + "io-lifetimes 0.7.3", + "lazy_static", + "rustix 0.35.9", "tokio", - "tracing", - "tracing-futures", - "url", - "wit-bindgen-wasmtime", + "wasi-cap-std-sync", + "wasi-common", + "wiggle", ] [[package]] @@ -4592,6 +4616,7 @@ dependencies = [ "anyhow", "wasi-cap-std-sync", "wasi-common", + "wasi-tokio", "wasmtime", "wiggle", ] @@ -4679,6 +4704,7 @@ dependencies = [ "tracing", "wasmtime", "wiggle-macro", + "witx", ] [[package]] @@ -4878,6 +4904,7 @@ version = "0.2.0" source = "git+https://github.com/bytecodealliance/wit-bindgen?rev=cb871cfa1ee460b51eb1d144b175b9aab9c50aba#cb871cfa1ee460b51eb1d144b175b9aab9c50aba" dependencies = [ "anyhow", + "async-trait", "bitflags", "thiserror", "wasmtime", diff --git a/Cargo.toml b/Cargo.toml index ae1d9c0f3d..44384ba56e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,6 @@ hippo-openapi = "0.10" hippo = { git = "https://github.com/deislabs/hippo-cli", tag = "v0.16.1" } lazy_static = "1.4.0" nix = { version = "0.24", features = ["signal"] } -outbound-redis = { path = "crates/outbound-redis" } path-absolutize = "3.0.11" regex = "1.5.5" reqwest = { version = "0.11", features = ["stream"] } @@ -31,7 +30,6 @@ serde_json = "1.0.82" sha2 = "0.10.2" spin-build = { path = "crates/build" } spin-config = { path = "crates/config" } -spin-engine = { path = "crates/engine" } spin-http-engine = { path = "crates/http" } spin-loader = { path = "crates/loader" } spin-manifest = { path = "crates/manifest" } @@ -44,10 +42,9 @@ tokio = { version = "1.11", features = [ "full" ] } toml = "0.5" tracing = { version = "0.1", features = [ "log" ] } tracing-futures = "0.2" -tracing-subscriber = { version = "0.3.7", features = [ "env-filter" ] } +tracing-subscriber = { version = "0.3", features = [ "env-filter" ] } url = "2.2.2" uuid = "^1.0" -wasi-outbound-http = { path = "crates/outbound-http" } wasmtime = "0.39.1" [target.'cfg(target_os = "linux")'.dependencies] @@ -70,9 +67,10 @@ e2e-tests = [] [workspace] members = [ + "crates/app", "crates/build", "crates/config", - "crates/engine", + "crates/core", "crates/http", "crates/loader", "crates/manifest", @@ -82,7 +80,6 @@ members = [ "crates/templates", "crates/testing", "crates/trigger", - "examples/spin-timer", "sdk/rust", "sdk/rust/macro" ] diff --git a/build.rs b/build.rs index b65cdc7a76..52f12997e4 100644 --- a/build.rs +++ b/build.rs @@ -48,7 +48,6 @@ error: the `wasm32-wasi` target is not installed "crates/http/benches/spin-http-benchmark", ); build_wasm_test_program("wagi-benchmark.wasm", "crates/http/benches/wagi-benchmark"); - build_wasm_test_program("echo.wasm", "examples/spin-timer/example"); cargo_build(RUST_HTTP_INTEGRATION_TEST); cargo_build(RUST_HTTP_INTEGRATION_ENV_TEST); diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml new file mode 100644 index 0000000000..1d8dedf4b3 --- /dev/null +++ b/crates/app/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "spin-app" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0" +async-trait = "0.1" +ouroboros = "0.15" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +spin-core = { path = "../core" } +thiserror = "1.0" diff --git a/crates/app/src/host_component.rs b/crates/app/src/host_component.rs new file mode 100644 index 0000000000..c428aefe45 --- /dev/null +++ b/crates/app/src/host_component.rs @@ -0,0 +1,51 @@ +use std::sync::Arc; + +use spin_core::{EngineBuilder, HostComponent, HostComponentsData}; + +use crate::AppComponent; + +pub trait DynamicHostComponent: HostComponent { + fn update_data(&self, data: &mut Self::Data, component: &AppComponent) -> anyhow::Result<()>; +} + +impl DynamicHostComponent for Arc { + fn update_data(&self, data: &mut Self::Data, component: &AppComponent) -> anyhow::Result<()> { + (**self).update_data(data, component) + } +} + +type DataUpdater = + Box anyhow::Result<()> + Send + Sync>; + +#[derive(Default)] +pub struct DynamicHostComponents { + data_updaters: Vec, +} + +impl DynamicHostComponents { + pub fn add_dynamic_host_component( + &mut self, + engine_builder: &mut EngineBuilder, + host_component: DHC, + ) -> anyhow::Result<()> { + let host_component = Arc::new(host_component); + let handle = engine_builder.add_host_component(host_component.clone())?; + self.data_updaters + .push(Box::new(move |host_components_data, component| { + let data = host_components_data.get_or_insert(handle); + host_component.update_data(data, component) + })); + Ok(()) + } + + pub fn update_data( + &self, + host_components_data: &mut HostComponentsData, + component: &AppComponent, + ) -> anyhow::Result<()> { + for data_updater in &self.data_updaters { + data_updater(host_components_data, component)?; + } + Ok(()) + } +} diff --git a/crates/app/src/lib.rs b/crates/app/src/lib.rs new file mode 100644 index 0000000000..92a3bf48c3 --- /dev/null +++ b/crates/app/src/lib.rs @@ -0,0 +1,267 @@ +mod host_component; +pub mod locked; +pub mod values; + +use ouroboros::self_referencing; +use serde::Deserialize; +use spin_core::{wasmtime, Engine, EngineBuilder, StoreBuilder}; + +use host_component::DynamicHostComponents; +use locked::{ContentPath, LockedApp, LockedComponent, LockedComponentSource, LockedTrigger}; + +pub use async_trait::async_trait; +pub use host_component::DynamicHostComponent; +pub use locked::Variable; + +// TODO(lann): Should this migrate to spin-loader? +#[async_trait] +pub trait Loader { + async fn load_app(&self, uri: &str) -> anyhow::Result; + + async fn load_module( + &self, + engine: &wasmtime::Engine, + source: &LockedComponentSource, + ) -> anyhow::Result; + + async fn mount_files( + &self, + store_builder: &mut StoreBuilder, + component: &AppComponent, + ) -> anyhow::Result<()>; +} + +pub struct AppLoader { + inner: Box, + dynamic_host_components: DynamicHostComponents, +} + +impl AppLoader { + pub fn new(loader: impl Loader + Send + Sync + 'static) -> Self { + Self { + inner: Box::new(loader), + dynamic_host_components: Default::default(), + } + } + + pub fn add_dynamic_host_component( + &mut self, + engine_builder: &mut EngineBuilder, + host_component: DHC, + ) -> anyhow::Result<()> { + self.dynamic_host_components + .add_dynamic_host_component(engine_builder, host_component) + } + + pub async fn load_app(&self, uri: String) -> Result { + let locked = self + .inner + .load_app(&uri) + .await + .map_err(Error::LoaderError)?; + Ok(App { + loader: self, + uri, + locked, + }) + } + + pub async fn load_owned_app(self, uri: String) -> Result { + OwnedApp::try_new_async(self, |loader| Box::pin(loader.load_app(uri))).await + } +} + +impl std::fmt::Debug for AppLoader { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AppLoader").finish() + } +} + +#[self_referencing] +#[derive(Debug)] +pub struct OwnedApp { + pub loader: AppLoader, + #[borrows(loader)] + #[covariant] + app: App<'this>, +} + +impl std::ops::Deref for OwnedApp { + type Target = App<'static>; + + fn deref(&self) -> &Self::Target { + unsafe { + // We know that App's lifetime param is for AppLoader, which is owned by self here. + std::mem::transmute::<&App, &App<'static>>(self.borrow_app()) + } + } +} + +#[derive(Debug)] +pub struct App<'a> { + loader: &'a AppLoader, + uri: String, + locked: LockedApp, +} + +impl<'a> App<'a> { + pub fn uri(&self) -> &str { + &self.uri + } + + pub fn get_metadata<'this, T: Deserialize<'this>>(&'this self, key: &str) -> Option> { + self.locked + .metadata + .get(key) + .map(|value| Ok(T::deserialize(value)?)) + } + + pub fn require_metadata<'this, T: Deserialize<'this>>(&'this self, key: &str) -> Result { + self.get_metadata(key) + .ok_or_else(|| Error::ManifestError(format!("missing required {key:?}")))? + } + + pub fn variables(&self) -> impl Iterator { + self.locked.variables.iter() + } + + pub fn components(&self) -> impl Iterator { + self.locked + .components + .iter() + .map(|locked| AppComponent { app: self, locked }) + } + + pub fn get_component(&self, component_id: &str) -> Option { + self.components() + .find(|component| component.locked.id == component_id) + } + + pub fn triggers(&self) -> impl Iterator { + self.locked + .triggers + .iter() + .map(|locked| AppTrigger { app: self, locked }) + } + + pub fn triggers_with_type(&'a self, trigger_type: &'a str) -> impl Iterator { + self.triggers() + .filter(move |trigger| trigger.locked.trigger_type == trigger_type) + } +} + +pub struct AppComponent<'a> { + pub app: &'a App<'a>, + locked: &'a LockedComponent, +} + +impl<'a> AppComponent<'a> { + pub fn id(&self) -> &str { + &self.locked.id + } + + pub fn source(&self) -> &LockedComponentSource { + &self.locked.source + } + + pub fn files(&self) -> std::slice::Iter { + self.locked.files.iter() + } + + pub fn get_metadata>(&self, key: &str) -> Option> { + self.locked + .metadata + .get(key) + .map(|value| Ok(T::deserialize(value)?)) + } + + pub fn config(&self) -> impl Iterator { + self.locked.config.iter() + } + + pub async fn load_module( + &self, + engine: &Engine, + ) -> Result { + self.app + .loader + .inner + .load_module(engine.as_ref(), &self.locked.source) + .await + .map_err(Error::LoaderError) + } + + pub async fn apply_store_config(&self, builder: &mut StoreBuilder) -> Result<()> { + builder.env(&self.locked.env).map_err(Error::CoreError)?; + + let loader = self.app.loader; + loader + .inner + .mount_files(builder, self) + .await + .map_err(Error::LoaderError)?; + + loader + .dynamic_host_components + .update_data(builder.host_components_data(), self) + .map_err(Error::HostComponentError)?; + + Ok(()) + } +} + +pub struct AppTrigger<'a> { + pub app: &'a App<'a>, + locked: &'a LockedTrigger, +} + +impl<'a> AppTrigger<'a> { + pub fn id(&self) -> &str { + &self.locked.id + } + + pub fn trigger_type(&self) -> &str { + &self.locked.trigger_type + } + + pub fn component(&self) -> Result> { + let component_id = self.locked.trigger_config.get("component").ok_or_else(|| { + Error::ManifestError(format!( + "trigger {:?} missing 'component' config field", + self.locked.id + )) + })?; + let component_id = component_id.as_str().ok_or_else(|| { + Error::ManifestError(format!( + "trigger {:?} 'component' field has unexpected value {:?}", + self.locked.id, component_id + )) + })?; + self.app.get_component(component_id).ok_or_else(|| { + Error::ManifestError(format!( + "missing component {:?} configured for trigger {:?}", + component_id, self.locked.id + )) + }) + } + + pub fn typed_config>(&self) -> Result { + Ok(Config::deserialize(&self.locked.trigger_config)?) + } +} + +pub type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("spin core error: {0}")] + CoreError(anyhow::Error), + #[error("host component error: {0}")] + HostComponentError(anyhow::Error), + #[error("loader error: {0}")] + LoaderError(anyhow::Error), + #[error("manifest error: {0}")] + ManifestError(String), + #[error("json error: {0}")] + JsonError(#[from] serde_json::Error), +} diff --git a/crates/app/src/locked.rs b/crates/app/src/locked.rs new file mode 100644 index 0000000000..83c90bebb8 --- /dev/null +++ b/crates/app/src/locked.rs @@ -0,0 +1,154 @@ +use std::path::PathBuf; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::values::ValuesMap; + +// LockedMap gives deterministic encoding, which we want. +pub type LockedMap = std::collections::BTreeMap; + +/// A LockedApp represents a "fully resolved" Spin application. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LockedApp { + /// Locked schema version + pub spin_lock_version: FixedVersion<0>, + /// Application metadata + #[serde(default, skip_serializing_if = "ValuesMap::is_empty")] + pub metadata: ValuesMap, + /// Custom config variables + #[serde(default, skip_serializing_if = "LockedMap::is_empty")] + pub variables: LockedMap, + /// Application triggers + pub triggers: Vec, + /// Application components + pub components: Vec, +} + +impl LockedApp { + pub fn from_json(contents: &[u8]) -> serde_json::Result { + serde_json::from_slice(contents) + } + + pub fn to_json(&self) -> serde_json::Result> { + serde_json::to_vec_pretty(&self) + } +} + +/// A LockedComponent represents a "fully resolved" Spin component. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LockedComponent { + /// Application-unique component identifier + pub id: String, + /// Component metadata + #[serde(default, skip_serializing_if = "ValuesMap::is_empty")] + pub metadata: ValuesMap, + /// Wasm source + pub source: LockedComponentSource, + /// WASI environment variables + #[serde(default, skip_serializing_if = "LockedMap::is_empty")] + pub env: LockedMap, + /// WASI filesystem contents + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub files: Vec, + /// Custom config values + #[serde(default, skip_serializing_if = "LockedMap::is_empty")] + pub config: LockedMap, +} + +/// A LockedComponentSource specifies a Wasm source. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LockedComponentSource { + /// Wasm source content type (e.g. "application/wasm") + pub content_type: String, + /// Wasm source content specification + #[serde(flatten)] + pub content: ContentRef, +} + +/// A ContentPath specifies content mapped to a WASI path. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ContentPath { + /// Content specification + #[serde(flatten)] + pub content: ContentRef, + /// WASI mount path + pub path: PathBuf, +} + +/// A ContentRef represents content used by an application. +/// +/// At least one of `source` or `digest` must be specified. Implementations may +/// require one or the other (or both). +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct ContentRef { + /// A URI where the content can be accessed. Implementations may support + /// different URI schemes. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, + /// If set, the content must have the given SHA-256 digest. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub digest: Option, +} + +/// A LockedTrigger specifies configuration for an application trigger. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LockedTrigger { + /// Application-unique trigger identifier + pub id: String, + /// Trigger type (e.g. "http") + pub trigger_type: String, + /// Trigger-type-specific configuration + pub trigger_config: Value, +} + +/// A Variable specifies a custom configuration variable. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Variable { + /// The variable's default value. If unset, the variable is required. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub default: Option, + /// If set, the variable's value may be sensitive and e.g. shouldn't be logged. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub secret: bool, +} + +/// FixedVersion represents a schema version field with a const value. +#[allow(unused)] +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(into = "usize", try_from = "usize")] +pub struct FixedVersion; + +impl From> for usize { + fn from(_: FixedVersion) -> usize { + V + } +} + +impl From> for String { + fn from(_: FixedVersion) -> String { + V.to_string() + } +} + +impl TryFrom for FixedVersion { + type Error = String; + + fn try_from(value: usize) -> Result { + if value != V { + return Err(format!("invalid version {} != {}", value, V)); + } + Ok(Self) + } +} + +impl TryFrom for FixedVersion { + type Error = String; + + fn try_from(value: String) -> Result { + let value: usize = value + .parse() + .map_err(|err| format!("invalid version: {}", err))?; + value.try_into() + } +} diff --git a/crates/app/src/values.rs b/crates/app/src/values.rs new file mode 100644 index 0000000000..6a764a662f --- /dev/null +++ b/crates/app/src/values.rs @@ -0,0 +1,57 @@ +use serde::Serialize; +use serde_json::Value; + +// ValuesMap stores dynamically-typed values. +pub type ValuesMap = serde_json::Map; + +/// ValuesMapBuilder assists in building a ValuesMap. +#[derive(Default)] +pub struct ValuesMapBuilder(ValuesMap); + +impl ValuesMapBuilder { + pub fn new() -> Self { + Self::default() + } + + pub fn string(&mut self, key: impl Into, value: impl Into) -> &mut Self { + self.entry(key, value.into()) + } + + pub fn string_option( + &mut self, + key: impl Into, + value: Option>, + ) -> &mut Self { + if let Some(value) = value { + self.0.insert(key.into(), value.into().into()); + } + self + } + + pub fn string_array>( + &mut self, + key: impl Into, + iter: impl IntoIterator, + ) -> &mut Self { + self.entry(key, iter.into_iter().map(|s| s.into()).collect::>()) + } + + pub fn entry(&mut self, key: impl Into, value: impl Into) -> &mut Self { + self.0.insert(key.into(), value.into()); + self + } + + pub fn serializable( + &mut self, + key: impl Into, + value: impl Serialize, + ) -> serde_json::Result<&mut Self> { + let value = serde_json::to_value(value)?; + self.0.insert(key.into(), value); + Ok(self) + } + + pub fn build(&mut self) -> ValuesMap { + std::mem::take(&mut self.0) + } +} diff --git a/crates/config/Cargo.toml b/crates/config/Cargo.toml index d7edb3c71f..960dbe38a3 100644 --- a/crates/config/Cargo.toml +++ b/crates/config/Cargo.toml @@ -6,9 +6,18 @@ authors = [ "Fermyon Engineering " ] [dependencies] anyhow = "1.0" -serde = { version = "1.0", features = [ "derive" ] } +async-trait = "0.1" +dotenvy = "0.15" +once_cell = "1" +spin-app = { path = "../app" } +spin-core = { path = "../core" } thiserror = "1" -wit-bindgen-wasmtime = { git = "https://github.com/bytecodealliance/wit-bindgen", rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" } +tokio = { version = "1", features = ["rt-multi-thread"] } + +[dependencies.wit-bindgen-wasmtime] +git = "https://github.com/bytecodealliance/wit-bindgen" +rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" +features = ["async"] [dev-dependencies] toml = "0.5" diff --git a/crates/config/src/host_component.rs b/crates/config/src/host_component.rs index 20e7528bc6..6d98ea1d8c 100644 --- a/crates/config/src/host_component.rs +++ b/crates/config/src/host_component.rs @@ -1,40 +1,82 @@ -use std::sync::Arc; +use std::sync::{Arc, Mutex}; -use crate::{Error, Key, Resolver, TreePath}; +use async_trait::async_trait; +use once_cell::sync::OnceCell; +use spin_app::{AppComponent, DynamicHostComponent}; +use spin_core::HostComponent; -mod wit { - wit_bindgen_wasmtime::export!("../../wit/ephemeral/spin-config.wit"); +use crate::{Error, Key, Provider, Resolver}; + +wit_bindgen_wasmtime::export!({paths: ["../../wit/ephemeral/spin-config.wit"], async: *}); + +pub struct ConfigHostComponent { + providers: Mutex>>, + resolver: Arc>, } -pub use wit::spin_config::add_to_linker; -/// A component configuration interface implementation. -pub struct ComponentConfig { - component_root: TreePath, - resolver: Arc, +impl ConfigHostComponent { + pub fn new(providers: Vec>) -> Self { + Self { + providers: Mutex::new(providers), + resolver: Default::default(), + } + } } -impl ComponentConfig { - pub fn new(component_id: impl Into, resolver: Arc) -> crate::Result { - let component_root = TreePath::new(component_id).or_else(|_| { - // Temporary mitigation for https://github.com/fermyon/spin/issues/337 - TreePath::new("invalid.path.issue_337") +impl HostComponent for ConfigHostComponent { + type Data = ComponentConfig; + + fn add_to_linker( + linker: &mut spin_core::Linker, + get: impl Fn(&mut spin_core::Data) -> &mut Self::Data + Send + Sync + Copy + 'static, + ) -> anyhow::Result<()> { + spin_config::add_to_linker(linker, get) + } + + fn build_data(&self) -> Self::Data { + ComponentConfig { + resolver: self.resolver.clone(), + component_id: None, + } + } +} + +impl DynamicHostComponent for ConfigHostComponent { + fn update_data(&self, data: &mut Self::Data, component: &AppComponent) -> anyhow::Result<()> { + self.resolver.get_or_try_init(|| { + let mut resolver = Resolver::new(component.app)?; + for provider in self.providers.lock().unwrap().drain(..) { + resolver.add_provider(provider); + } + Ok::<_, anyhow::Error>(resolver) })?; - Ok(Self { - component_root, - resolver, - }) + data.component_id = Some(component.id().to_string()); + Ok(()) } } -impl wit::spin_config::SpinConfig for ComponentConfig { - fn get_config(&mut self, key: &str) -> Result { +/// A component configuration interface implementation. +pub struct ComponentConfig { + resolver: Arc>, + component_id: Option, +} + +#[async_trait] +impl spin_config::SpinConfig for ComponentConfig { + async fn get_config(&mut self, key: &str) -> Result { + // Set by DynamicHostComponent::update_data + let component_id = self.component_id.as_deref().unwrap(); let key = Key::new(key)?; - let path = &self.component_root + key; - Ok(self.resolver.resolve(&path)?) + Ok(self + .resolver + .get() + .unwrap() + .resolve(component_id, key) + .await?) } } -impl From for wit::spin_config::Error { +impl From for spin_config::Error { fn from(err: Error) -> Self { match err { Error::InvalidKey(msg) => Self::InvalidKey(msg), diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 3df750684a..4fc1a4f1c9 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -1,14 +1,15 @@ -pub mod host_component; +mod host_component; pub mod provider; mod template; -mod tree; -use std::fmt::Debug; +use std::{borrow::Cow, collections::HashMap, fmt::Debug}; -pub use provider::Provider; -pub use tree::{Tree, TreePath}; +use anyhow::{anyhow, Context}; +use spin_app::{App, Variable}; +pub use host_component::ConfigHostComponent; +pub use provider::Provider; use template::{Part, Template}; /// A config resolution error. @@ -44,81 +45,102 @@ type Result = std::result::Result; /// A configuration resolver. #[derive(Debug, Default)] pub struct Resolver { - tree: Tree, + variables: HashMap, + // component_id -> key -> template + component_configs: HashMap>, providers: Vec>, } impl Resolver { /// Creates a Resolver for the given Tree. - pub fn new(tree: Tree) -> Result { + pub fn new(app: &App) -> anyhow::Result { + let variables = app + .variables() + .map(|(key, var)| (key.clone(), var.clone())) + .collect(); + + let mut component_configs = HashMap::new(); + for component in app.components() { + let templates: &mut HashMap<_, _> = component_configs + .entry(component.id().to_string()) + .or_default(); + for (key, val) in component.config() { + Key::validate(key).with_context(|| { + format!( + "invalid config key {:?} for component {:?}", + key, + component.id() + ) + })?; + let template = Template::new(val.as_str()).with_context(|| { + format!( + "invalid config value for {:?} for component {:?}", + key, + component.id() + ) + })?; + templates.insert(key.clone(), template); + } + } + Ok(Self { - tree, - providers: vec![], + variables, + component_configs, + providers: Default::default(), }) } /// Adds a config Provider to the Resolver. - pub fn add_provider(&mut self, provider: impl Provider + 'static) { - self.providers.push(Box::new(provider)); + pub fn add_provider(&mut self, provider: impl Into>) { + self.providers.push(provider.into()); } - /// Resolves a config value for the given path. - pub fn resolve(&self, path: &TreePath) -> Result { - self.resolve_path(path, 0) - } + /// Resolve a config value for the given component and key. + pub async fn resolve(&self, component_id: &str, key: Key<'_>) -> Result { + let component_config = self.component_configs.get(component_id).ok_or_else(|| { + Error::UnknownPath(format!("no config for component {component_id:?}")) + })?; - // Simple protection against infinite recursion - const RECURSION_LIMIT: usize = 100; + let template = component_config + .get(key.as_ref()) + .ok_or_else(|| Error::UnknownPath(format!("no config for {component_id:?}.{key:?}")))?; - // TODO(lann): make this non-recursive and/or "flatten" templates - fn resolve_path(&self, path: &TreePath, depth: usize) -> Result { - let depth = depth + 1; - if depth > Self::RECURSION_LIMIT { - return Err(Error::InvalidTemplate(format!( - "hit recursion limit at path: {}", - path - ))); + self.resolve_template(template).await.map_err(|err| { + Error::InvalidTemplate(format!( + "failed to resolve template for {component_id:?}.{key:?}: {err:?}" + )) + }) + } + + async fn resolve_template(&self, template: &Template) -> Result { + let mut resolved: Vec> = Vec::with_capacity(template.parts().len()); + for part in template.parts() { + resolved.push(match part { + Part::Lit(lit) => lit.as_ref().into(), + Part::Expr(expr) => self.resolve_expr(expr).await?, + }); } - let slot = self.tree.get(path)?; - // If we're resolving top-level config we are ready to query provider(s). - if path.size() == 1 { - let key = path.keys().next().unwrap(); - for provider in &self.providers { - if let Some(value) = provider.get(&key).map_err(Error::Provider)? { - return Ok(value); - } + Ok(resolved.concat()) + } + + async fn resolve_expr(&self, expr: &str) -> Result> { + let var = self + .variables + .get(expr) + .ok_or_else(|| Error::UnknownPath(format!("no variable named {expr:?}")))?; + + for provider in &self.providers { + if let Some(value) = provider.get(&Key(expr)).await.map_err(Error::Provider)? { + return Ok(value.into()); } } - // Resolve default template - if let Some(template) = &slot.default { - self.resolve_template(path, template, depth) - } else { - Err(Error::InvalidPath(format!( - "missing value at required path: {}", - path - ))) - } - } - fn resolve_template( - &self, - path: &TreePath, - template: &Template, - depth: usize, - ) -> Result { - template.parts().try_fold(String::new(), |value, part| { - Ok(match part { - Part::Lit(lit) => value + lit, - Part::Expr(expr) => { - let expr_path = if expr.starts_with('.') { - path.resolve_relative(expr)? - } else { - TreePath::new(expr.to_string())? - }; - value + &self.resolve_path(&expr_path, depth)? - } - }) - }) + match var.default { + Some(ref default) => Ok(default.into()), + None => Err(Error::Provider(anyhow!( + "no provider resolved variable {expr:?}" + ))), + } } } @@ -168,81 +190,8 @@ impl<'a> AsRef for Key<'a> { #[cfg(test)] mod tests { - use std::collections::HashMap; - - use toml::toml; - use super::*; - #[test] - fn resolver_resolve_defaults() { - let mut tree: Tree = toml! { - top_level = { default = "top" } - top_ref = { default = "{{ top_level }}+{{ top_level }}" } - top_required = { required = true } - } - .try_into() - .unwrap(); - tree.merge_defaults( - &TreePath::new("child").unwrap(), - toml! { - subtree_key = "sub" - top_ref = "{{ top_level }}" - recurse_ref = "{{ top_ref }}" - own_ref = "{{ .subtree_key }}" - } - .try_into::>() - .unwrap(), - ) - .unwrap(); - tree.merge_defaults( - &TreePath::new("child.grandchild").unwrap(), - toml! { - top_ref = "{{ top_level }}" - parent_ref = "{{ ..subtree_key }}" - mixed_ref = "{{ top_level }}/{{ ..recurse_ref }}" - } - .try_into::>() - .unwrap(), - ) - .unwrap(); - - let resolver = Resolver::new(tree).unwrap(); - for (path, expected) in [ - ("top_level", "top"), - ("top_ref", "top+top"), - ("child.subtree_key", "sub"), - ("child.top_ref", "top"), - ("child.recurse_ref", "top+top"), - ("child.own_ref", "sub"), - ("child.grandchild.top_ref", "top"), - ("child.grandchild.parent_ref", "sub"), - ("child.grandchild.mixed_ref", "top/top+top"), - ] { - let path = TreePath::new(path).unwrap(); - let value = resolver.resolve(&path).unwrap(); - assert_eq!(value, expected, "mismatch at {:?}", path); - } - } - - #[test] - fn resolver_recursion_limit() { - let resolver = Resolver::new( - toml! { - x = { default = "{{y}}" } - y = { default = "{{x}}" } - } - .try_into() - .unwrap(), - ) - .unwrap(); - let path = "x".to_string().try_into().unwrap(); - assert!(matches!( - resolver.resolve(&path), - Err(Error::InvalidTemplate(_)) - )); - } - #[test] fn keys_good() { for key in ["a", "abc", "a1b2c3", "a_1", "a_1_b_3"] { diff --git a/crates/config/src/provider.rs b/crates/config/src/provider.rs index b4a0c50777..9f3799937a 100644 --- a/crates/config/src/provider.rs +++ b/crates/config/src/provider.rs @@ -1,12 +1,15 @@ use std::fmt::Debug; +use async_trait::async_trait; + use crate::Key; /// Environment variable based provider. pub mod env; /// A config provider. +#[async_trait] pub trait Provider: Debug + Send + Sync { /// Returns the value at the given config path, if it exists. - fn get(&self, key: &Key) -> anyhow::Result>; + async fn get(&self, key: &Key) -> anyhow::Result>; } diff --git a/crates/config/src/provider/env.rs b/crates/config/src/provider/env.rs index b4dff88b13..354fa84fc5 100644 --- a/crates/config/src/provider/env.rs +++ b/crates/config/src/provider/env.rs @@ -1,57 +1,73 @@ -use std::collections::HashMap; +use std::{collections::HashMap, path::PathBuf, sync::Mutex}; -use anyhow::Context; +use anyhow::{Context, Result}; +use async_trait::async_trait; use crate::{Key, Provider}; -pub const DEFAULT_PREFIX: &str = "SPIN_APP"; - /// A config Provider that uses environment variables. #[derive(Debug)] pub struct EnvProvider { prefix: String, - envs: HashMap, + dotenv_path: Option, + dotenv_cache: Mutex>>, } impl EnvProvider { /// Creates a new EnvProvider. - pub fn new(prefix: impl Into, envs: HashMap) -> Self { + pub fn new(prefix: impl Into, dotenv_path: Option) -> Self { Self { prefix: prefix.into(), - envs, + dotenv_path, + dotenv_cache: Default::default(), } } -} - -impl Default for EnvProvider { - fn default() -> Self { - Self { - prefix: DEFAULT_PREFIX.to_string(), - envs: HashMap::new(), - } - } -} -impl Provider for EnvProvider { - fn get(&self, key: &Key) -> anyhow::Result> { + fn get_sync(&self, key: &Key) -> Result> { let env_key = format!("{}_{}", &self.prefix, key.as_ref().to_ascii_uppercase()); match std::env::var(&env_key) { - Err(std::env::VarError::NotPresent) => { - if let Some(value) = self.envs.get(&env_key) { - return Ok(Some(value.to_string())); - } - - Ok(None) - } + Err(std::env::VarError::NotPresent) => self.get_dotenv(&env_key), other => other .map(Some) .with_context(|| format!("failed to resolve env var {}", &env_key)), } } + + fn get_dotenv(&self, key: &str) -> Result> { + if self.dotenv_path.is_none() { + return Ok(None); + } + let mut maybe_cache = self + .dotenv_cache + .lock() + .expect("dotenv_cache lock poisoned"); + let cache = match maybe_cache.as_mut() { + Some(cache) => cache, + None => maybe_cache.insert(self.load_dotenv()?), + }; + Ok(cache.get(key).cloned()) + } + + fn load_dotenv(&self) -> Result> { + let path = self.dotenv_path.as_deref().unwrap(); + Ok(dotenvy::from_path_iter(path) + .into_iter() + .flatten() + .collect::, _>>()?) + } +} + +#[async_trait] +impl Provider for EnvProvider { + async fn get(&self, key: &Key) -> Result> { + tokio::task::block_in_place(|| self.get_sync(key)) + } } #[cfg(test)] mod test { + use std::env::temp_dir; + use super::*; #[test] @@ -64,20 +80,22 @@ mod test { "dotenv_val".to_string(), ); assert_eq!( - EnvProvider::new("TESTING_SPIN", envs.clone()) - .get(&key1) + EnvProvider::new("TESTING_SPIN", None) + .get_sync(&key1) .unwrap(), Some("val".to_string()) ); + } - let key2 = Key::new("env_key2").unwrap(); - envs.insert( - "TESTING_SPIN_ENV_KEY2".to_string(), - "dotenv_val".to_string(), - ); + #[test] + fn provider_get_dotenv() { + let dotenv_path = temp_dir().join("spin-env-provider-test"); + std::fs::write(&dotenv_path, b"TESTING_SPIN_ENV_KEY2=dotenv_val").unwrap(); + + let key = Key::new("env_key2").unwrap(); assert_eq!( - EnvProvider::new("TESTING_SPIN", envs.clone()) - .get(&key2) + EnvProvider::new("TESTING_SPIN", Some(dotenv_path)) + .get_sync(&key) .unwrap(), Some("dotenv_val".to_string()) ); @@ -86,6 +104,11 @@ mod test { #[test] fn provider_get_missing() { let key = Key::new("please_do_not_ever_set_this_during_tests").unwrap(); - assert_eq!(EnvProvider::default().get(&key).unwrap(), None); + assert_eq!( + EnvProvider::new("TESTING_SPIN", Default::default()) + .get_sync(&key) + .unwrap(), + None + ); } } diff --git a/crates/config/src/template.rs b/crates/config/src/template.rs index bc9e9ef0ea..783834bf2d 100644 --- a/crates/config/src/template.rs +++ b/crates/config/src/template.rs @@ -40,7 +40,7 @@ impl Template { Ok(Template(parts)) } - pub(crate) fn parts(&self) -> impl Iterator { + pub(crate) fn parts(&self) -> std::slice::Iter { self.0.iter() } } diff --git a/crates/config/src/tree.rs b/crates/config/src/tree.rs deleted file mode 100644 index d39019f25b..0000000000 --- a/crates/config/src/tree.rs +++ /dev/null @@ -1,312 +0,0 @@ -use std::collections::BTreeMap; -use std::collections::HashMap; - -use serde::Deserialize; -use serde::Serialize; - -use crate::template::Template; -use crate::{Error, Key, Result}; - -/// A configuration tree. -#[derive(Clone, Debug, Default, Deserialize, Serialize)] -pub struct Tree(BTreeMap); - -impl Tree { - pub(crate) fn get(&self, path: &TreePath) -> Result<&Slot> { - self.0 - .get(path) - .ok_or_else(|| Error::InvalidPath(format!("no slot at path: {}", path))) - } - - pub fn merge(&mut self, base: &TreePath, other: Tree) -> Result<()> { - for (subpath, slot) in other.0.into_iter() { - self.merge_slot(base + &subpath, slot)?; - } - Ok(()) - } - - pub fn merge_defaults( - &mut self, - base: &TreePath, - defaults: impl IntoIterator, - ) -> Result<()> { - for (ref key, default) in defaults { - let path = base + Key::new(key)?; - let slot = Slot::from_default(default)?; - self.merge_slot(path, slot)?; - } - Ok(()) - } - - fn merge_slot(&mut self, path: TreePath, slot: Slot) -> Result<()> { - if self.0.contains_key(&path) { - return Err(Error::InvalidPath(format!( - "duplicate key at path: {}", - path - ))); - } - self.0.insert(path, slot); - Ok(()) - } -} - -/// A path into a config tree. -#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Deserialize, Serialize)] -#[serde(try_from = "String")] -pub struct TreePath(String); - -impl TreePath { - /// Creates a ConfigPath from a String. - pub fn new(path: impl Into) -> Result { - let path = path.into(); - if path.is_empty() { - return Err(Error::InvalidPath("empty".to_string())); - } - path.split('.').try_for_each(Key::validate)?; - Ok(TreePath(path)) - } - - /// Returns the number of keys in this Path. - pub fn size(&self) -> usize { - self.0.matches('.').count() + 1 - } - - /// Resolves the given relative path (starting with at least one '.'). - pub fn resolve_relative(&self, rel: &str) -> Result { - if rel.is_empty() { - return Err(Error::InvalidPath("rel may not be empty".to_string())); - } - let key = rel.trim_start_matches('.'); - let dots = rel.len() - key.len(); - if dots == 0 { - return Err(Error::InvalidPath("rel must start with a '.'".to_string())); - } - // Remove last `dots` components from path. - let path = match self.0.rmatch_indices('.').chain([(0, "")]).nth(dots - 1) { - Some((0, _)) => key.to_string(), - Some((idx, _)) => format!("{}.{}", &self.0[..idx], key), - None => { - return Err(Error::InvalidPath(format!( - "rel has too many dots relative to base path {}", - self - ))) - } - }; - Ok(Self(path)) - } - - /// Produces an iterator over the keys of the path. - pub fn keys(&self) -> impl Iterator> { - self.0.split('.').map(Key) - } -} - -impl AsRef for TreePath { - fn as_ref(&self) -> &str { - self.0.as_ref() - } -} - -impl std::fmt::Display for TreePath { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.as_ref()) - } -} - -impl std::ops::Add for &TreePath { - type Output = TreePath; - fn add(self, rhs: &TreePath) -> Self::Output { - TreePath(format!("{}.{}", self.0, rhs.0)) - } -} - -impl std::ops::Add> for &TreePath { - type Output = TreePath; - fn add(self, key: Key) -> Self::Output { - TreePath(format!("{}.{}", self.0, key.0)) - } -} - -impl TryFrom for TreePath { - type Error = Error; - fn try_from(value: String) -> Result { - Self::new(value) - } -} - -#[derive(Clone, Default, PartialEq, Deserialize, Serialize)] -#[serde(into = "RawSlot", try_from = "RawSlot")] -pub(crate) struct Slot { - pub secret: bool, - pub default: Option