From 28e80137619394f5bda6d583d7773d71bd8e80be Mon Sep 17 00:00:00 2001 From: Syrus Akbary Date: Sat, 11 Nov 2023 23:27:35 -0800 Subject: [PATCH 1/5] Moved TtyBridge and TtyState to runtime --- lib/cli/src/commands/run/mod.rs | 2 +- lib/cli/src/commands/run/wasi.rs | 2 +- lib/wasi-web/src/runtime.rs | 10 ++-- lib/wasix/src/lib.rs | 13 ++--- lib/wasix/src/os/tty/mod.rs | 45 ---------------- lib/wasix/src/runtime/mod.rs | 33 ++---------- lib/wasix/src/{os/tty => runtime}/tty_sys.rs | 55 +++++++++++++++++--- lib/wasix/src/syscalls/wasix/tty_set.rs | 2 +- 8 files changed, 65 insertions(+), 97 deletions(-) rename lib/wasix/src/{os/tty => runtime}/tty_sys.rs (86%) diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index c7af2cc2c97..e06cd2d706f 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -813,7 +813,7 @@ impl wasmer_wasix::Runtime for Monitorin self.runtime.http_client() } - fn tty(&self) -> Option<&(dyn wasmer_wasix::os::TtyBridge + Send + Sync)> { + fn tty(&self) -> Option<&(dyn wasmer_wasix::runtime::TtyBridge + Send + Sync)> { self.runtime.tty() } } diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index a2862c60b70..372eb5b5fde 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -18,7 +18,6 @@ use wasmer_wasix::{ capabilities::Capabilities, default_fs_backing, get_wasi_versions, http::HttpClient, - os::{tty_sys::SysTty, TtyBridge}, rewind_ext, runners::{MappedCommand, MappedDirectory}, runtime::{ @@ -32,6 +31,7 @@ use wasmer_wasix::{ tokio::{RuntimeOrHandle, TokioTaskManager}, VirtualTaskManagerExt, }, + SysTty, TtyBridge, }, types::__WASI_STDIN_FILENO, wasmer_wasix_types::wasi::Errno, diff --git a/lib/wasi-web/src/runtime.rs b/lib/wasi-web/src/runtime.rs index 202068dddd0..1425e7aa567 100644 --- a/lib/wasi-web/src/runtime.rs +++ b/lib/wasi-web/src/runtime.rs @@ -19,14 +19,14 @@ use wasm_bindgen::prelude::*; use wasm_bindgen_futures::JsFuture; use wasmer_wasix::{ http::{DynHttpClient, HttpRequest, HttpResponse}, - os::{TtyBridge, TtyOptions}, runtime::{ + TtyBridge, TtyOptions, module_cache::ModuleCache, package_loader::{BuiltinPackageLoader, PackageLoader}, resolver::{Source, WapmSource}, task_manager::TaskWasm, }, - VirtualFile, VirtualTaskManager, WasiThreadError, WasiTtyState, + VirtualFile, VirtualTaskManager, WasiThreadError, TtyState, }; use web_sys::WebGl2RenderingContext; @@ -458,8 +458,8 @@ impl TtyBridge for WebRuntime { self.tty.set_line_feeds(true); } - fn tty_get(&self) -> WasiTtyState { - WasiTtyState { + fn tty_get(&self) -> TtyState { + TtyState { cols: self.tty.cols(), rows: self.tty.rows(), width: 800, @@ -473,7 +473,7 @@ impl TtyBridge for WebRuntime { } } - fn tty_set(&self, tty_state: WasiTtyState) { + fn tty_set(&self, tty_state: TtyState) { self.tty.set_cols(tty_state.cols); self.tty.set_rows(tty_state.rows); self.tty.set_echo(tty_state.echo); diff --git a/lib/wasix/src/lib.rs b/lib/wasix/src/lib.rs index b273a4f5e89..d8a03813f25 100644 --- a/lib/wasix/src/lib.rs +++ b/lib/wasix/src/lib.rs @@ -85,16 +85,13 @@ use wasmer_wasix_types::wasi::{Errno, ExitCode}; pub use crate::{ fs::{default_fs_backing, Fd, WasiFs, WasiInodes, VIRTUAL_ROOT_FD}, - os::{ - task::{ - control_plane::WasiControlPlane, - process::{WasiProcess, WasiProcessId}, - thread::{WasiThread, WasiThreadError, WasiThreadHandle, WasiThreadId}, - }, - WasiTtyState, + os::task::{ + control_plane::WasiControlPlane, + process::{WasiProcess, WasiProcessId}, + thread::{WasiThread, WasiThreadError, WasiThreadHandle, WasiThreadId}, }, rewind::*, - runtime::{task_manager::VirtualTaskManager, PluggableRuntime, Runtime}, + runtime::{task_manager::VirtualTaskManager, PluggableRuntime, Runtime, TtyState}, state::{ WasiEnv, WasiEnvBuilder, WasiEnvInit, WasiFunctionEnv, WasiInstanceHandles, WasiStateCreationError, ALL_RIGHTS, diff --git a/lib/wasix/src/os/tty/mod.rs b/lib/wasix/src/os/tty/mod.rs index dc2ed6cf842..7c3e34a10b4 100644 --- a/lib/wasix/src/os/tty/mod.rs +++ b/lib/wasix/src/os/tty/mod.rs @@ -14,8 +14,6 @@ use super::task::signal::SignalHandlerAbi; const TTY_MOBILE_PAUSE: u128 = std::time::Duration::from_millis(200).as_nanos(); -pub mod tty_sys; - #[derive(Debug)] pub enum InputEvent { Key, @@ -411,46 +409,3 @@ impl Tty { }) } } - -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct WasiTtyState { - pub cols: u32, - pub rows: u32, - pub width: u32, - pub height: u32, - pub stdin_tty: bool, - pub stdout_tty: bool, - pub stderr_tty: bool, - pub echo: bool, - pub line_buffered: bool, - pub line_feeds: bool, -} - -impl Default for WasiTtyState { - fn default() -> Self { - Self { - rows: 80, - cols: 25, - width: 800, - height: 600, - stdin_tty: true, - stdout_tty: true, - stderr_tty: true, - echo: false, - line_buffered: false, - line_feeds: true, - } - } -} - -/// Provides access to a TTY. -pub trait TtyBridge { - /// Resets the values - fn reset(&self); - - /// Retrieve the current TTY state. - fn tty_get(&self) -> WasiTtyState; - - /// Set the TTY state. - fn tty_set(&self, _tty_state: WasiTtyState); -} diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index 3cb1b552ffc..692e41e0d8f 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -2,17 +2,16 @@ pub mod module_cache; pub mod package_loader; pub mod resolver; pub mod task_manager; +pub mod tty_sys; pub use self::task_manager::{SpawnMemoryType, VirtualTaskManager}; +pub use self::tty_sys::{SysTty, TtyBridge, TtyState}; use self::{ module_cache::{CacheError, ModuleHash}, task_manager::InlineWaker, }; -use std::{ - fmt, - sync::{Arc, Mutex}, -}; +use std::{fmt, sync::Arc}; use derivative::Derivative; use futures::future::BoxFuture; @@ -21,13 +20,11 @@ use wasmer::Module; use crate::{ http::{DynHttpClient, HttpClient}, - os::TtyBridge, runtime::{ module_cache::{ModuleCache, ThreadLocalCache}, package_loader::{PackageLoader, UnsupportedPackageLoader}, resolver::{MultiSource, Source, WapmSource}, }, - WasiTtyState, }; /// Runtime components used when running WebAssembly programs. @@ -144,30 +141,6 @@ pub async fn load_module( Ok(module) } -#[derive(Debug, Default)] -pub struct DefaultTty { - state: Mutex, -} - -impl TtyBridge for DefaultTty { - fn reset(&self) { - let mut state = self.state.lock().unwrap(); - state.echo = false; - state.line_buffered = false; - state.line_feeds = false - } - - fn tty_get(&self) -> WasiTtyState { - let state = self.state.lock().unwrap(); - state.clone() - } - - fn tty_set(&self, tty_state: WasiTtyState) { - let mut state = self.state.lock().unwrap(); - *state = tty_state; - } -} - #[derive(Clone, Derivative)] #[derivative(Debug)] pub struct PluggableRuntime { diff --git a/lib/wasix/src/os/tty/tty_sys.rs b/lib/wasix/src/runtime/tty_sys.rs similarity index 86% rename from lib/wasix/src/os/tty/tty_sys.rs rename to lib/wasix/src/runtime/tty_sys.rs index fbbf9cc00a6..254f6a6cb8a 100644 --- a/lib/wasix/src/os/tty/tty_sys.rs +++ b/lib/wasix/src/runtime/tty_sys.rs @@ -1,5 +1,48 @@ -use super::TtyBridge; -use crate::WasiTtyState; +/// A module to handle the Tty state (via [`TtyState`]) and +/// bridge it to the host system via the [`TtyBridge`] trait. + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct TtyState { + pub cols: u32, + pub rows: u32, + pub width: u32, + pub height: u32, + pub stdin_tty: bool, + pub stdout_tty: bool, + pub stderr_tty: bool, + pub echo: bool, + pub line_buffered: bool, + pub line_feeds: bool, +} + +impl Default for TtyState { + fn default() -> Self { + Self { + rows: 80, + cols: 25, + width: 800, + height: 600, + stdin_tty: true, + stdout_tty: true, + stderr_tty: true, + echo: false, + line_buffered: false, + line_feeds: true, + } + } +} + +/// Provides access to a TTY. +pub trait TtyBridge { + /// Resets the values + fn reset(&self); + + /// Retrieve the current TTY state. + fn tty_get(&self) -> TtyState; + + /// Set the TTY state. + fn tty_set(&self, _tty_state: TtyState); +} /// [`TtyBridge`] implementation for Unix systems. #[derive(Debug, Default, Clone)] @@ -10,7 +53,7 @@ impl TtyBridge for SysTty { sys::reset().ok(); } - fn tty_get(&self) -> WasiTtyState { + fn tty_get(&self) -> TtyState { let echo = sys::is_mode_echo(); let line_buffered = sys::is_mode_line_buffering(); let line_feeds = sys::is_mode_line_feeds(); @@ -19,7 +62,7 @@ impl TtyBridge for SysTty { let stderr_tty = sys::is_stderr_tty(); if let Some((w, h)) = term_size::dimensions() { - WasiTtyState { + TtyState { cols: w as u32, rows: h as u32, width: 800, @@ -32,7 +75,7 @@ impl TtyBridge for SysTty { line_feeds, } } else { - WasiTtyState { + TtyState { rows: 80, cols: 25, width: 800, @@ -47,7 +90,7 @@ impl TtyBridge for SysTty { } } - fn tty_set(&self, tty_state: WasiTtyState) { + fn tty_set(&self, tty_state: TtyState) { if tty_state.echo { sys::set_mode_echo().ok(); } else { diff --git a/lib/wasix/src/syscalls/wasix/tty_set.rs b/lib/wasix/src/syscalls/wasix/tty_set.rs index 0f46e1e25ab..9c216689854 100644 --- a/lib/wasix/src/syscalls/wasix/tty_set.rs +++ b/lib/wasix/src/syscalls/wasix/tty_set.rs @@ -26,7 +26,7 @@ pub fn tty_set( %line_feeds ); - let state = crate::os::tty::WasiTtyState { + let state = crate::runtime::TtyState { cols: state.cols, rows: state.rows, width: state.width, From ae666a32fcdc49ea16cf325e589d44d1346de520 Mon Sep 17 00:00:00 2001 From: Syrus Akbary Date: Sat, 11 Nov 2023 23:38:06 -0800 Subject: [PATCH 2/5] Moved BinaryPackage and Runner to runtime --- lib/wasix/src/bin_factory/mod.rs | 7 ++----- lib/wasix/src/runners/mod.rs | 5 ++--- .../src/{bin_factory => runtime}/binary_package.rs | 0 lib/wasix/src/runtime/mod.rs | 11 +++++++---- lib/wasix/src/{runners => runtime}/runner.rs | 0 5 files changed, 11 insertions(+), 12 deletions(-) rename lib/wasix/src/{bin_factory => runtime}/binary_package.rs (100%) rename lib/wasix/src/{runners => runtime}/runner.rs (100%) diff --git a/lib/wasix/src/bin_factory/mod.rs b/lib/wasix/src/bin_factory/mod.rs index 4fba6d838d6..cba54f338ce 100644 --- a/lib/wasix/src/bin_factory/mod.rs +++ b/lib/wasix/src/bin_factory/mod.rs @@ -9,13 +9,10 @@ use anyhow::Context; use virtual_fs::{AsyncReadExt, FileSystem}; use webc::Container; -mod binary_package; mod exec; -pub use self::{ - binary_package::*, - exec::{spawn_exec, spawn_exec_module}, -}; +pub use self::exec::{spawn_exec, spawn_exec_module}; +pub use crate::runtime::binary_package::{BinaryPackage, BinaryPackageCommand}; use crate::{os::command::Commands, Runtime}; #[derive(Debug, Clone)] diff --git a/lib/wasix/src/runners/mod.rs b/lib/wasix/src/runners/mod.rs index ed9fa74f006..2ac34409ba6 100644 --- a/lib/wasix/src/runners/mod.rs +++ b/lib/wasix/src/runners/mod.rs @@ -1,5 +1,3 @@ -mod runner; - #[cfg(feature = "webc_runner_rt_emscripten")] pub mod emscripten; pub mod wasi; @@ -7,7 +5,8 @@ mod wasi_common; #[cfg(feature = "webc_runner_rt_wcgi")] pub mod wcgi; -pub use self::{runner::Runner, wasi_common::MappedCommand}; +pub use self::wasi_common::MappedCommand; +pub use crate::runtime::runner::Runner; /// A directory that should be mapped from the host filesystem into a WASI /// instance (the "guest"). diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/runtime/binary_package.rs similarity index 100% rename from lib/wasix/src/bin_factory/binary_package.rs rename to lib/wasix/src/runtime/binary_package.rs diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index 692e41e0d8f..656ba74abc5 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -1,15 +1,18 @@ +pub mod binary_package; pub mod module_cache; pub mod package_loader; pub mod resolver; +pub mod runner; pub mod task_manager; pub mod tty_sys; +pub use self::binary_package::{BinaryPackage, BinaryPackageCommand}; +pub use self::runner::Runner; pub use self::task_manager::{SpawnMemoryType, VirtualTaskManager}; pub use self::tty_sys::{SysTty, TtyBridge, TtyState}; -use self::{ - module_cache::{CacheError, ModuleHash}, - task_manager::InlineWaker, -}; + +use self::module_cache::{CacheError, ModuleHash}; +use self::task_manager::InlineWaker; use std::{fmt, sync::Arc}; diff --git a/lib/wasix/src/runners/runner.rs b/lib/wasix/src/runtime/runner.rs similarity index 100% rename from lib/wasix/src/runners/runner.rs rename to lib/wasix/src/runtime/runner.rs From a89dea6f4dbc7d3d5fda689df2fbeafa9db9c78e Mon Sep 17 00:00:00 2001 From: Syrus Akbary Date: Sun, 12 Nov 2023 00:35:02 -0800 Subject: [PATCH 3/5] Moved things into wasmer-runner --- Cargo.lock | 42 +++++++ Cargo.toml | 1 + lib/runner/Cargo.toml | 109 ++++++++++++++++++ lib/runner/README.md | 7 ++ .../runtime => runner/src}/binary_package.rs | 15 +-- lib/{wasix => runner}/src/bindings/mod.rs | 0 .../src/bindings/wasix_http_client_v1.rs | 0 lib/{wasix => runner}/src/capabilities.rs | 0 lib/{wasix => runner}/src/http/client.rs | 0 lib/{wasix => runner}/src/http/client_impl.rs | 2 +- lib/{wasix => runner}/src/http/mod.rs | 0 lib/{wasix => runner}/src/http/reqwest.rs | 0 .../src/http/web_http_client.rs | 0 .../src/runtime/mod.rs => runner/src/lib.rs} | 12 +- .../src}/module_cache/fallback.rs | 4 +- .../src}/module_cache/filesystem.rs | 2 +- .../src}/module_cache/mod.rs | 0 .../src}/module_cache/shared.rs | 2 +- .../src}/module_cache/thread_local.rs | 2 +- .../src}/module_cache/types.rs | 4 +- .../src}/package_loader/builtin_loader.rs | 16 ++- .../src}/package_loader/load_package_tree.rs | 14 +-- .../src}/package_loader/mod.rs | 0 .../src}/package_loader/types.rs | 4 +- .../src}/package_loader/unsupported.rs | 8 +- .../src}/resolver/filesystem_source.rs | 4 +- .../src}/resolver/in_memory_source.rs | 20 ++-- .../runtime => runner/src}/resolver/inputs.rs | 10 +- .../runtime => runner/src}/resolver/mod.rs | 0 .../src}/resolver/multi_source.rs | 4 +- .../src}/resolver/outputs.rs | 2 +- .../src}/resolver/resolve.rs | 13 +-- .../runtime => runner/src}/resolver/source.rs | 4 +- .../runtime => runner/src}/resolver/utils.rs | 0 .../src}/resolver/wapm_source.rs | 6 +- .../src}/resolver/web_source.rs | 6 +- .../src/runtime => runner/src}/runner.rs | 2 +- lib/runner/src/snapshot.rs | 30 +++++ .../src}/task_manager/mod.rs | 8 +- .../src}/task_manager/thread.rs | 0 .../src}/task_manager/tokio.rs | 0 .../src/runtime => runner/src}/tty_sys.rs | 0 lib/wasix/src/lib.rs | 9 +- lib/wasix/src/utils/store.rs | 31 +---- 44 files changed, 269 insertions(+), 124 deletions(-) create mode 100644 lib/runner/Cargo.toml create mode 100644 lib/runner/README.md rename lib/{wasix/src/runtime => runner/src}/binary_package.rs (91%) rename lib/{wasix => runner}/src/bindings/mod.rs (100%) rename lib/{wasix => runner}/src/bindings/wasix_http_client_v1.rs (100%) rename lib/{wasix => runner}/src/capabilities.rs (100%) rename lib/{wasix => runner}/src/http/client.rs (100%) rename lib/{wasix => runner}/src/http/client_impl.rs (99%) rename lib/{wasix => runner}/src/http/mod.rs (100%) rename lib/{wasix => runner}/src/http/reqwest.rs (100%) rename lib/{wasix => runner}/src/http/web_http_client.rs (100%) rename lib/{wasix/src/runtime/mod.rs => runner/src/lib.rs} (97%) rename lib/{wasix/src/runtime => runner/src}/module_cache/fallback.rs (98%) rename lib/{wasix/src/runtime => runner/src}/module_cache/filesystem.rs (99%) rename lib/{wasix/src/runtime => runner/src}/module_cache/mod.rs (100%) rename lib/{wasix/src/runtime => runner/src}/module_cache/shared.rs (96%) rename lib/{wasix/src/runtime => runner/src}/module_cache/thread_local.rs (97%) rename lib/{wasix/src/runtime => runner/src}/module_cache/types.rs (98%) rename lib/{wasix/src/runtime => runner/src}/package_loader/builtin_loader.rs (96%) rename lib/{wasix/src/runtime => runner/src}/package_loader/load_package_tree.rs (97%) rename lib/{wasix/src/runtime => runner/src}/package_loader/mod.rs (100%) rename lib/{wasix/src/runtime => runner/src}/package_loader/types.rs (92%) rename lib/{wasix/src/runtime => runner/src}/package_loader/unsupported.rs (84%) rename lib/{wasix/src/runtime => runner/src}/resolver/filesystem_source.rs (95%) rename lib/{wasix/src/runtime => runner/src}/resolver/in_memory_source.rs (87%) rename lib/{wasix/src/runtime => runner/src}/resolver/inputs.rs (97%) rename lib/{wasix/src/runtime => runner/src}/resolver/mod.rs (100%) rename lib/{wasix/src/runtime => runner/src}/resolver/multi_source.rs (96%) rename lib/{wasix/src/runtime => runner/src}/resolver/outputs.rs (98%) rename lib/{wasix/src/runtime => runner/src}/resolver/resolve.rs (99%) rename lib/{wasix/src/runtime => runner/src}/resolver/source.rs (96%) rename lib/{wasix/src/runtime => runner/src}/resolver/utils.rs (100%) rename lib/{wasix/src/runtime => runner/src}/resolver/wapm_source.rs (99%) rename lib/{wasix/src/runtime => runner/src}/resolver/web_source.rs (98%) rename lib/{wasix/src/runtime => runner/src}/runner.rs (90%) create mode 100644 lib/runner/src/snapshot.rs rename lib/{wasix/src/runtime => runner/src}/task_manager/mod.rs (98%) rename lib/{wasix/src/runtime => runner/src}/task_manager/thread.rs (100%) rename lib/{wasix/src/runtime => runner/src}/task_manager/tokio.rs (100%) rename lib/{wasix/src/runtime => runner/src}/tty_sys.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index cb1a8594715..84e075e455c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6113,6 +6113,47 @@ dependencies = [ "whoami", ] +[[package]] +name = "wasmer-runner" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "bincode", + "bytes", + "cfg-if", + "dashmap", + "derivative", + "futures", + "hex", + "http", + "libc", + "once_cell", + "petgraph", + "semver 1.0.20", + "serde", + "serde_cbor", + "serde_json", + "sha2", + "tempfile", + "term_size", + "termios", + "thiserror", + "tokio", + "tracing", + "url", + "virtual-fs", + "virtual-mio 0.3.0", + "virtual-net 0.6.1", + "wai-bindgen-wasmer", + "wasm-bindgen", + "wasmer", + "wasmer-types", + "wasmer-wasix-types", + "webc", + "weezl", +] + [[package]] name = "wasmer-sys-utils" version = "0.16.0" @@ -6385,6 +6426,7 @@ dependencies = [ "wasmer-compiler-singlepass", "wasmer-emscripten", "wasmer-middlewares", + "wasmer-runner", "wasmer-types", "wasmer-wasix", "wasmer-wast", diff --git a/Cargo.toml b/Cargo.toml index 0508e5de946..871c5db951b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ wasi-test-generator = { version = "=4.2.3", path = "tests/wasi-wast", optional = wasmer-cache = { version = "=4.2.3", path = "lib/cache", optional = true } wasmer-types = { version = "=4.2.3", path = "lib/types" } wasmer-middlewares = { version = "=4.2.3", path = "lib/middlewares", optional = true } +wasmer-runner = { version = "=0.1.0", path = "lib/runner", optional = true } cfg-if = "1.0" tokio = { version = "1", features = [ "rt", diff --git a/lib/runner/Cargo.toml b/lib/runner/Cargo.toml new file mode 100644 index 00000000000..16298510fb0 --- /dev/null +++ b/lib/runner/Cargo.toml @@ -0,0 +1,109 @@ +[package] +name = "wasmer-runner" +version = "0.1.0" +description = "Wasmer Runner allows plugging in different implementations." +categories = ["wasm", "os"] +keywords = ["wasm", "webassembly", "wasi", "sandbox", "ABI"] +readme = "README.md" +authors.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +cfg-if = "1.0" +thiserror = "1" +tracing = { version = "0.1.37" } +# getrandom = "0.2" +wasmer-wasix-types = { path = "../wasi-types", version = "0.16.0", features = [ "enable-serde" ] } +wasmer-types = { path = "../types", version = "=4.2.3", default-features = false } +wasmer = { path = "../api", version = "=4.2.3", default-features = true } +webc.workspace = true +virtual-mio = { path = "../virtual-io", version = "0.3.0", default-features = false } +virtual-fs = { path = "../virtual-fs", version = "0.9.0", default-features = false, features = ["webc-fs"] } +virtual-net = { path = "../virtual-net", version = "0.6.1", default-features = false } +# wasmer-emscripten = { path = "../emscripten", version = "=4.2.3", optional = true } +# typetag = { version = "0.1", optional = true } +serde = { version = "1.0", default-features = false, features = ["derive"] } +bincode = { version = "1.3" } +# chrono = { version = "^0.4.31", default-features = false, features = [ "wasmbind", "std", "clock" ], optional = true } +derivative = { version = "^2" } +bytes = "1" +# webc = { workspace = true } +serde_cbor = { version = "0.11.2" } +anyhow = { version = "1.0.66" } +# lazy_static = "1.4" +sha2 = { version = "0.10" } +# waker-fn = { version = "1.1" } +# cooked-waker = "^5" +# rand = "0.8" +tokio = { version = "1", features = ["sync", "macros", "time", "rt", "rt-multi-thread"], default_features = false } +futures = { version = "0.3" } +# # used by feature='os' +async-trait = { version = "^0.1" } +# urlencoding = { version = "^2" } +# serde_derive = { version = "^1" } +serde_json = { version = "^1" } +# serde_yaml = { version = "^0.8" } +weezl = { version = "^0.1" } +hex = { version = "^0.4" } +term_size = { version = "0.3" } +# linked_hash_set = { version = "0.1" } +http = "0.2.8" +wai-bindgen-wasmer = { path = "../wai-bindgen-wasmer", version = "0.16.0", features = ["tracing"] } +# heapless = "0.7.16" +once_cell = "1.17.0" +# pin-project = "1.0.12" +semver = "1.0.17" +dashmap = "5.4.0" +tempfile = "3.6.0" +# # Used by the WCGI runner +# hyper = { version = "0.14", features = ["server", "stream"], optional = true } +# wcgi = { version = "0.1.2", optional = true } +# wcgi-host = { version = "0.1.2", optional = true } +# tower-http = { version = "0.4.0", features = ["trace", "util", "catch-panic", "cors"], optional = true } +# tower = { version = "0.4.13", features = ["make", "util"], optional = true } +url = "2.3.1" +petgraph = "0.6.3" +# rayon = { version = "1.7.0", optional = true } +wasm-bindgen = { version = "0.2.87", optional = true } +# js-sys = { version = "0.3.64", optional = true } +# wasm-bindgen-futures = { version = "0.4.37", optional = true } +# web-sys = { version = "0.3.64", features = ["Request", "RequestInit", "Window", "WorkerGlobalScope", "RequestMode", "Response", "Headers"], optional = true } + +# [target.'cfg(not(target_arch = "riscv64"))'.dependencies.reqwest] +# version = "0.11" +# default-features = false +# features = ["rustls-tls", "json"] +# optional = true + +# [target.'cfg(target_arch = "riscv64")'.dependencies.reqwest] +# version = "0.11" +# default-features = false +# features = ["native-tls", "json"] +# optional = true + +[target.'cfg(unix)'.dependencies] +libc = { version = "^0.2", default-features = false } + +[target.'cfg(all(unix, not(target_os="ios")))'.dependencies] +termios = { version = "0.3" } + +# [target.'cfg(windows)'.dependencies] +# winapi = "0.3" + +# [dev-dependencies] +# wasmer = { path = "../api", version = "=4.2.3", default-features = false, features = ["wat", "js-serializable-module"] } +# tokio = { version = "1", features = [ "sync", "macros", "rt" ], default_features = false } +# pretty_assertions = "1.3.0" +# wasm-bindgen-test = "0.3.0" + +# [target.'cfg(target_arch = "wasm32")'.dev-dependencies] +# wasm-bindgen-test = "0.3.0" +# tracing-wasm = "0.2" + +# [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +# tracing-subscriber = { version = "^0.3" } +# wasmer = { path = "../api", version = "=4.2.3", default-features = false, features = ["wat", "js-serializable-module", "cranelift"] } diff --git a/lib/runner/README.md b/lib/runner/README.md new file mode 100644 index 00000000000..ddf05cde5f3 --- /dev/null +++ b/lib/runner/README.md @@ -0,0 +1,7 @@ +# `wasmer-runner` [![Build Status](https://github.com/wasmerio/wasmer/workflows/build/badge.svg?style=flat-square)](https://github.com/wasmerio/wasmer/actions?query=workflow%3Abuild) [![Join Wasmer Slack](https://img.shields.io/static/v1?label=Slack&message=join%20chat&color=brighgreen&style=flat-square)](https://slack.wasmer.io) [![MIT License](https://img.shields.io/github/license/wasmerio/wasmer.svg?style=flat-square)](https://github.com/wasmerio/wasmer/blob/master/LICENSE) + +This library provides the proper abstractions to manage and run Wasmer packages anywhere. + +There are different libraries that use it to implement their own runners: +* `wasmer-emscripten` +* `wasmer-wasix` diff --git a/lib/wasix/src/runtime/binary_package.rs b/lib/runner/src/binary_package.rs similarity index 91% rename from lib/wasix/src/runtime/binary_package.rs rename to lib/runner/src/binary_package.rs index cc785a78d98..5df3cd619e7 100644 --- a/lib/wasix/src/runtime/binary_package.rs +++ b/lib/runner/src/binary_package.rs @@ -8,10 +8,8 @@ use virtual_fs::FileSystem; use webc::{compat::SharedBytes, Container}; use crate::{ - runtime::{ - module_cache::ModuleHash, - resolver::{PackageId, PackageInfo, PackageSpecifier, ResolveError}, - }, + module_cache::ModuleHash, + resolver::{PackageId, PackageInfo, PackageSpecifier, ResolveError}, Runtime, }; @@ -89,7 +87,7 @@ impl BinaryPackage { version: root.version.clone(), }; - let resolution = crate::runtime::resolver::resolve(&root_id, &root, &*source).await?; + let resolution = crate::resolver::resolve(&root_id, &root, &*source).await?; let pkg = rt .package_loader() .load_package_tree(container, &resolution) @@ -117,7 +115,7 @@ impl BinaryPackage { let root = runtime.package_loader().load(&root_summary).await?; let id = root_summary.package_id(); - let resolution = crate::runtime::resolver::resolve(&id, &root_summary.pkg, &source) + let resolution = crate::resolver::resolve(&id, &root_summary.pkg, &source) .await .context("Dependency resolution failed")?; let pkg = runtime @@ -158,8 +156,7 @@ mod tests { use virtual_fs::AsyncReadExt; use crate::{ - runtime::{package_loader::BuiltinPackageLoader, task_manager::VirtualTaskManager}, - PluggableRuntime, + package_loader::BuiltinPackageLoader, task_manager::VirtualTaskManager, PluggableRuntime, }; use super::*; @@ -167,7 +164,7 @@ mod tests { fn task_manager() -> Arc { cfg_if::cfg_if! { if #[cfg(feature = "sys-thread")] { - Arc::new(crate::runtime::task_manager::tokio::TokioTaskManager::new(tokio::runtime::Handle::current())) + Arc::new(crate::task_manager::tokio::TokioTaskManager::new(tokio::runtime::Handle::current())) } else { unimplemented!("Unable to get the task manager") } diff --git a/lib/wasix/src/bindings/mod.rs b/lib/runner/src/bindings/mod.rs similarity index 100% rename from lib/wasix/src/bindings/mod.rs rename to lib/runner/src/bindings/mod.rs diff --git a/lib/wasix/src/bindings/wasix_http_client_v1.rs b/lib/runner/src/bindings/wasix_http_client_v1.rs similarity index 100% rename from lib/wasix/src/bindings/wasix_http_client_v1.rs rename to lib/runner/src/bindings/wasix_http_client_v1.rs diff --git a/lib/wasix/src/capabilities.rs b/lib/runner/src/capabilities.rs similarity index 100% rename from lib/wasix/src/capabilities.rs rename to lib/runner/src/capabilities.rs diff --git a/lib/wasix/src/http/client.rs b/lib/runner/src/http/client.rs similarity index 100% rename from lib/wasix/src/http/client.rs rename to lib/runner/src/http/client.rs diff --git a/lib/wasix/src/http/client_impl.rs b/lib/runner/src/http/client_impl.rs similarity index 99% rename from lib/wasix/src/http/client_impl.rs rename to lib/runner/src/http/client_impl.rs index 97d23dbd8e4..4bbb2fb434c 100644 --- a/lib/wasix/src/http/client_impl.rs +++ b/lib/runner/src/http/client_impl.rs @@ -7,7 +7,7 @@ use crate::{ bindings::wasix_http_client_v1 as sys, capabilities::Capabilities, http::{DynHttpClient, HttpClientCapabilityV1}, - runtime::task_manager::InlineWaker, + task_manager::InlineWaker, Runtime, WasiEnv, }; diff --git a/lib/wasix/src/http/mod.rs b/lib/runner/src/http/mod.rs similarity index 100% rename from lib/wasix/src/http/mod.rs rename to lib/runner/src/http/mod.rs diff --git a/lib/wasix/src/http/reqwest.rs b/lib/runner/src/http/reqwest.rs similarity index 100% rename from lib/wasix/src/http/reqwest.rs rename to lib/runner/src/http/reqwest.rs diff --git a/lib/wasix/src/http/web_http_client.rs b/lib/runner/src/http/web_http_client.rs similarity index 100% rename from lib/wasix/src/http/web_http_client.rs rename to lib/runner/src/http/web_http_client.rs diff --git a/lib/wasix/src/runtime/mod.rs b/lib/runner/src/lib.rs similarity index 97% rename from lib/wasix/src/runtime/mod.rs rename to lib/runner/src/lib.rs index 656ba74abc5..ee389584ff0 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/runner/src/lib.rs @@ -1,8 +1,12 @@ pub mod binary_package; +pub mod bindings; +pub mod capabilities; +pub mod http; pub mod module_cache; pub mod package_loader; pub mod resolver; pub mod runner; +pub mod snapshot; pub mod task_manager; pub mod tty_sys; @@ -23,11 +27,9 @@ use wasmer::Module; use crate::{ http::{DynHttpClient, HttpClient}, - runtime::{ - module_cache::{ModuleCache, ThreadLocalCache}, - package_loader::{PackageLoader, UnsupportedPackageLoader}, - resolver::{MultiSource, Source, WapmSource}, - }, + module_cache::{ModuleCache, ThreadLocalCache}, + package_loader::{PackageLoader, UnsupportedPackageLoader}, + resolver::{MultiSource, Source, WapmSource}, }; /// Runtime components used when running WebAssembly programs. diff --git a/lib/wasix/src/runtime/module_cache/fallback.rs b/lib/runner/src/module_cache/fallback.rs similarity index 98% rename from lib/wasix/src/runtime/module_cache/fallback.rs rename to lib/runner/src/module_cache/fallback.rs index 3c1ffa28fa3..f280be8bd6a 100644 --- a/lib/wasix/src/runtime/module_cache/fallback.rs +++ b/lib/runner/src/module_cache/fallback.rs @@ -1,6 +1,6 @@ use wasmer::{Engine, Module}; -use crate::runtime::module_cache::{CacheError, ModuleCache, ModuleHash}; +use crate::module_cache::{CacheError, ModuleCache, ModuleHash}; /// [`FallbackCache`] is a combinator for the [`ModuleCache`] trait that enables /// the chaining of two caching strategies together, typically via @@ -104,7 +104,7 @@ mod tests { use std::sync::atomic::{AtomicUsize, Ordering}; use super::*; - use crate::runtime::module_cache::SharedCache; + use crate::module_cache::SharedCache; const ADD_WAT: &[u8] = br#"( module diff --git a/lib/wasix/src/runtime/module_cache/filesystem.rs b/lib/runner/src/module_cache/filesystem.rs similarity index 99% rename from lib/wasix/src/runtime/module_cache/filesystem.rs rename to lib/runner/src/module_cache/filesystem.rs index 7ac89de9584..a0eaf0b2e61 100644 --- a/lib/wasix/src/runtime/module_cache/filesystem.rs +++ b/lib/runner/src/module_cache/filesystem.rs @@ -6,7 +6,7 @@ use std::{ use tempfile::NamedTempFile; use wasmer::{Engine, Module}; -use crate::runtime::module_cache::{CacheError, ModuleCache, ModuleHash}; +use crate::module_cache::{CacheError, ModuleCache, ModuleHash}; /// A cache that saves modules to a folder on the host filesystem using /// [`Module::serialize()`]. diff --git a/lib/wasix/src/runtime/module_cache/mod.rs b/lib/runner/src/module_cache/mod.rs similarity index 100% rename from lib/wasix/src/runtime/module_cache/mod.rs rename to lib/runner/src/module_cache/mod.rs diff --git a/lib/wasix/src/runtime/module_cache/shared.rs b/lib/runner/src/module_cache/shared.rs similarity index 96% rename from lib/wasix/src/runtime/module_cache/shared.rs rename to lib/runner/src/module_cache/shared.rs index 012faf36983..6c22fdd7847 100644 --- a/lib/wasix/src/runtime/module_cache/shared.rs +++ b/lib/runner/src/module_cache/shared.rs @@ -1,7 +1,7 @@ use dashmap::DashMap; use wasmer::{Engine, Module}; -use crate::runtime::module_cache::{CacheError, ModuleCache, ModuleHash}; +use crate::module_cache::{CacheError, ModuleCache, ModuleHash}; /// A [`ModuleCache`] based on a [DashMap]<[ModuleHash], [Module]>. #[derive(Debug, Default, Clone)] diff --git a/lib/wasix/src/runtime/module_cache/thread_local.rs b/lib/runner/src/module_cache/thread_local.rs similarity index 97% rename from lib/wasix/src/runtime/module_cache/thread_local.rs rename to lib/runner/src/module_cache/thread_local.rs index 5c04037028b..c4f025c2c26 100644 --- a/lib/wasix/src/runtime/module_cache/thread_local.rs +++ b/lib/runner/src/module_cache/thread_local.rs @@ -2,7 +2,7 @@ use std::{cell::RefCell, collections::HashMap}; use wasmer::{Engine, Module}; -use crate::runtime::module_cache::{CacheError, ModuleCache, ModuleHash}; +use crate::module_cache::{CacheError, ModuleCache, ModuleHash}; std::thread_local! { static CACHED_MODULES: RefCell> diff --git a/lib/wasix/src/runtime/module_cache/types.rs b/lib/runner/src/module_cache/types.rs similarity index 98% rename from lib/wasix/src/runtime/module_cache/types.rs rename to lib/runner/src/module_cache/types.rs index 9d801007c12..e35554ee246 100644 --- a/lib/wasix/src/runtime/module_cache/types.rs +++ b/lib/runner/src/module_cache/types.rs @@ -7,7 +7,7 @@ use std::{ use sha2::{Digest, Sha256}; use wasmer::{Engine, Module}; -use crate::runtime::module_cache::FallbackCache; +use crate::module_cache::FallbackCache; /// A cache for compiled WebAssembly modules. /// @@ -52,7 +52,7 @@ pub trait ModuleCache: Debug { /// be significantly slower than the previous one. /// /// ```rust - /// use wasmer_wasix::runtime::module_cache::{ + /// use wasmer_runner::module_cache::{ /// ModuleCache, ThreadLocalCache, FileSystemCache, SharedCache, /// }; /// diff --git a/lib/wasix/src/runtime/package_loader/builtin_loader.rs b/lib/runner/src/package_loader/builtin_loader.rs similarity index 96% rename from lib/wasix/src/runtime/package_loader/builtin_loader.rs rename to lib/runner/src/package_loader/builtin_loader.rs index ecdc7aae620..5962ad27102 100644 --- a/lib/wasix/src/runtime/package_loader/builtin_loader.rs +++ b/lib/runner/src/package_loader/builtin_loader.rs @@ -16,12 +16,10 @@ use webc::{ }; use crate::{ - bin_factory::BinaryPackage, + binary_package::BinaryPackage, http::{HttpClient, HttpRequest, USER_AGENT}, - runtime::{ - package_loader::PackageLoader, - resolver::{DistributionInfo, PackageSummary, Resolution, WebcHash}, - }, + package_loader::PackageLoader, + resolver::{DistributionInfo, PackageSummary, Resolution, WebcHash}, }; /// The builtin [`PackageLoader`] that is used by the `wasmer` CLI and @@ -99,7 +97,7 @@ impl BuiltinPackageLoader { #[tracing::instrument(level = "debug", skip_all, fields(%dist.webc, %dist.webc_sha256))] async fn download(&self, dist: &DistributionInfo) -> Result { if dist.webc.scheme() == "file" { - match crate::runtime::resolver::utils::file_path_from_url(&dist.webc) { + match crate::resolver::utils::file_path_from_url(&dist.webc) { Ok(path) => { // FIXME: This will block the thread let bytes = std::fs::read(&path) @@ -139,7 +137,7 @@ impl BuiltinPackageLoader { if !response.is_ok() { let url = &dist.webc; - return Err(crate::runtime::resolver::utils::http_error(&response) + return Err(crate::resolver::utils::http_error(&response) .context(format!("The GET request to \"{url}\" failed"))); } @@ -347,12 +345,12 @@ mod tests { use crate::{ http::{HttpRequest, HttpResponse}, - runtime::resolver::PackageInfo, + resolver::PackageInfo, }; use super::*; - const PYTHON: &[u8] = include_bytes!("../../../../c-api/examples/assets/python-0.1.0.wasmer"); + const PYTHON: &[u8] = include_bytes!("../../../c-api/examples/assets/python-0.1.0.wasmer"); #[derive(Debug)] pub(crate) struct DummyClient { diff --git a/lib/wasix/src/runtime/package_loader/load_package_tree.rs b/lib/runner/src/package_loader/load_package_tree.rs similarity index 97% rename from lib/wasix/src/runtime/package_loader/load_package_tree.rs rename to lib/runner/src/package_loader/load_package_tree.rs index b92337c8845..c647ebd04d1 100644 --- a/lib/wasix/src/runtime/package_loader/load_package_tree.rs +++ b/lib/runner/src/package_loader/load_package_tree.rs @@ -16,13 +16,11 @@ use webc::{ }; use crate::{ - bin_factory::{BinaryPackage, BinaryPackageCommand}, - runtime::{ - package_loader::PackageLoader, - resolver::{ - DependencyGraph, ItemLocation, PackageId, PackageSummary, Resolution, - ResolvedFileSystemMapping, ResolvedPackage, - }, + binary_package::{BinaryPackage, BinaryPackageCommand}, + package_loader::PackageLoader, + resolver::{ + DependencyGraph, ItemLocation, PackageId, PackageSummary, Resolution, + ResolvedFileSystemMapping, ResolvedPackage, }, }; @@ -244,7 +242,7 @@ async fn fetch_dependencies( packages.remove(&pkg.root_package); let packages = packages.into_iter().filter_map(|id| { - let crate::runtime::resolver::Node { pkg, dist, .. } = &graph[&id]; + let crate::resolver::Node { pkg, dist, .. } = &graph[&id]; let summary = PackageSummary { pkg: pkg.clone(), dist: dist.clone()?, diff --git a/lib/wasix/src/runtime/package_loader/mod.rs b/lib/runner/src/package_loader/mod.rs similarity index 100% rename from lib/wasix/src/runtime/package_loader/mod.rs rename to lib/runner/src/package_loader/mod.rs diff --git a/lib/wasix/src/runtime/package_loader/types.rs b/lib/runner/src/package_loader/types.rs similarity index 92% rename from lib/wasix/src/runtime/package_loader/types.rs rename to lib/runner/src/package_loader/types.rs index afb3e924828..811e7b321db 100644 --- a/lib/wasix/src/runtime/package_loader/types.rs +++ b/lib/runner/src/package_loader/types.rs @@ -4,8 +4,8 @@ use anyhow::Error; use webc::compat::Container; use crate::{ - bin_factory::BinaryPackage, - runtime::resolver::{PackageSummary, Resolution}, + binary_package::BinaryPackage, + resolver::{PackageSummary, Resolution}, }; #[async_trait::async_trait] diff --git a/lib/wasix/src/runtime/package_loader/unsupported.rs b/lib/runner/src/package_loader/unsupported.rs similarity index 84% rename from lib/wasix/src/runtime/package_loader/unsupported.rs rename to lib/runner/src/package_loader/unsupported.rs index be5488b1edb..7298d1393e4 100644 --- a/lib/wasix/src/runtime/package_loader/unsupported.rs +++ b/lib/runner/src/package_loader/unsupported.rs @@ -2,11 +2,9 @@ use anyhow::Error; use webc::compat::Container; use crate::{ - bin_factory::BinaryPackage, - runtime::{ - package_loader::PackageLoader, - resolver::{PackageSummary, Resolution}, - }, + binary_package::BinaryPackage, + package_loader::PackageLoader, + resolver::{PackageSummary, Resolution}, }; /// A [`PackageLoader`] implementation which will always error out. diff --git a/lib/wasix/src/runtime/resolver/filesystem_source.rs b/lib/runner/src/resolver/filesystem_source.rs similarity index 95% rename from lib/wasix/src/runtime/resolver/filesystem_source.rs rename to lib/runner/src/resolver/filesystem_source.rs index c6c0f44d522..d6513f07b33 100644 --- a/lib/wasix/src/runtime/resolver/filesystem_source.rs +++ b/lib/runner/src/resolver/filesystem_source.rs @@ -1,7 +1,7 @@ use anyhow::Context; use webc::compat::Container; -use crate::runtime::resolver::{ +use crate::resolver::{ DistributionInfo, PackageInfo, PackageSpecifier, PackageSummary, QueryError, Source, WebcHash, }; @@ -36,7 +36,7 @@ impl Source for FileSystemSource { let container = tokio::task::block_in_place(|| Container::from_disk(&path)) .with_context(|| format!("Unable to parse \"{}\"", path.display()))?; - let url = crate::runtime::resolver::utils::url_from_file_path(&path) + let url = crate::resolver::utils::url_from_file_path(&path) .ok_or_else(|| anyhow::anyhow!("Unable to turn \"{}\" into a URL", path.display()))?; let pkg = PackageInfo::from_manifest(container.manifest()) diff --git a/lib/wasix/src/runtime/resolver/in_memory_source.rs b/lib/runner/src/resolver/in_memory_source.rs similarity index 87% rename from lib/wasix/src/runtime/resolver/in_memory_source.rs rename to lib/runner/src/resolver/in_memory_source.rs index 0de44ace5c7..bcf6bee5d1f 100644 --- a/lib/wasix/src/runtime/resolver/in_memory_source.rs +++ b/lib/runner/src/resolver/in_memory_source.rs @@ -7,7 +7,7 @@ use std::{ use anyhow::{Context, Error}; use semver::Version; -use crate::runtime::resolver::{PackageSpecifier, PackageSummary, QueryError, Source}; +use crate::resolver::{PackageSpecifier, PackageSummary, QueryError, Source}; /// A [`Source`] that tracks packages in memory. /// @@ -126,17 +126,17 @@ impl Source for InMemorySource { mod tests { use tempfile::TempDir; - use crate::runtime::resolver::{ + use crate::resolver::{ inputs::{DistributionInfo, FileSystemMapping, PackageInfo}, Dependency, WebcHash, }; use super::*; - const PYTHON: &[u8] = include_bytes!("../../../../c-api/examples/assets/python-0.1.0.wasmer"); - const COREUTILS_16: &[u8] = include_bytes!("../../../../../tests/integration/cli/tests/webc/coreutils-1.0.16-e27dbb4f-2ef2-4b44-b46a-ddd86497c6d7.webc"); - const COREUTILS_11: &[u8] = include_bytes!("../../../../../tests/integration/cli/tests/webc/coreutils-1.0.11-9d7746ca-694f-11ed-b932-dead3543c068.webc"); - const BASH: &[u8] = include_bytes!("../../../../../tests/integration/cli/tests/webc/bash-1.0.16-f097441a-a80b-4e0d-87d7-684918ef4bb6.webc"); + const PYTHON: &[u8] = include_bytes!("../../../c-api/examples/assets/python-0.1.0.wasmer"); + const COREUTILS_16: &[u8] = include_bytes!("../../../../tests/integration/cli/tests/webc/coreutils-1.0.16-e27dbb4f-2ef2-4b44-b46a-ddd86497c6d7.webc"); + const COREUTILS_11: &[u8] = include_bytes!("../../../../tests/integration/cli/tests/webc/coreutils-1.0.11-9d7746ca-694f-11ed-b932-dead3543c068.webc"); + const BASH: &[u8] = include_bytes!("../../../../tests/integration/cli/tests/webc/bash-1.0.16-f097441a-a80b-4e0d-87d7-684918ef4bb6.webc"); #[test] fn load_a_directory_tree() { @@ -170,7 +170,7 @@ mod tests { alias: "coreutils".to_string(), pkg: "sharrattj/coreutils@^1.0.16".parse().unwrap() }], - commands: vec![crate::runtime::resolver::Command { + commands: vec![crate::resolver::Command { name: "bash".to_string(), }], entrypoint: Some("bash".to_string()), @@ -182,10 +182,8 @@ mod tests { }], }, dist: DistributionInfo { - webc: crate::runtime::resolver::utils::url_from_file_path( - bash.canonicalize().unwrap() - ) - .unwrap(), + webc: crate::resolver::utils::url_from_file_path(bash.canonicalize().unwrap()) + .unwrap(), webc_sha256: WebcHash::from_bytes([ 161, 101, 23, 194, 244, 92, 186, 213, 143, 33, 200, 128, 238, 23, 185, 174, 180, 195, 144, 145, 78, 17, 227, 159, 118, 64, 83, 153, 0, 205, 253, 215, diff --git a/lib/wasix/src/runtime/resolver/inputs.rs b/lib/runner/src/resolver/inputs.rs similarity index 97% rename from lib/wasix/src/runtime/resolver/inputs.rs rename to lib/runner/src/resolver/inputs.rs index a232c46c1bb..8313d021701 100644 --- a/lib/wasix/src/runtime/resolver/inputs.rs +++ b/lib/runner/src/resolver/inputs.rs @@ -15,7 +15,7 @@ use webc::{ Container, }; -use crate::runtime::resolver::PackageId; +use crate::resolver::PackageId; /// A reference to *some* package somewhere that the user wants to run. /// @@ -25,7 +25,7 @@ use crate::runtime::resolver::PackageId; /// [`Source`][source] will eventually query. Consumers of [`PackageSpecifier`] /// should be wary of sandbox escapes. /// -/// [source]: crate::runtime::resolver::Source +/// [source]: crate::resolver::Source #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum PackageSpecifier { Registry { @@ -146,7 +146,7 @@ impl Dependency { /// Some metadata a [`Source`][source] can provide about a package without /// needing to download the entire `*.webc` file. /// -/// [source]: crate::runtime::resolver::Source +/// [source]: crate::resolver::Source #[derive(Debug, Clone, PartialEq, Eq)] pub struct PackageSummary { pub pkg: PackageInfo, @@ -162,7 +162,7 @@ impl PackageSummary { let path = path.as_ref().canonicalize()?; let container = Container::from_disk(&path)?; let webc_sha256 = WebcHash::for_file(&path)?; - let url = crate::runtime::resolver::utils::url_from_file_path(&path).ok_or_else(|| { + let url = crate::resolver::utils::url_from_file_path(&path).ok_or_else(|| { anyhow::anyhow!("Unable to turn \"{}\" into a file:// URL", path.display()) })?; @@ -213,7 +213,7 @@ impl PackageInfo { let commands = manifest .commands .iter() - .map(|(name, _value)| crate::runtime::resolver::Command { + .map(|(name, _value)| crate::resolver::Command { name: name.to_string(), }) .collect(); diff --git a/lib/wasix/src/runtime/resolver/mod.rs b/lib/runner/src/resolver/mod.rs similarity index 100% rename from lib/wasix/src/runtime/resolver/mod.rs rename to lib/runner/src/resolver/mod.rs diff --git a/lib/wasix/src/runtime/resolver/multi_source.rs b/lib/runner/src/resolver/multi_source.rs similarity index 96% rename from lib/wasix/src/runtime/resolver/multi_source.rs rename to lib/runner/src/resolver/multi_source.rs index e7782418917..4145e1b2e4a 100644 --- a/lib/wasix/src/runtime/resolver/multi_source.rs +++ b/lib/runner/src/resolver/multi_source.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use crate::runtime::resolver::{PackageSpecifier, PackageSummary, QueryError, Source}; +use crate::resolver::{PackageSpecifier, PackageSummary, QueryError, Source}; /// A [`Source`] that works by querying multiple [`Source`]s in succession. /// @@ -15,7 +15,7 @@ use crate::runtime::resolver::{PackageSpecifier, PackageSummary, QueryError, Sou /// will be treated as the canonical source for that [`Dependency`][dep] and no /// further [`Source`]s will be queried. /// -/// [dep]: crate::runtime::resolver::Dependency +/// [dep]: crate::resolver::Dependency #[derive(Debug, Clone)] pub struct MultiSource { sources: Vec>, diff --git a/lib/wasix/src/runtime/resolver/outputs.rs b/lib/runner/src/resolver/outputs.rs similarity index 98% rename from lib/wasix/src/runtime/resolver/outputs.rs rename to lib/runner/src/resolver/outputs.rs index 028fc0c14c6..74a096e0b5e 100644 --- a/lib/wasix/src/runtime/resolver/outputs.rs +++ b/lib/runner/src/resolver/outputs.rs @@ -11,7 +11,7 @@ use petgraph::{ }; use semver::Version; -use crate::runtime::resolver::{DistributionInfo, PackageInfo}; +use crate::resolver::{DistributionInfo, PackageInfo}; #[derive(Debug, Clone)] pub struct Resolution { diff --git a/lib/wasix/src/runtime/resolver/resolve.rs b/lib/runner/src/resolver/resolve.rs similarity index 99% rename from lib/wasix/src/runtime/resolver/resolve.rs rename to lib/runner/src/resolver/resolve.rs index 9407cd7ec7f..6a9d0d15d05 100644 --- a/lib/wasix/src/runtime/resolver/resolve.rs +++ b/lib/runner/src/resolver/resolve.rs @@ -9,7 +9,7 @@ use petgraph::{ }; use semver::Version; -use crate::runtime::resolver::{ +use crate::resolver::{ outputs::{Edge, Node}, DependencyGraph, ItemLocation, PackageId, PackageInfo, PackageSpecifier, PackageSummary, QueryError, Resolution, ResolvedPackage, Source, @@ -395,7 +395,7 @@ fn resolve_package(dependency_graph: &DependencyGraph) -> Result &mut Self { - self.summary - .pkg - .commands - .push(crate::runtime::resolver::Command { - name: name.to_string(), - }); + self.summary.pkg.commands.push(crate::resolver::Command { + name: name.to_string(), + }); self } diff --git a/lib/wasix/src/runtime/resolver/source.rs b/lib/runner/src/resolver/source.rs similarity index 96% rename from lib/wasix/src/runtime/resolver/source.rs rename to lib/runner/src/resolver/source.rs index de4a2584764..1eb9116ae38 100644 --- a/lib/wasix/src/runtime/resolver/source.rs +++ b/lib/runner/src/resolver/source.rs @@ -1,6 +1,6 @@ use std::fmt::{Debug, Display}; -use crate::runtime::resolver::{PackageSpecifier, PackageSummary}; +use crate::resolver::{PackageSpecifier, PackageSummary}; /// Something that packages can be downloaded from. #[async_trait::async_trait] @@ -14,7 +14,7 @@ pub trait Source: Sync + Debug { /// will be at least one [`PackageSummary`], otherwise implementations /// should return [`QueryError::NotFound`] or [`QueryError::NoMatches`]. /// - /// [dep]: crate::runtime::resolver::Dependency + /// [dep]: crate::resolver::Dependency async fn query(&self, package: &PackageSpecifier) -> Result, QueryError>; /// Run [`Source::query()`] and get the [`PackageSummary`] for the latest diff --git a/lib/wasix/src/runtime/resolver/utils.rs b/lib/runner/src/resolver/utils.rs similarity index 100% rename from lib/wasix/src/runtime/resolver/utils.rs rename to lib/runner/src/resolver/utils.rs diff --git a/lib/wasix/src/runtime/resolver/wapm_source.rs b/lib/runner/src/resolver/wapm_source.rs similarity index 99% rename from lib/wasix/src/runtime/resolver/wapm_source.rs rename to lib/runner/src/resolver/wapm_source.rs index 071d8a7df49..a6756aafcc6 100644 --- a/lib/wasix/src/runtime/resolver/wapm_source.rs +++ b/lib/runner/src/resolver/wapm_source.rs @@ -12,7 +12,7 @@ use webc::metadata::Manifest; use crate::{ http::{HttpClient, HttpRequest, USER_AGENT}, - runtime::resolver::{ + resolver::{ DistributionInfo, PackageInfo, PackageSpecifier, PackageSummary, QueryError, Source, WebcHash, }, @@ -425,7 +425,7 @@ mod tests { use crate::{ http::HttpResponse, - runtime::resolver::inputs::{DistributionInfo, FileSystemMapping, PackageInfo}, + resolver::inputs::{DistributionInfo, FileSystemMapping, PackageInfo}, }; use super::*; @@ -494,7 +494,7 @@ mod tests { version: Version::new(0, 6, 0), dependencies: Vec::new(), commands: vec![ - crate::runtime::resolver::Command { + crate::resolver::Command { name: "wasmer-pack".to_string(), }, ], diff --git a/lib/wasix/src/runtime/resolver/web_source.rs b/lib/runner/src/resolver/web_source.rs similarity index 98% rename from lib/wasix/src/runtime/resolver/web_source.rs rename to lib/runner/src/resolver/web_source.rs index dbac364402e..081d4ad6bc1 100644 --- a/lib/wasix/src/runtime/resolver/web_source.rs +++ b/lib/runner/src/resolver/web_source.rs @@ -15,7 +15,7 @@ use webc::compat::Container; use crate::{ http::{HttpClient, HttpRequest}, - runtime::resolver::{ + resolver::{ DistributionInfo, PackageInfo, PackageSpecifier, PackageSummary, QueryError, Source, WebcHash, }, @@ -387,8 +387,8 @@ mod tests { use super::*; - const PYTHON: &[u8] = include_bytes!("../../../../c-api/examples/assets/python-0.1.0.wasmer"); - const COREUTILS: &[u8] = include_bytes!("../../../../../tests/integration/cli/tests/webc/coreutils-1.0.16-e27dbb4f-2ef2-4b44-b46a-ddd86497c6d7.webc"); + const PYTHON: &[u8] = include_bytes!("../../../c-api/examples/assets/python-0.1.0.wasmer"); + const COREUTILS: &[u8] = include_bytes!("../../../../tests/integration/cli/tests/webc/coreutils-1.0.16-e27dbb4f-2ef2-4b44-b46a-ddd86497c6d7.webc"); const DUMMY_URL: &str = "http://my-registry.io/some/package"; const DUMMY_URL_HASH: &str = "4D7481F44E1D971A8C60D3C7BD505E2727602CF9369ED623920E029C2BA2351D"; diff --git a/lib/wasix/src/runtime/runner.rs b/lib/runner/src/runner.rs similarity index 90% rename from lib/wasix/src/runtime/runner.rs rename to lib/runner/src/runner.rs index 2504501f0f2..dc214e659f4 100644 --- a/lib/wasix/src/runtime/runner.rs +++ b/lib/runner/src/runner.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use anyhow::Error; use webc::metadata::Command; -use crate::{bin_factory::BinaryPackage, Runtime}; +use crate::{binary_package::BinaryPackage, Runtime}; /// Trait that all runners have to implement pub trait Runner { diff --git a/lib/runner/src/snapshot.rs b/lib/runner/src/snapshot.rs new file mode 100644 index 00000000000..047b09d9aef --- /dev/null +++ b/lib/runner/src/snapshot.rs @@ -0,0 +1,30 @@ +/// A snapshot that captures the runtime state of an instance. +#[derive(Default, serde::Serialize, serde::Deserialize, Clone, Debug)] +pub struct InstanceSnapshot { + /// Values of all globals, indexed by the same index used in Webassembly. + pub globals: Vec, +} + +impl InstanceSnapshot { + pub fn serialize(&self) -> Result, bincode::Error> { + bincode::serialize(self) + } + + pub fn deserialize(data: &[u8]) -> Result { + bincode::deserialize(data) + } +} + +pub fn capture_snapshot(store: &mut impl wasmer::AsStoreMut) -> InstanceSnapshot { + let objs = store.objects_mut(); + let globals = objs.as_u128_globals(); + InstanceSnapshot { globals } +} + +pub fn restore_snapshot(store: &mut impl wasmer::AsStoreMut, snapshot: &InstanceSnapshot) { + let objs = store.objects_mut(); + + for (index, value) in snapshot.globals.iter().enumerate() { + objs.set_global_unchecked(index, *value); + } +} diff --git a/lib/wasix/src/runtime/task_manager/mod.rs b/lib/runner/src/task_manager/mod.rs similarity index 98% rename from lib/wasix/src/runtime/task_manager/mod.rs rename to lib/runner/src/task_manager/mod.rs index 2e6d024fe27..c0bb4fd6eb3 100644 --- a/lib/wasix/src/runtime/task_manager/mod.rs +++ b/lib/runner/src/task_manager/mod.rs @@ -13,8 +13,12 @@ use wasmer::{AsStoreMut, AsStoreRef, Memory, MemoryType, Module, Store, StoreMut use wasmer_wasix_types::wasi::{Errno, ExitCode}; use crate::os::task::thread::WasiThreadError; -use crate::syscalls::AsyncifyFuture; -use crate::{capture_snapshot, InstanceSnapshot, WasiEnv, WasiFunctionEnv, WasiThread}; +use crate::snapshot::{capture_snapshot, InstanceSnapshot}; +use crate::{WasiEnv, WasiFunctionEnv, WasiThread}; + +/// Future that will be polled by asyncify methods +#[doc(hidden)] +pub type AsyncifyFuture = dyn Future + Send + Sync + 'static; pub use virtual_mio::waker::*; diff --git a/lib/wasix/src/runtime/task_manager/thread.rs b/lib/runner/src/task_manager/thread.rs similarity index 100% rename from lib/wasix/src/runtime/task_manager/thread.rs rename to lib/runner/src/task_manager/thread.rs diff --git a/lib/wasix/src/runtime/task_manager/tokio.rs b/lib/runner/src/task_manager/tokio.rs similarity index 100% rename from lib/wasix/src/runtime/task_manager/tokio.rs rename to lib/runner/src/task_manager/tokio.rs diff --git a/lib/wasix/src/runtime/tty_sys.rs b/lib/runner/src/tty_sys.rs similarity index 100% rename from lib/wasix/src/runtime/tty_sys.rs rename to lib/runner/src/tty_sys.rs diff --git a/lib/wasix/src/lib.rs b/lib/wasix/src/lib.rs index d8a03813f25..ebcb8608f95 100644 --- a/lib/wasix/src/lib.rs +++ b/lib/wasix/src/lib.rs @@ -42,21 +42,14 @@ mod macros; pub mod bin_factory; pub mod os; // TODO: should this be pub? -pub mod net; -// TODO: should this be pub? -pub mod capabilities; pub mod fs; -pub mod http; +pub mod net; mod rewind; pub mod runners; -pub mod runtime; mod state; mod syscalls; mod utils; -/// WAI based bindings. -mod bindings; - #[allow(unused_imports)] use bytes::{Bytes, BytesMut}; use os::task::control_plane::ControlPlaneError; diff --git a/lib/wasix/src/utils/store.rs b/lib/wasix/src/utils/store.rs index 047b09d9aef..2b7d1f729d2 100644 --- a/lib/wasix/src/utils/store.rs +++ b/lib/wasix/src/utils/store.rs @@ -1,30 +1 @@ -/// A snapshot that captures the runtime state of an instance. -#[derive(Default, serde::Serialize, serde::Deserialize, Clone, Debug)] -pub struct InstanceSnapshot { - /// Values of all globals, indexed by the same index used in Webassembly. - pub globals: Vec, -} - -impl InstanceSnapshot { - pub fn serialize(&self) -> Result, bincode::Error> { - bincode::serialize(self) - } - - pub fn deserialize(data: &[u8]) -> Result { - bincode::deserialize(data) - } -} - -pub fn capture_snapshot(store: &mut impl wasmer::AsStoreMut) -> InstanceSnapshot { - let objs = store.objects_mut(); - let globals = objs.as_u128_globals(); - InstanceSnapshot { globals } -} - -pub fn restore_snapshot(store: &mut impl wasmer::AsStoreMut, snapshot: &InstanceSnapshot) { - let objs = store.objects_mut(); - - for (index, value) in snapshot.globals.iter().enumerate() { - objs.set_global_unchecked(index, *value); - } -} +pub use wasmer_runner::snapshot::{capture_snapshot, restore_snapshot, InstanceSnapshot}; From 738a7bf7d9b59784510eed6431e3ef2c6193e85e Mon Sep 17 00:00:00 2001 From: Syrus Akbary Date: Sun, 12 Nov 2023 09:26:33 -0800 Subject: [PATCH 4/5] Fix linting --- lib/runner/src/task_manager/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/runner/src/task_manager/mod.rs b/lib/runner/src/task_manager/mod.rs index c0bb4fd6eb3..268686ec691 100644 --- a/lib/runner/src/task_manager/mod.rs +++ b/lib/runner/src/task_manager/mod.rs @@ -13,8 +13,8 @@ use wasmer::{AsStoreMut, AsStoreRef, Memory, MemoryType, Module, Store, StoreMut use wasmer_wasix_types::wasi::{Errno, ExitCode}; use crate::os::task::thread::WasiThreadError; -use crate::snapshot::{capture_snapshot, InstanceSnapshot}; use crate::{WasiEnv, WasiFunctionEnv, WasiThread}; +use crate::snapshot::{InstanceSnapshot, capture_snapshot}; /// Future that will be polled by asyncify methods #[doc(hidden)] From 3da878db25d8379d32b5d547f03ea29dd5de1db7 Mon Sep 17 00:00:00 2001 From: Syrus Akbary Date: Sun, 12 Nov 2023 14:10:30 -0800 Subject: [PATCH 5/5] Update cargo toml --- lib/runner/Cargo.toml | 55 ------------------------------------------- 1 file changed, 55 deletions(-) diff --git a/lib/runner/Cargo.toml b/lib/runner/Cargo.toml index 16298510fb0..31a0d806993 100644 --- a/lib/runner/Cargo.toml +++ b/lib/runner/Cargo.toml @@ -16,7 +16,6 @@ rust-version.workspace = true cfg-if = "1.0" thiserror = "1" tracing = { version = "0.1.37" } -# getrandom = "0.2" wasmer-wasix-types = { path = "../wasi-types", version = "0.16.0", features = [ "enable-serde" ] } wasmer-types = { path = "../types", version = "=4.2.3", default-features = false } wasmer = { path = "../api", version = "=4.2.3", default-features = true } @@ -24,86 +23,32 @@ webc.workspace = true virtual-mio = { path = "../virtual-io", version = "0.3.0", default-features = false } virtual-fs = { path = "../virtual-fs", version = "0.9.0", default-features = false, features = ["webc-fs"] } virtual-net = { path = "../virtual-net", version = "0.6.1", default-features = false } -# wasmer-emscripten = { path = "../emscripten", version = "=4.2.3", optional = true } -# typetag = { version = "0.1", optional = true } serde = { version = "1.0", default-features = false, features = ["derive"] } bincode = { version = "1.3" } -# chrono = { version = "^0.4.31", default-features = false, features = [ "wasmbind", "std", "clock" ], optional = true } derivative = { version = "^2" } bytes = "1" -# webc = { workspace = true } serde_cbor = { version = "0.11.2" } anyhow = { version = "1.0.66" } -# lazy_static = "1.4" sha2 = { version = "0.10" } -# waker-fn = { version = "1.1" } -# cooked-waker = "^5" -# rand = "0.8" tokio = { version = "1", features = ["sync", "macros", "time", "rt", "rt-multi-thread"], default_features = false } futures = { version = "0.3" } -# # used by feature='os' async-trait = { version = "^0.1" } -# urlencoding = { version = "^2" } -# serde_derive = { version = "^1" } serde_json = { version = "^1" } -# serde_yaml = { version = "^0.8" } weezl = { version = "^0.1" } hex = { version = "^0.4" } term_size = { version = "0.3" } -# linked_hash_set = { version = "0.1" } http = "0.2.8" wai-bindgen-wasmer = { path = "../wai-bindgen-wasmer", version = "0.16.0", features = ["tracing"] } -# heapless = "0.7.16" once_cell = "1.17.0" -# pin-project = "1.0.12" semver = "1.0.17" dashmap = "5.4.0" tempfile = "3.6.0" -# # Used by the WCGI runner -# hyper = { version = "0.14", features = ["server", "stream"], optional = true } -# wcgi = { version = "0.1.2", optional = true } -# wcgi-host = { version = "0.1.2", optional = true } -# tower-http = { version = "0.4.0", features = ["trace", "util", "catch-panic", "cors"], optional = true } -# tower = { version = "0.4.13", features = ["make", "util"], optional = true } url = "2.3.1" petgraph = "0.6.3" -# rayon = { version = "1.7.0", optional = true } wasm-bindgen = { version = "0.2.87", optional = true } -# js-sys = { version = "0.3.64", optional = true } -# wasm-bindgen-futures = { version = "0.4.37", optional = true } -# web-sys = { version = "0.3.64", features = ["Request", "RequestInit", "Window", "WorkerGlobalScope", "RequestMode", "Response", "Headers"], optional = true } - -# [target.'cfg(not(target_arch = "riscv64"))'.dependencies.reqwest] -# version = "0.11" -# default-features = false -# features = ["rustls-tls", "json"] -# optional = true - -# [target.'cfg(target_arch = "riscv64")'.dependencies.reqwest] -# version = "0.11" -# default-features = false -# features = ["native-tls", "json"] -# optional = true [target.'cfg(unix)'.dependencies] libc = { version = "^0.2", default-features = false } [target.'cfg(all(unix, not(target_os="ios")))'.dependencies] termios = { version = "0.3" } - -# [target.'cfg(windows)'.dependencies] -# winapi = "0.3" - -# [dev-dependencies] -# wasmer = { path = "../api", version = "=4.2.3", default-features = false, features = ["wat", "js-serializable-module"] } -# tokio = { version = "1", features = [ "sync", "macros", "rt" ], default_features = false } -# pretty_assertions = "1.3.0" -# wasm-bindgen-test = "0.3.0" - -# [target.'cfg(target_arch = "wasm32")'.dev-dependencies] -# wasm-bindgen-test = "0.3.0" -# tracing-wasm = "0.2" - -# [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] -# tracing-subscriber = { version = "^0.3" } -# wasmer = { path = "../api", version = "=4.2.3", default-features = false, features = ["wat", "js-serializable-module", "cranelift"] }