From a8984f0effbd77b75b0082f68116f76b9ef9c10d Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 7 Dec 2020 18:20:08 +0100 Subject: [PATCH 01/73] Fix unused variable warning in client-local example --- derive/examples/client-local.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/derive/examples/client-local.rs b/derive/examples/client-local.rs index 09a8c0148..a0fdf6fe1 100644 --- a/derive/examples/client-local.rs +++ b/derive/examples/client-local.rs @@ -50,8 +50,8 @@ fn main() { futures::pin_mut!(server); futures::select! { - server = server => {}, - client = client => {}, + _server = server => {}, + _client = client => {}, } }); } From 907cc08e428089931f6b5bcf34e70bd106ddb92a Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 7 Dec 2020 18:23:51 +0100 Subject: [PATCH 02/73] Mark tokio_service import as trait-specific --- ipc/src/server.rs | 2 +- tcp/src/server.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ipc/src/server.rs b/ipc/src/server.rs index 47aaa9f29..8a71a4132 100644 --- a/ipc/src/server.rs +++ b/ipc/src/server.rs @@ -7,7 +7,7 @@ use crate::select_with_weak::SelectWithWeakExt; use futures01::{future, sync::oneshot, Future, Sink, Stream}; use parity_tokio_ipc::Endpoint; use parking_lot::Mutex; -use tokio_service::{self, Service as TokioService}; +use tokio_service::{self, Service as _}; use crate::server_utils::{ codecs, reactor, session, diff --git a/tcp/src/server.rs b/tcp/src/server.rs index 6e7d3693d..81120f62b 100644 --- a/tcp/src/server.rs +++ b/tcp/src/server.rs @@ -2,7 +2,7 @@ use std; use std::net::SocketAddr; use std::sync::Arc; -use tokio_service::Service as TokioService; +use tokio_service::Service as _; use futures01::sync::oneshot; use futures01::{future, Future, Sink, Stream}; From 867c72f063b2237d105d5b2980d7b6d3d430547d Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 7 Dec 2020 18:26:49 +0100 Subject: [PATCH 03/73] Unify parity-tokio-ipc dependencies Rather than separately pulling 0.2 and 0.4 versions. --- core-client/transports/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-client/transports/Cargo.toml b/core-client/transports/Cargo.toml index a1804b8c1..275e859c2 100644 --- a/core-client/transports/Cargo.toml +++ b/core-client/transports/Cargo.toml @@ -49,7 +49,7 @@ futures01 = { version = "0.1.26", package = "futures", optional = true } hyper = { version = "0.12", optional = true } hyper-tls = { version = "0.3.2", optional = true } jsonrpc-server-utils = { version = "16.0", path = "../../server-utils", optional = true } -parity-tokio-ipc = { version = "0.2", optional = true } +parity-tokio-ipc = { version = "0.4", optional = true } tokio = { version = "0.1", optional = true } websocket = { version = "0.24", optional = true } From 51c2feeaebcf26ca421f59658dd79cc23a2a14ae Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 7 Dec 2020 20:12:18 +0100 Subject: [PATCH 04/73] WIP: Try to adapt ipc to use futures03 --- core-client/transports/Cargo.toml | 2 +- ipc/Cargo.toml | 2 +- ipc/src/meta.rs | 4 +- ipc/src/server.rs | 63 +++++++++++++------------- server-utils/Cargo.toml | 2 + server-utils/src/lib.rs | 1 + server-utils/src/stream_codec.rs | 73 +++++++++++++++++++++++++++++++ 7 files changed, 112 insertions(+), 35 deletions(-) diff --git a/core-client/transports/Cargo.toml b/core-client/transports/Cargo.toml index 275e859c2..9b2601758 100644 --- a/core-client/transports/Cargo.toml +++ b/core-client/transports/Cargo.toml @@ -49,7 +49,7 @@ futures01 = { version = "0.1.26", package = "futures", optional = true } hyper = { version = "0.12", optional = true } hyper-tls = { version = "0.3.2", optional = true } jsonrpc-server-utils = { version = "16.0", path = "../../server-utils", optional = true } -parity-tokio-ipc = { version = "0.4", optional = true } +parity-tokio-ipc = { version = "0.5", optional = true } tokio = { version = "0.1", optional = true } websocket = { version = "0.24", optional = true } diff --git a/ipc/Cargo.toml b/ipc/Cargo.toml index ff38dee14..ca8acc1f2 100644 --- a/ipc/Cargo.toml +++ b/ipc/Cargo.toml @@ -16,7 +16,7 @@ log = "0.4" tokio-service = "0.1" jsonrpc-core = { version = "16.0", path = "../core" } jsonrpc-server-utils = { version = "16.0", path = "../server-utils" } -parity-tokio-ipc = "0.4" +parity-tokio-ipc = "0.5" parking_lot = "0.11.0" [dev-dependencies] diff --git a/ipc/src/meta.rs b/ipc/src/meta.rs index a1ae788d4..497eaf086 100644 --- a/ipc/src/meta.rs +++ b/ipc/src/meta.rs @@ -1,3 +1,5 @@ +use std::path::Path; + use crate::jsonrpc::futures::channel::mpsc; use crate::jsonrpc::Metadata; use crate::server_utils::session; @@ -7,7 +9,7 @@ pub struct RequestContext<'a> { /// Session ID pub session_id: session::SessionId, /// Remote UDS endpoint - pub endpoint_addr: &'a ::parity_tokio_ipc::RemoteId, + pub endpoint_addr: &'a Path, /// Direct pipe sender pub sender: mpsc::UnboundedSender, } diff --git a/ipc/src/server.rs b/ipc/src/server.rs index 8a71a4132..4804012c4 100644 --- a/ipc/src/server.rs +++ b/ipc/src/server.rs @@ -12,7 +12,6 @@ use tokio_service::{self, Service as _}; use crate::server_utils::{ codecs, reactor, session, tokio::{reactor::Handle, runtime::TaskExecutor}, - tokio_codec::Framed, }; pub use parity_tokio_ipc::SecurityAttributes; @@ -149,7 +148,6 @@ where /// Creates a new server from the given endpoint. pub fn start(self, path: &str) -> std::io::Result { let executor = self.executor.initialize()?; - let reactor = self.reactor; let rpc_handler = self.handler; let endpoint_addr = path.to_owned(); let meta_extractor = self.meta_extractor; @@ -173,15 +171,8 @@ where } } - // Make sure to construct Handle::default() inside Tokio runtime - let reactor = if cfg!(windows) { - #[allow(deprecated)] - reactor.unwrap_or_else(Handle::current) - } else { - reactor.unwrap_or_else(Handle::default) - }; - - let connections = match endpoint.incoming(&reactor) { + let endpoint_addr = endpoint.path().to_owned(); + let connections = match endpoint.incoming() { Ok(connections) => connections, Err(e) => { start_signal @@ -193,7 +184,8 @@ where let mut id = 0u64; - let server = connections.map(move |(io_stream, remote_id)| { + use futures03::TryStreamExt; + let server = connections.compat().map(move |io_stream| { id = id.wrapping_add(1); let session_id = id; let session_stats = session_stats.clone(); @@ -204,41 +196,48 @@ where let (sender, receiver) = mpsc::unbounded(); let meta = meta_extractor.extract(&RequestContext { - endpoint_addr: &remote_id, + endpoint_addr: endpoint_addr.as_ref(), session_id, sender, }); let service = Service::new(rpc_handler.clone(), meta); - let (writer, reader) = Framed::new( - io_stream, - codecs::StreamCodec::new(incoming_separator.clone(), outgoing_separator.clone()), - ) - .split(); + use crate::server_utils::tokio_util; + let codec = codecs::StreamCodec::new(incoming_separator.clone(), outgoing_separator.clone()); + use tokio_util::codec::Decoder as _; + use futures03::StreamExt; + let (writer, reader) = codec.framed(io_stream).split(); + + // let (writer, reader) = Framed::new(io_stream, codec).split(); let responses = reader - .map(move |req| { + .map(move |req: std::io::Result| { + // FIXME: !! + let req = req.unwrap(); + + use futures03::compat::Future01CompatExt; service .call(req) .then(|result| match result { - Err(_) => future::ok(None), - Ok(some_result) => future::ok(some_result), + Err(_) => future::ok::<_, ()>(None), + Ok(some_result) => future::ok::<_, ()>(some_result), }) .map_err(|_: ()| std::io::ErrorKind::Other.into()) + .compat() }) .buffer_unordered(client_buffer_size) + .compat() .filter_map(|x| x) // we use `select_with_weak` here, instead of `select`, to close the stream // as soon as the ipc pipe is closed - .select_with_weak(futures03::TryStreamExt::compat(futures03::StreamExt::map(receiver, Ok))); - - let writer = writer.send_all(responses).then(move |_| { - trace!(target: "ipc", "Peer: service finished"); - if let Some(stats) = session_stats.as_ref() { - stats.close_session(session_id) - } - Ok(()) - }); - - writer + .select_with_weak(futures03::TryStreamExt::compat(futures03::StreamExt::map(receiver, std::io::Result::Ok))); + + responses.forward(futures03::SinkExt::compat(writer)) + .and_then(move |_| { + trace!(target: "ipc", "Peer: service finished"); + if let Some(stats) = session_stats.as_ref() { + stats.close_session(session_id) + } + Ok(()) + }) }); start_signal .send(Ok(())) diff --git a/server-utils/Cargo.toml b/server-utils/Cargo.toml index 4324500ab..3ac3a458c 100644 --- a/server-utils/Cargo.toml +++ b/server-utils/Cargo.toml @@ -12,6 +12,7 @@ version = "16.0.0" [dependencies] bytes = "0.4" +bytes05 = { version = "0.5", package = "bytes" } futures01 = { version = "0.1", package = "futures" } globset = "0.4" jsonrpc-core = { version = "16.0", path = "../core" } @@ -19,6 +20,7 @@ lazy_static = "1.1.0" log = "0.4" tokio = { version = "0.1.15" } tokio-codec = { version = "0.1" } +tokio-util = { version = "0.3", features = ["codec", "compat"] } unicase = "2.0" [badges] diff --git a/server-utils/src/lib.rs b/server-utils/src/lib.rs index e13342007..be946800e 100644 --- a/server-utils/src/lib.rs +++ b/server-utils/src/lib.rs @@ -10,6 +10,7 @@ extern crate lazy_static; pub use tokio; pub use tokio_codec; +pub use tokio_util; pub mod cors; pub mod hosts; diff --git a/server-utils/src/stream_codec.rs b/server-utils/src/stream_codec.rs index d7cb268b9..5d4e50036 100644 --- a/server-utils/src/stream_codec.rs +++ b/server-utils/src/stream_codec.rs @@ -48,6 +48,66 @@ fn is_whitespace(byte: u8) -> bool { } } +impl tokio_util::codec::Decoder for StreamCodec { + type Item = String; + type Error = io::Error; + + fn decode(&mut self, buf: &mut bytes05::BytesMut) -> io::Result> { + if let Separator::Byte(separator) = self.incoming_separator { + if let Some(i) = buf.as_ref().iter().position(|&b| b == separator) { + let line = buf.split_to(i); + let _ = buf.split_to(1); + + match str::from_utf8(&line.as_ref()) { + Ok(s) => Ok(Some(s.to_string())), + Err(_) => Err(io::Error::new(io::ErrorKind::Other, "invalid UTF-8")), + } + } else { + Ok(None) + } + } else { + let mut depth = 0; + let mut in_str = false; + let mut is_escaped = false; + let mut start_idx = 0; + let mut whitespaces = 0; + + for idx in 0..buf.as_ref().len() { + let byte = buf.as_ref()[idx]; + + if (byte == b'{' || byte == b'[') && !in_str { + if depth == 0 { + start_idx = idx; + } + depth += 1; + } else if (byte == b'}' || byte == b']') && !in_str { + depth -= 1; + } else if byte == b'"' && !is_escaped { + in_str = !in_str; + } else if is_whitespace(byte) { + whitespaces += 1; + } + if byte == b'\\' && !is_escaped && in_str { + is_escaped = true; + } else { + is_escaped = false; + } + + if depth == 0 && idx != start_idx && idx - start_idx + 1 > whitespaces { + let bts = buf.split_to(idx + 1); + match String::from_utf8(bts.as_ref().to_vec()) { + Ok(val) => return Ok(Some(val)), + Err(_) => { + return Ok(None); + } // skip non-utf requests (TODO: log error?) + }; + } + } + Ok(None) + } + } +} + impl Decoder for StreamCodec { type Item = String; type Error = io::Error; @@ -122,6 +182,19 @@ impl Encoder for StreamCodec { } } +impl tokio_util::codec::Encoder for StreamCodec { + type Error = io::Error; + + fn encode(&mut self, msg: String, buf: &mut bytes05::BytesMut) -> io::Result<()> { + let mut payload = msg.into_bytes(); + if let Separator::Byte(separator) = self.outgoing_separator { + payload.push(separator); + } + buf.extend_from_slice(&payload); + Ok(()) + } +} + #[cfg(test)] mod tests { From c7f403d9ad1dfeb71fc4baa46eae36640d22be86 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Tue, 8 Dec 2020 01:15:16 +0100 Subject: [PATCH 05/73] WIP: Use patched parity-tokio-ipc Otherwise we can't return a future due to borrow on Endpoint::incoming --- Cargo.toml | 3 +++ core-client/transports/Cargo.toml | 2 +- ipc/Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b509827d5..4c27e6340 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,3 +14,6 @@ members = [ "test", "ws", ] + +[patch.crates-io] +parity-tokio-ipc = { git = "https://github.com/Xanewok/parity-tokio-ipc", branch = "endpoint-incoming-static" } diff --git a/core-client/transports/Cargo.toml b/core-client/transports/Cargo.toml index 9b2601758..8d83e403a 100644 --- a/core-client/transports/Cargo.toml +++ b/core-client/transports/Cargo.toml @@ -49,7 +49,7 @@ futures01 = { version = "0.1.26", package = "futures", optional = true } hyper = { version = "0.12", optional = true } hyper-tls = { version = "0.3.2", optional = true } jsonrpc-server-utils = { version = "16.0", path = "../../server-utils", optional = true } -parity-tokio-ipc = { version = "0.5", optional = true } +parity-tokio-ipc = { version = "0.7", optional = true } tokio = { version = "0.1", optional = true } websocket = { version = "0.24", optional = true } diff --git a/ipc/Cargo.toml b/ipc/Cargo.toml index ca8acc1f2..f2cff8bfa 100644 --- a/ipc/Cargo.toml +++ b/ipc/Cargo.toml @@ -16,7 +16,7 @@ log = "0.4" tokio-service = "0.1" jsonrpc-core = { version = "16.0", path = "../core" } jsonrpc-server-utils = { version = "16.0", path = "../server-utils" } -parity-tokio-ipc = "0.5" +parity-tokio-ipc = "0.7" parking_lot = "0.11.0" [dev-dependencies] From 4461c61b17111528649aa18f38097b4fa3b6b7bb Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Tue, 8 Dec 2020 02:21:15 +0100 Subject: [PATCH 06/73] WIP: PoC with tokio_compat --- ipc/src/server.rs | 9 +++++++++ server-utils/Cargo.toml | 1 + server-utils/src/lib.rs | 1 + server-utils/src/reactor.rs | 12 +++++++++++- 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/ipc/src/server.rs b/ipc/src/server.rs index 4804012c4..8baa6bbc8 100644 --- a/ipc/src/server.rs +++ b/ipc/src/server.rs @@ -12,6 +12,7 @@ use tokio_service::{self, Service as _}; use crate::server_utils::{ codecs, reactor, session, tokio::{reactor::Handle, runtime::TaskExecutor}, + tokio_compat, }; pub use parity_tokio_ipc::SecurityAttributes; @@ -105,6 +106,12 @@ where self } + /// Sets shared different event loop executor. + pub fn event_loop_executor_compat(mut self, executor: tokio_compat::runtime::TaskExecutor) -> Self { + self.executor = reactor::UninitializedExecutor::Compat(executor); + self + } + /// Sets different event loop I/O reactor. pub fn event_loop_reactor(mut self, reactor: Handle) -> Self { self.reactor = Some(reactor); @@ -568,7 +575,9 @@ mod tests { }; let io = MetaIoHandler::>::default(); + let mut rt = tokio_compat::runtime::Builder::new().build().unwrap(); let builder = ServerBuilder::with_meta_extractor(io, session_metadata_extractor); + let builder = builder.event_loop_executor_compat(rt.executor()); let server = builder.start(path).expect("Server must run with no issues"); { let _ = UnixStream::connect(path).wait().expect("Socket should connect"); diff --git a/server-utils/Cargo.toml b/server-utils/Cargo.toml index 3ac3a458c..995dacefe 100644 --- a/server-utils/Cargo.toml +++ b/server-utils/Cargo.toml @@ -21,6 +21,7 @@ log = "0.4" tokio = { version = "0.1.15" } tokio-codec = { version = "0.1" } tokio-util = { version = "0.3", features = ["codec", "compat"] } +tokio-compat = { version = "0.1" } unicase = "2.0" [badges] diff --git a/server-utils/src/lib.rs b/server-utils/src/lib.rs index be946800e..86f9887d2 100644 --- a/server-utils/src/lib.rs +++ b/server-utils/src/lib.rs @@ -11,6 +11,7 @@ extern crate lazy_static; pub use tokio; pub use tokio_codec; pub use tokio_util; +pub use tokio_compat; pub mod cors; pub mod hosts; diff --git a/server-utils/src/reactor.rs b/server-utils/src/reactor.rs index 1d1917db3..7d93ee504 100644 --- a/server-utils/src/reactor.rs +++ b/server-utils/src/reactor.rs @@ -15,6 +15,8 @@ use futures01::Future; pub enum UninitializedExecutor { /// Shared instance of executor. Shared(tokio::runtime::TaskExecutor), + /// Shared instance of executor. + Compat(tokio_compat::runtime::TaskExecutor), /// Event Loop should be spawned by the transport. Unspawned, } @@ -33,6 +35,7 @@ impl UninitializedExecutor { pub fn init_with_name>(self, name: T) -> io::Result { match self { UninitializedExecutor::Shared(executor) => Ok(Executor::Shared(executor)), + UninitializedExecutor::Compat(executor) => Ok(Executor::Compat(executor)), UninitializedExecutor::Unspawned => RpcEventLoop::with_name(Some(name.into())).map(Executor::Spawned), } } @@ -43,6 +46,8 @@ impl UninitializedExecutor { pub enum Executor { /// Shared instance Shared(tokio::runtime::TaskExecutor), + /// Shared instance + Compat(tokio_compat::runtime::TaskExecutor), /// Spawned Event Loop Spawned(RpcEventLoop), } @@ -52,6 +57,7 @@ impl Executor { pub fn executor(&self) -> tokio::runtime::TaskExecutor { match *self { Executor::Shared(ref executor) => executor.clone(), + Executor::Compat(..) => panic!(), Executor::Spawned(ref eloop) => eloop.executor(), } } @@ -61,7 +67,11 @@ impl Executor { where F: Future + Send + 'static, { - self.executor().spawn(future) + match self { + Executor::Shared(exe) => exe.spawn(future), + Executor::Compat(exe) => exe.spawn(future), + Executor::Spawned(eloop) => eloop.executor().spawn(future), + } } /// Closes underlying event loop (if any!). From c1409c21c877be5a1d404c67fe5abe56dced0854 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Tue, 8 Dec 2020 17:42:01 +0100 Subject: [PATCH 07/73] Fix ipc test failure --- ipc/src/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipc/src/server.rs b/ipc/src/server.rs index 8baa6bbc8..2888a674b 100644 --- a/ipc/src/server.rs +++ b/ipc/src/server.rs @@ -238,7 +238,7 @@ where .select_with_weak(futures03::TryStreamExt::compat(futures03::StreamExt::map(receiver, std::io::Result::Ok))); responses.forward(futures03::SinkExt::compat(writer)) - .and_then(move |_| { + .then(move |_| { trace!(target: "ipc", "Peer: service finished"); if let Some(stats) = session_stats.as_ref() { stats.close_session(session_id) From 224c4a0cae158829e5dfe28ddff9eaef39167361 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Tue, 8 Dec 2020 17:47:49 +0100 Subject: [PATCH 08/73] Fix and adapt rest of IPC --- ipc/src/server.rs | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/ipc/src/server.rs b/ipc/src/server.rs index 2888a674b..3d58c1898 100644 --- a/ipc/src/server.rs +++ b/ipc/src/server.rs @@ -4,7 +4,7 @@ use crate::jsonrpc::futures::channel::mpsc; use crate::jsonrpc::{middleware, MetaIoHandler, Metadata, Middleware}; use crate::meta::{MetaExtractor, NoopExtractor, RequestContext}; use crate::select_with_weak::SelectWithWeakExt; -use futures01::{future, sync::oneshot, Future, Sink, Stream}; +use futures01::{future, sync::oneshot, Future, Stream}; use parity_tokio_ipc::Endpoint; use parking_lot::Mutex; use tokio_service::{self, Service as _}; @@ -13,6 +13,7 @@ use crate::server_utils::{ codecs, reactor, session, tokio::{reactor::Handle, runtime::TaskExecutor}, tokio_compat, + tokio_util, }; pub use parity_tokio_ipc::SecurityAttributes; @@ -208,30 +209,19 @@ where sender, }); let service = Service::new(rpc_handler.clone(), meta); - use crate::server_utils::tokio_util; let codec = codecs::StreamCodec::new(incoming_separator.clone(), outgoing_separator.clone()); - use tokio_util::codec::Decoder as _; - use futures03::StreamExt; - let (writer, reader) = codec.framed(io_stream).split(); + let framed = tokio_util::codec::Decoder::framed(codec, io_stream); + let (writer, reader) = futures03::StreamExt::split(framed); - // let (writer, reader) = Framed::new(io_stream, codec).split(); let responses = reader - .map(move |req: std::io::Result| { - // FIXME: !! - let req = req.unwrap(); - - use futures03::compat::Future01CompatExt; + .compat() + .map(move |req| { service .call(req) - .then(|result| match result { - Err(_) => future::ok::<_, ()>(None), - Ok(some_result) => future::ok::<_, ()>(some_result), - }) + .then(|result| future::ok::<_, ()>(result.unwrap_or(None))) .map_err(|_: ()| std::io::ErrorKind::Other.into()) - .compat() }) .buffer_unordered(client_buffer_size) - .compat() .filter_map(|x| x) // we use `select_with_weak` here, instead of `select`, to close the stream // as soon as the ipc pipe is closed From 8ed4be70967dd4ec6fbe2e739a9aec151b5117ea Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Tue, 8 Dec 2020 18:00:31 +0100 Subject: [PATCH 09/73] Prefer using tokio_compat everywhere for now --- http/src/lib.rs | 6 +++--- ipc/src/server.rs | 12 ++---------- server-utils/src/reactor.rs | 31 +++++++++++-------------------- tcp/src/server.rs | 4 ++-- ws/src/metadata.rs | 2 +- ws/src/server_builder.rs | 2 +- ws/src/session.rs | 2 +- 7 files changed, 21 insertions(+), 38 deletions(-) diff --git a/http/src/lib.rs b/http/src/lib.rs index 9fc4a2a7c..114129209 100644 --- a/http/src/lib.rs +++ b/http/src/lib.rs @@ -53,7 +53,7 @@ pub use crate::handler::ServerHandler; pub use crate::response::Response; pub use crate::server_utils::cors::{self, AccessControlAllowOrigin, AllowCors, Origin}; pub use crate::server_utils::hosts::{DomainsValidation, Host}; -pub use crate::server_utils::{tokio, SuspendableStream}; +pub use crate::server_utils::{tokio, tokio_compat, SuspendableStream}; pub use crate::utils::{cors_allow_headers, cors_allow_origin, is_host_allowed}; /// Action undertaken by a middleware. @@ -300,7 +300,7 @@ where /// Utilize existing event loop executor to poll RPC results. /// /// Applies only to 1 of the threads. Other threads will spawn their own Event Loops. - pub fn event_loop_executor(mut self, executor: tokio::runtime::TaskExecutor) -> Self { + pub fn event_loop_executor(mut self, executor: tokio_compat::runtime::TaskExecutor) -> Self { self.executor = UninitializedExecutor::Shared(executor); self } @@ -519,7 +519,7 @@ fn serve>( mpsc::Sender>, oneshot::Sender<()>, ), - executor: tokio::runtime::TaskExecutor, + executor: tokio_compat::runtime::TaskExecutor, addr: SocketAddr, cors_domains: CorsDomains, cors_max_age: Option, diff --git a/ipc/src/server.rs b/ipc/src/server.rs index 3d58c1898..e80912cd8 100644 --- a/ipc/src/server.rs +++ b/ipc/src/server.rs @@ -11,8 +11,8 @@ use tokio_service::{self, Service as _}; use crate::server_utils::{ codecs, reactor, session, - tokio::{reactor::Handle, runtime::TaskExecutor}, - tokio_compat, + tokio::reactor::Handle, + tokio_compat::runtime::TaskExecutor, tokio_util, }; @@ -107,12 +107,6 @@ where self } - /// Sets shared different event loop executor. - pub fn event_loop_executor_compat(mut self, executor: tokio_compat::runtime::TaskExecutor) -> Self { - self.executor = reactor::UninitializedExecutor::Compat(executor); - self - } - /// Sets different event loop I/O reactor. pub fn event_loop_reactor(mut self, reactor: Handle) -> Self { self.reactor = Some(reactor); @@ -565,9 +559,7 @@ mod tests { }; let io = MetaIoHandler::>::default(); - let mut rt = tokio_compat::runtime::Builder::new().build().unwrap(); let builder = ServerBuilder::with_meta_extractor(io, session_metadata_extractor); - let builder = builder.event_loop_executor_compat(rt.executor()); let server = builder.start(path).expect("Server must run with no issues"); { let _ = UnixStream::connect(path).wait().expect("Socket should connect"); diff --git a/server-utils/src/reactor.rs b/server-utils/src/reactor.rs index 7d93ee504..d30d1df5e 100644 --- a/server-utils/src/reactor.rs +++ b/server-utils/src/reactor.rs @@ -6,7 +6,6 @@ //! that `tokio::runtime` can be multi-threaded. use std::io; -use tokio; use futures01::Future; @@ -14,9 +13,7 @@ use futures01::Future; #[derive(Debug)] pub enum UninitializedExecutor { /// Shared instance of executor. - Shared(tokio::runtime::TaskExecutor), - /// Shared instance of executor. - Compat(tokio_compat::runtime::TaskExecutor), + Shared(tokio_compat::runtime::TaskExecutor), /// Event Loop should be spawned by the transport. Unspawned, } @@ -35,7 +32,6 @@ impl UninitializedExecutor { pub fn init_with_name>(self, name: T) -> io::Result { match self { UninitializedExecutor::Shared(executor) => Ok(Executor::Shared(executor)), - UninitializedExecutor::Compat(executor) => Ok(Executor::Compat(executor)), UninitializedExecutor::Unspawned => RpcEventLoop::with_name(Some(name.into())).map(Executor::Spawned), } } @@ -45,19 +41,16 @@ impl UninitializedExecutor { #[derive(Debug)] pub enum Executor { /// Shared instance - Shared(tokio::runtime::TaskExecutor), - /// Shared instance - Compat(tokio_compat::runtime::TaskExecutor), + Shared(tokio_compat::runtime::TaskExecutor), /// Spawned Event Loop Spawned(RpcEventLoop), } impl Executor { /// Get tokio executor associated with this event loop. - pub fn executor(&self) -> tokio::runtime::TaskExecutor { - match *self { + pub fn executor(&self) -> tokio_compat::runtime::TaskExecutor { + match self { Executor::Shared(ref executor) => executor.clone(), - Executor::Compat(..) => panic!(), Executor::Spawned(ref eloop) => eloop.executor(), } } @@ -69,7 +62,6 @@ impl Executor { { match self { Executor::Shared(exe) => exe.spawn(future), - Executor::Compat(exe) => exe.spawn(future), Executor::Spawned(eloop) => eloop.executor().spawn(future), } } @@ -92,9 +84,9 @@ impl Executor { /// A handle to running event loop. Dropping the handle will cause event loop to finish. #[derive(Debug)] pub struct RpcEventLoop { - executor: tokio::runtime::TaskExecutor, + executor: tokio_compat::runtime::TaskExecutor, close: Option>, - handle: Option, + runtime: Option, } impl Drop for RpcEventLoop { @@ -113,34 +105,33 @@ impl RpcEventLoop { pub fn with_name(name: Option) -> io::Result { let (stop, stopped) = futures01::oneshot(); - let mut tb = tokio::runtime::Builder::new(); + let mut tb = tokio_compat::runtime::Builder::new(); tb.core_threads(1); if let Some(name) = name { tb.name_prefix(name); } - let mut runtime = tb.build()?; + let runtime = tb.build()?; let executor = runtime.executor(); let terminate = futures01::empty().select(stopped).map(|_| ()).map_err(|_| ()); runtime.spawn(terminate); - let handle = runtime.shutdown_on_idle(); Ok(RpcEventLoop { executor, close: Some(stop), - handle: Some(handle), + runtime: Some(runtime), }) } /// Get executor for this event loop. - pub fn executor(&self) -> tokio::runtime::TaskExecutor { + pub fn executor(&self) -> tokio_compat::runtime::TaskExecutor { self.executor.clone() } /// Blocks current thread and waits until the event loop is finished. pub fn wait(mut self) -> Result<(), ()> { - self.handle.take().ok_or(())?.wait() + self.runtime.take().ok_or(())?.shutdown_on_idle().wait() } /// Finishes this event loop. diff --git a/tcp/src/server.rs b/tcp/src/server.rs index 81120f62b..c5ddb1447 100644 --- a/tcp/src/server.rs +++ b/tcp/src/server.rs @@ -8,7 +8,7 @@ use futures01::sync::oneshot; use futures01::{future, Future, Sink, Stream}; use crate::jsonrpc::{middleware, MetaIoHandler, Metadata, Middleware}; -use crate::server_utils::{codecs, reactor, tokio, tokio_codec::Framed, SuspendableStream}; +use crate::server_utils::{codecs, reactor, tokio, tokio_compat, tokio_codec::Framed, SuspendableStream}; use crate::dispatch::{Dispatcher, PeerMessageQueue, SenderChannels}; use crate::meta::{MetaExtractor, NoopExtractor, RequestContext}; @@ -60,7 +60,7 @@ where } /// Utilize existing event loop executor. - pub fn event_loop_executor(mut self, handle: tokio::runtime::TaskExecutor) -> Self { + pub fn event_loop_executor(mut self, handle: tokio_compat::runtime::TaskExecutor) -> Self { self.executor = reactor::UninitializedExecutor::Shared(handle); self } diff --git a/ws/src/metadata.rs b/ws/src/metadata.rs index 624be5830..95b555481 100644 --- a/ws/src/metadata.rs +++ b/ws/src/metadata.rs @@ -3,7 +3,7 @@ use std::sync::{atomic, Arc}; use crate::core; use crate::core::futures::channel::mpsc; -use crate::server_utils::{session, tokio::runtime::TaskExecutor}; +use crate::server_utils::{session, tokio_compat::runtime::TaskExecutor}; use crate::ws; use crate::error; diff --git a/ws/src/server_builder.rs b/ws/src/server_builder.rs index e10978f34..10124462b 100644 --- a/ws/src/server_builder.rs +++ b/ws/src/server_builder.rs @@ -69,7 +69,7 @@ where } /// Utilize existing event loop executor to poll RPC results. - pub fn event_loop_executor(mut self, executor: server_utils::tokio::runtime::TaskExecutor) -> Self { + pub fn event_loop_executor(mut self, executor: server_utils::tokio_compat::runtime::TaskExecutor) -> Self { self.executor = UninitializedExecutor::Shared(executor); self } diff --git a/ws/src/session.rs b/ws/src/session.rs index 65d76fd38..43fd4be77 100644 --- a/ws/src/session.rs +++ b/ws/src/session.rs @@ -11,7 +11,7 @@ use slab::Slab; use crate::server_utils::cors::Origin; use crate::server_utils::hosts::Host; use crate::server_utils::session::{SessionId, SessionStats}; -use crate::server_utils::tokio::runtime::TaskExecutor; +use crate::server_utils::tokio_compat::runtime::TaskExecutor; use crate::server_utils::Pattern; use crate::ws; From cab4479909286f59015269c6ef89736f7bd683e5 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Tue, 8 Dec 2020 20:02:22 +0100 Subject: [PATCH 10/73] Prefer tower-service 0.3 over tokio-service 0.1 The former uses Tokio 0.2 --- ipc/Cargo.toml | 2 +- ipc/src/server.rs | 27 +++++++++++++++++---------- tcp/Cargo.toml | 2 +- tcp/src/server.rs | 7 ++++--- tcp/src/service.rs | 22 ++++++++++++---------- 5 files changed, 35 insertions(+), 25 deletions(-) diff --git a/ipc/Cargo.toml b/ipc/Cargo.toml index f2cff8bfa..0f1ddb6f0 100644 --- a/ipc/Cargo.toml +++ b/ipc/Cargo.toml @@ -13,7 +13,7 @@ version = "16.0.0" futures01 = { version = "0.1", package = "futures" } futures03 = { version = "0.3", package = "futures", features = [ "compat" ] } log = "0.4" -tokio-service = "0.1" +tower-service = "0.3" jsonrpc-core = { version = "16.0", path = "../core" } jsonrpc-server-utils = { version = "16.0", path = "../server-utils" } parity-tokio-ipc = "0.7" diff --git a/ipc/src/server.rs b/ipc/src/server.rs index e80912cd8..b6df6fd68 100644 --- a/ipc/src/server.rs +++ b/ipc/src/server.rs @@ -1,13 +1,16 @@ +use std::future::Future; +use std::pin::Pin; use std::sync::Arc; +use std::task::{Context, Poll}; use crate::jsonrpc::futures::channel::mpsc; use crate::jsonrpc::{middleware, MetaIoHandler, Metadata, Middleware}; use crate::meta::{MetaExtractor, NoopExtractor, RequestContext}; use crate::select_with_weak::SelectWithWeakExt; -use futures01::{future, sync::oneshot, Future, Stream}; +use futures01::{future, sync::oneshot, Future as _, Stream}; use parity_tokio_ipc::Endpoint; use parking_lot::Mutex; -use tokio_service::{self, Service as _}; +use tower_service::Service as _; use crate::server_utils::{ codecs, reactor, session, @@ -31,22 +34,24 @@ impl> Service { } } -impl> tokio_service::Service for Service +impl> tower_service::Service for Service where S::Future: Unpin, S::CallFuture: Unpin, { - type Request = String; type Response = Option; - type Error = (); - type Future = Box, Error = ()> + Send>; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } - fn call(&self, req: Self::Request) -> Self::Future { - use futures03::{FutureExt, TryFutureExt}; + fn call(&mut self, req: String) -> Self::Future { + use futures03::FutureExt; trace!(target: "ipc", "Received request: {}", req); - Box::new(self.handler.handle_request(&req, self.meta.clone()).map(Ok).compat()) + Box::pin(self.handler.handle_request(&req, self.meta.clone()).map(Ok)) } } @@ -202,7 +207,7 @@ where session_id, sender, }); - let service = Service::new(rpc_handler.clone(), meta); + let mut service = Service::new(rpc_handler.clone(), meta); let codec = codecs::StreamCodec::new(incoming_separator.clone(), outgoing_separator.clone()); let framed = tokio_util::codec::Decoder::framed(codec, io_stream); let (writer, reader) = futures03::StreamExt::split(framed); @@ -210,8 +215,10 @@ where let responses = reader .compat() .map(move |req| { + use futures03::TryFutureExt; service .call(req) + .compat() .then(|result| future::ok::<_, ()>(result.unwrap_or(None))) .map_err(|_: ()| std::io::ErrorKind::Other.into()) }) diff --git a/tcp/Cargo.toml b/tcp/Cargo.toml index 365d8667a..baa8cb9b8 100644 --- a/tcp/Cargo.toml +++ b/tcp/Cargo.toml @@ -17,7 +17,7 @@ jsonrpc-core = { version = "16.0", path = "../core" } jsonrpc-server-utils = { version = "16.0", path = "../server-utils" } log = "0.4" parking_lot = "0.11.0" -tokio-service = "0.1" +tower-service = "0.3" [dev-dependencies] lazy_static = "1.0" diff --git a/tcp/src/server.rs b/tcp/src/server.rs index c5ddb1447..84b26b129 100644 --- a/tcp/src/server.rs +++ b/tcp/src/server.rs @@ -2,7 +2,7 @@ use std; use std::net::SocketAddr; use std::sync::Arc; -use tokio_service::Service as _; +use tower_service::Service as _; use futures01::sync::oneshot; use futures01::{future, Future, Sink, Stream}; @@ -113,15 +113,16 @@ where }; let meta = meta_extractor.extract(&context); - let service = Service::new(peer_addr, rpc_handler.clone(), meta); + let mut service = Service::new(peer_addr, rpc_handler.clone(), meta); let (writer, reader) = Framed::new( socket, codecs::StreamCodec::new(incoming_separator.clone(), outgoing_separator.clone()), ) .split(); + use futures03::TryFutureExt; let responses = reader.and_then(move |req| { - service.call(req).then(|response| match response { + service.call(req).compat().then(|response| match response { Err(e) => { warn!(target: "tcp", "Error while processing request: {:?}", e); future::ok(String::new()) diff --git a/tcp/src/service.rs b/tcp/src/service.rs index cb0f4b7b2..fc07403fc 100644 --- a/tcp/src/service.rs +++ b/tcp/src/service.rs @@ -1,9 +1,10 @@ +use std::future::Future; use std::net::SocketAddr; +use std::pin::Pin; use std::sync::Arc; +use std::task::{Context, Poll}; -use crate::jsonrpc::futures::FutureExt; use crate::jsonrpc::{middleware, MetaIoHandler, Metadata, Middleware}; -use futures01::Future; pub struct Service = middleware::Noop> { handler: Arc>, @@ -21,26 +22,27 @@ impl> Service { } } -impl> tokio_service::Service for Service +impl> tower_service::Service for Service where S::Future: Unpin, S::CallFuture: Unpin, { // These types must match the corresponding protocol types: - type Request = String; type Response = Option; - // For non-streaming protocols, service errors are always io::Error type Error = (); // The future for computing the response; box it for simplicity. - type Future = Box, Error = ()> + Send>; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } // Produce a future for computing a response from a request. - fn call(&self, req: Self::Request) -> Self::Future { + fn call(&mut self, req: String) -> Self::Future { + use futures03::FutureExt; trace!(target: "tcp", "Accepted request from peer {}: {}", &self.peer_addr, req); - Box::new(futures03::compat::Compat::new( - self.handler.handle_request(&req, self.meta.clone()).map(|v| Ok(v)), - )) + Box::pin(self.handler.handle_request(&req, self.meta.clone()).map(Ok)) } } From 015e918877d3b73e9b5176531c13a6d941fdf30a Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Tue, 8 Dec 2020 22:00:34 +0100 Subject: [PATCH 11/73] Make tokio-codec optional --- server-utils/Cargo.toml | 6 +++--- server-utils/src/lib.rs | 3 ++- server-utils/src/stream_codec.rs | 16 ++++++++-------- tcp/Cargo.toml | 2 +- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/server-utils/Cargo.toml b/server-utils/Cargo.toml index 995dacefe..f6314b150 100644 --- a/server-utils/Cargo.toml +++ b/server-utils/Cargo.toml @@ -11,15 +11,15 @@ repository = "https://github.com/paritytech/jsonrpc" version = "16.0.0" [dependencies] -bytes = "0.4" +bytes04 = { version = "0.4", package = "bytes" } bytes05 = { version = "0.5", package = "bytes" } futures01 = { version = "0.1", package = "futures" } globset = "0.4" jsonrpc-core = { version = "16.0", path = "../core" } lazy_static = "1.1.0" log = "0.4" -tokio = { version = "0.1.15" } -tokio-codec = { version = "0.1" } +tokio = { version = "0.1.15", default-features = false, features = ["timer", "tcp"] } +tokio-codec = { version = "0.1", optional = true } tokio-util = { version = "0.3", features = ["codec", "compat"] } tokio-compat = { version = "0.1" } unicase = "2.0" diff --git a/server-utils/src/lib.rs b/server-utils/src/lib.rs index 86f9887d2..f1f6a62d3 100644 --- a/server-utils/src/lib.rs +++ b/server-utils/src/lib.rs @@ -9,9 +9,10 @@ extern crate log; extern crate lazy_static; pub use tokio; -pub use tokio_codec; pub use tokio_util; pub use tokio_compat; +#[cfg(feature = "tokio-codec")] +pub use tokio_codec; pub mod cors; pub mod hosts; diff --git a/server-utils/src/stream_codec.rs b/server-utils/src/stream_codec.rs index 5d4e50036..448ecfc0f 100644 --- a/server-utils/src/stream_codec.rs +++ b/server-utils/src/stream_codec.rs @@ -1,6 +1,4 @@ -use bytes::BytesMut; use std::{io, str}; -use tokio_codec::{Decoder, Encoder}; /// Separator for enveloping messages in streaming codecs #[derive(Debug, Clone)] @@ -108,11 +106,12 @@ impl tokio_util::codec::Decoder for StreamCodec { } } -impl Decoder for StreamCodec { +#[cfg(feature = "tokio-codec")] +impl tokio_codec::Decoder for StreamCodec { type Item = String; type Error = io::Error; - fn decode(&mut self, buf: &mut BytesMut) -> io::Result> { + fn decode(&mut self, buf: &mut bytes04::BytesMut) -> io::Result> { if let Separator::Byte(separator) = self.incoming_separator { if let Some(i) = buf.as_ref().iter().position(|&b| b == separator) { let line = buf.split_to(i); @@ -168,11 +167,12 @@ impl Decoder for StreamCodec { } } -impl Encoder for StreamCodec { +#[cfg(feature = "tokio-codec")] +impl tokio_codec::Encoder for StreamCodec { type Item = String; type Error = io::Error; - fn encode(&mut self, msg: String, buf: &mut BytesMut) -> io::Result<()> { + fn encode(&mut self, msg: String, buf: &mut bytes04::BytesMut) -> io::Result<()> { let mut payload = msg.into_bytes(); if let Separator::Byte(separator) = self.outgoing_separator { payload.push(separator); @@ -199,8 +199,8 @@ impl tokio_util::codec::Encoder for StreamCodec { mod tests { use super::StreamCodec; - use bytes::{BufMut, BytesMut}; - use tokio_codec::Decoder; + use bytes05::{BufMut, BytesMut}; + use tokio_util::codec::Decoder; #[test] fn simple_encode() { diff --git a/tcp/Cargo.toml b/tcp/Cargo.toml index baa8cb9b8..2fbcfe41a 100644 --- a/tcp/Cargo.toml +++ b/tcp/Cargo.toml @@ -14,7 +14,7 @@ futures01 = { version = "0.1", package = "futures" } # TODO remove when we no longer need compat (use jsonrpc-core re-export instead) futures03 = { version = "0.3", features = ["compat"], package = "futures" } jsonrpc-core = { version = "16.0", path = "../core" } -jsonrpc-server-utils = { version = "16.0", path = "../server-utils" } +jsonrpc-server-utils = { version = "16.0", path = "../server-utils", features = ["tokio-codec"] } log = "0.4" parking_lot = "0.11.0" tower-service = "0.3" From ebc3df9221f69bd7f27470ba118044fca8e582b6 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Tue, 8 Dec 2020 23:11:07 +0100 Subject: [PATCH 12/73] Don't use tokio-codec in the ipc server --- ipc/Cargo.toml | 1 + ipc/src/server.rs | 28 ++++++++++++++++------------ server-utils/Cargo.toml | 2 +- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/ipc/Cargo.toml b/ipc/Cargo.toml index 0f1ddb6f0..0a419c44f 100644 --- a/ipc/Cargo.toml +++ b/ipc/Cargo.toml @@ -25,6 +25,7 @@ lazy_static = "1.0" [target.'cfg(not(windows))'.dev-dependencies] tokio-uds = "0.2" +tokio02 = { package = "tokio", version = "0.2", default-features = false, features = ["uds"] } [badges] travis-ci = { repository = "paritytech/jsonrpc", branch = "master"} diff --git a/ipc/src/server.rs b/ipc/src/server.rs index b6df6fd68..0b4854ab5 100644 --- a/ipc/src/server.rs +++ b/ipc/src/server.rs @@ -339,10 +339,9 @@ impl CloseHandle { mod tests { use super::*; - use futures01::{Future, Sink, Stream}; + use futures01::{Future, Stream}; use jsonrpc_core::Value; use jsonrpc_server_utils::tokio::{self, timer::Delay}; - use jsonrpc_server_utils::tokio_codec::Decoder; use std::thread; use std::time::{self, Duration, Instant}; use tokio_uds::UnixStream; @@ -360,17 +359,22 @@ mod tests { } fn dummy_request_str(path: &str, data: &str) -> String { - let stream_future = UnixStream::connect(path); - let reply = stream_future.and_then(|stream| { - let stream = codecs::StreamCodec::stream_incoming().framed(stream); - let reply = stream - .send(data.to_owned()) - .and_then(move |stream| stream.into_future().map_err(|(err, _)| err)) - .and_then(|(reply, _)| future::ok(reply.expect("there should be one reply"))); - reply - }); + use futures03::{StreamExt, SinkExt}; + + let reply = async move { + use tokio02::net::UnixStream; + + let stream: UnixStream = UnixStream::connect(path).await?; + let codec = codecs::StreamCodec::stream_incoming(); + let mut stream = tokio_util::codec::Decoder::framed(codec, stream); + stream.send(data.to_owned()).await?; + let (reply, _) = stream.into_future().await; + + reply.expect("there should be one reply") + }; - reply.wait().expect("wait for reply") + let mut rt = tokio02::runtime::Runtime::new().unwrap(); + rt.block_on(reply).expect("wait for reply") } #[test] diff --git a/server-utils/Cargo.toml b/server-utils/Cargo.toml index f6314b150..cf1b0484b 100644 --- a/server-utils/Cargo.toml +++ b/server-utils/Cargo.toml @@ -18,7 +18,7 @@ globset = "0.4" jsonrpc-core = { version = "16.0", path = "../core" } lazy_static = "1.1.0" log = "0.4" -tokio = { version = "0.1.15", default-features = false, features = ["timer", "tcp"] } +tokio = { version = "0.1.15", default-features = false, features = ["timer", "tcp", "reactor", "rt-full"] } tokio-codec = { version = "0.1", optional = true } tokio-util = { version = "0.3", features = ["codec", "compat"] } tokio-compat = { version = "0.1" } From 22a4323fd0c7375a23997555fe31e9a045c4bd51 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Tue, 8 Dec 2020 23:36:19 +0100 Subject: [PATCH 13/73] Don't explicitly depend on futures01 in server-utils --- server-utils/Cargo.toml | 2 +- server-utils/src/reactor.rs | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/server-utils/Cargo.toml b/server-utils/Cargo.toml index cf1b0484b..7d3778a8e 100644 --- a/server-utils/Cargo.toml +++ b/server-utils/Cargo.toml @@ -13,7 +13,7 @@ version = "16.0.0" [dependencies] bytes04 = { version = "0.4", package = "bytes" } bytes05 = { version = "0.5", package = "bytes" } -futures01 = { version = "0.1", package = "futures" } +futures03 = { version = "0.3", package = "futures" } globset = "0.4" jsonrpc-core = { version = "16.0", path = "../core" } lazy_static = "1.1.0" diff --git a/server-utils/src/reactor.rs b/server-utils/src/reactor.rs index d30d1df5e..d6f75fd32 100644 --- a/server-utils/src/reactor.rs +++ b/server-utils/src/reactor.rs @@ -7,7 +7,7 @@ use std::io; -use futures01::Future; +use tokio::prelude::Future; /// Possibly uninitialized event loop executor. #[derive(Debug)] @@ -85,7 +85,7 @@ impl Executor { #[derive(Debug)] pub struct RpcEventLoop { executor: tokio_compat::runtime::TaskExecutor, - close: Option>, + close: Option>, runtime: Option, } @@ -103,7 +103,7 @@ impl RpcEventLoop { /// Spawns a new named thread with the `EventLoop`. pub fn with_name(name: Option) -> io::Result { - let (stop, stopped) = futures01::oneshot(); + let (stop, stopped) = futures03::channel::oneshot::channel(); let mut tb = tokio_compat::runtime::Builder::new(); tb.core_threads(1); @@ -114,8 +114,7 @@ impl RpcEventLoop { let runtime = tb.build()?; let executor = runtime.executor(); - let terminate = futures01::empty().select(stopped).map(|_| ()).map_err(|_| ()); - runtime.spawn(terminate); + runtime.spawn_std(async { let _ = stopped.await; }); Ok(RpcEventLoop { executor, From 53241d410493d55a254bb0bf722e211b75c7b263 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Tue, 8 Dec 2020 23:59:13 +0100 Subject: [PATCH 14/73] Make tokio optional in server-utils --- http/Cargo.toml | 2 +- ipc/Cargo.toml | 2 +- ipc/src/server.rs | 2 +- server-utils/Cargo.toml | 3 ++- server-utils/src/lib.rs | 3 +++ server-utils/src/reactor.rs | 14 +------------- tcp/Cargo.toml | 2 +- tcp/src/server.rs | 2 +- 8 files changed, 11 insertions(+), 19 deletions(-) diff --git a/http/Cargo.toml b/http/Cargo.toml index fdccef242..1afb6f0a4 100644 --- a/http/Cargo.toml +++ b/http/Cargo.toml @@ -15,7 +15,7 @@ futures01 = { version = "0.1", package = "futures" } futures03 = { version = "0.3", package = "futures", features = ["compat"] } hyper = "0.12" jsonrpc-core = { version = "16.0", path = "../core" } -jsonrpc-server-utils = { version = "16.0", path = "../server-utils" } +jsonrpc-server-utils = { version = "16.0", path = "../server-utils", features = ["tokio"] } log = "0.4" net2 = "0.2" parking_lot = "0.11.0" diff --git a/ipc/Cargo.toml b/ipc/Cargo.toml index 0a419c44f..440c7deaf 100644 --- a/ipc/Cargo.toml +++ b/ipc/Cargo.toml @@ -15,7 +15,7 @@ futures03 = { version = "0.3", package = "futures", features = [ "compat" ] } log = "0.4" tower-service = "0.3" jsonrpc-core = { version = "16.0", path = "../core" } -jsonrpc-server-utils = { version = "16.0", path = "../server-utils" } +jsonrpc-server-utils = { version = "16.0", path = "../server-utils", features = ["tokio"] } parity-tokio-ipc = "0.7" parking_lot = "0.11.0" diff --git a/ipc/src/server.rs b/ipc/src/server.rs index 0b4854ab5..7ccab4a31 100644 --- a/ipc/src/server.rs +++ b/ipc/src/server.rs @@ -167,7 +167,7 @@ where let security_attributes = self.security_attributes; let client_buffer_size = self.client_buffer_size; - executor.spawn(future::lazy(move || { + executor.executor().spawn(future::lazy(move || { let mut endpoint = Endpoint::new(endpoint_addr); endpoint.set_security_attributes(security_attributes); diff --git a/server-utils/Cargo.toml b/server-utils/Cargo.toml index 7d3778a8e..e3d14853b 100644 --- a/server-utils/Cargo.toml +++ b/server-utils/Cargo.toml @@ -13,12 +13,13 @@ version = "16.0.0" [dependencies] bytes04 = { version = "0.4", package = "bytes" } bytes05 = { version = "0.5", package = "bytes" } +futures01 = { version = "0.1", package = "futures" } futures03 = { version = "0.3", package = "futures" } globset = "0.4" jsonrpc-core = { version = "16.0", path = "../core" } lazy_static = "1.1.0" log = "0.4" -tokio = { version = "0.1.15", default-features = false, features = ["timer", "tcp", "reactor", "rt-full"] } +tokio = { version = "0.1.15", default-features = false, features = ["timer", "tcp", "reactor", "rt-full"], optional = true } tokio-codec = { version = "0.1", optional = true } tokio-util = { version = "0.3", features = ["codec", "compat"] } tokio-compat = { version = "0.1" } diff --git a/server-utils/src/lib.rs b/server-utils/src/lib.rs index f1f6a62d3..0f40d6c85 100644 --- a/server-utils/src/lib.rs +++ b/server-utils/src/lib.rs @@ -8,6 +8,7 @@ extern crate log; #[macro_use] extern crate lazy_static; +#[cfg(feature = "tokio")] pub use tokio; pub use tokio_util; pub use tokio_compat; @@ -20,9 +21,11 @@ mod matcher; pub mod reactor; pub mod session; mod stream_codec; +#[cfg(feature = "tokio")] mod suspendable_stream; pub use crate::matcher::Pattern; +#[cfg(feature = "tokio")] pub use crate::suspendable_stream::SuspendableStream; /// Codecs utilities diff --git a/server-utils/src/reactor.rs b/server-utils/src/reactor.rs index d6f75fd32..a054b24c4 100644 --- a/server-utils/src/reactor.rs +++ b/server-utils/src/reactor.rs @@ -7,8 +7,6 @@ use std::io; -use tokio::prelude::Future; - /// Possibly uninitialized event loop executor. #[derive(Debug)] pub enum UninitializedExecutor { @@ -55,17 +53,6 @@ impl Executor { } } - /// Spawn a future onto the Tokio runtime. - pub fn spawn(&self, future: F) - where - F: Future + Send + 'static, - { - match self { - Executor::Shared(exe) => exe.spawn(future), - Executor::Spawned(eloop) => eloop.executor().spawn(future), - } - } - /// Closes underlying event loop (if any!). pub fn close(self) { if let Executor::Spawned(eloop) = self { @@ -130,6 +117,7 @@ impl RpcEventLoop { /// Blocks current thread and waits until the event loop is finished. pub fn wait(mut self) -> Result<(), ()> { + use futures01::Future; self.runtime.take().ok_or(())?.shutdown_on_idle().wait() } diff --git a/tcp/Cargo.toml b/tcp/Cargo.toml index 2fbcfe41a..66b5d280d 100644 --- a/tcp/Cargo.toml +++ b/tcp/Cargo.toml @@ -14,7 +14,7 @@ futures01 = { version = "0.1", package = "futures" } # TODO remove when we no longer need compat (use jsonrpc-core re-export instead) futures03 = { version = "0.3", features = ["compat"], package = "futures" } jsonrpc-core = { version = "16.0", path = "../core" } -jsonrpc-server-utils = { version = "16.0", path = "../server-utils", features = ["tokio-codec"] } +jsonrpc-server-utils = { version = "16.0", path = "../server-utils", features = ["tokio", "tokio-codec"] } log = "0.4" parking_lot = "0.11.0" tower-service = "0.3" diff --git a/tcp/src/server.rs b/tcp/src/server.rs index 84b26b129..157c12c7c 100644 --- a/tcp/src/server.rs +++ b/tcp/src/server.rs @@ -91,7 +91,7 @@ where let executor = self.executor.initialize()?; - executor.spawn(future::lazy(move || { + executor.executor().spawn(future::lazy(move || { let start = move || { let listener = tokio::net::TcpListener::bind(&address)?; let connections = SuspendableStream::new(listener.incoming()); From a6d31f08cbd5d28a839292137af290639d232d96 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Wed, 9 Dec 2020 13:52:27 +0100 Subject: [PATCH 15/73] WIP: Prepare to spawn a 0.3 future in IPC server --- ipc/src/server.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ipc/src/server.rs b/ipc/src/server.rs index 7ccab4a31..e73e01cd0 100644 --- a/ipc/src/server.rs +++ b/ipc/src/server.rs @@ -167,7 +167,7 @@ where let security_attributes = self.security_attributes; let client_buffer_size = self.client_buffer_size; - executor.executor().spawn(future::lazy(move || { + let fut = future::lazy(move || { let mut endpoint = Endpoint::new(endpoint_addr); endpoint.set_security_attributes(security_attributes); @@ -255,7 +255,11 @@ where }) .map_err(|_| ()), ) - })); + }); + use futures03::compat::Future01CompatExt; + use futures03::FutureExt; + let fut = Box::pin(fut.compat().map(drop)); + executor.executor().spawn_std(fut); let handle = InnerHandles { executor: Some(executor), From ffe6da7c56e7954027cfce42747ce5f483d523f5 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Wed, 9 Dec 2020 14:54:58 +0100 Subject: [PATCH 16/73] Simplify service.call in IPC --- ipc/src/server.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ipc/src/server.rs b/ipc/src/server.rs index e73e01cd0..5e6aa104b 100644 --- a/ipc/src/server.rs +++ b/ipc/src/server.rs @@ -216,11 +216,9 @@ where .compat() .map(move |req| { use futures03::TryFutureExt; - service - .call(req) + service.call(req) + .map_err(|()| std::io::ErrorKind::Other.into()) .compat() - .then(|result| future::ok::<_, ()>(result.unwrap_or(None))) - .map_err(|_: ()| std::io::ErrorKind::Other.into()) }) .buffer_unordered(client_buffer_size) .filter_map(|x| x) From 5e2921b7d100daf8fbf096e9d782fe903e43196e Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Wed, 9 Dec 2020 15:20:02 +0100 Subject: [PATCH 17/73] WIP: Alow to opt out of Tokio 0.1 in server-utils --- server-utils/Cargo.toml | 8 +++- server-utils/src/lib.rs | 3 ++ server-utils/src/reactor.rs | 86 +++++++++++++++++++++++++++++++------ 3 files changed, 83 insertions(+), 14 deletions(-) diff --git a/server-utils/Cargo.toml b/server-utils/Cargo.toml index e3d14853b..8b481e3fe 100644 --- a/server-utils/Cargo.toml +++ b/server-utils/Cargo.toml @@ -22,8 +22,14 @@ log = "0.4" tokio = { version = "0.1.15", default-features = false, features = ["timer", "tcp", "reactor", "rt-full"], optional = true } tokio-codec = { version = "0.1", optional = true } tokio-util = { version = "0.3", features = ["codec", "compat"] } -tokio-compat = { version = "0.1" } +tokio-compat = { version = "0.1", optional = true } +tokio02 = { package = "tokio", version = "0.2", optional = true, features = ["rt-threaded", "io-driver"] } + unicase = "2.0" +[features] +default = ["tokio-compat"] +# tokio-compat and tokio02 are mutually-exclusive + [badges] travis-ci = { repository = "paritytech/jsonrpc", branch = "master"} diff --git a/server-utils/src/lib.rs b/server-utils/src/lib.rs index 0f40d6c85..1596afc44 100644 --- a/server-utils/src/lib.rs +++ b/server-utils/src/lib.rs @@ -11,7 +11,10 @@ extern crate lazy_static; #[cfg(feature = "tokio")] pub use tokio; pub use tokio_util; +#[cfg(feature = "tokio-compat")] pub use tokio_compat; +#[cfg(feature = "tokio02")] +pub use tokio02; #[cfg(feature = "tokio-codec")] pub use tokio_codec; diff --git a/server-utils/src/reactor.rs b/server-utils/src/reactor.rs index a054b24c4..7e98eea0d 100644 --- a/server-utils/src/reactor.rs +++ b/server-utils/src/reactor.rs @@ -7,11 +7,22 @@ use std::io; +#[cfg(feature = "tokio-compat")] +use tokio_compat::runtime; +#[cfg(feature = "tokio-compat")] +/// Task executor for Tokio-compat runtime. +pub type TaskExecutor = tokio_compat::runtime::TaskExecutor; +#[cfg(feature = "tokio02")] +use tokio02::runtime; +#[cfg(feature = "tokio02")] +/// Task executor for Tokio 0.2 runtime. +pub type TaskExecutor = tokio02::runtime::Handle; + /// Possibly uninitialized event loop executor. #[derive(Debug)] pub enum UninitializedExecutor { /// Shared instance of executor. - Shared(tokio_compat::runtime::TaskExecutor), + Shared(TaskExecutor), /// Event Loop should be spawned by the transport. Unspawned, } @@ -39,14 +50,24 @@ impl UninitializedExecutor { #[derive(Debug)] pub enum Executor { /// Shared instance - Shared(tokio_compat::runtime::TaskExecutor), + Shared(TaskExecutor), /// Spawned Event Loop Spawned(RpcEventLoop), } impl Executor { + #[cfg(feature = "tokio-compat")] + /// Get tokio executor associated with this event loop. + pub fn executor(&self) -> TaskExecutor { + match self { + Executor::Shared(ref executor) => executor.clone(), + Executor::Spawned(ref eloop) => eloop.executor(), + } + } + + #[cfg(feature = "tokio02")] /// Get tokio executor associated with this event loop. - pub fn executor(&self) -> tokio_compat::runtime::TaskExecutor { + pub fn executor(&self) -> TaskExecutor { match self { Executor::Shared(ref executor) => executor.clone(), Executor::Spawned(ref eloop) => eloop.executor(), @@ -71,9 +92,9 @@ impl Executor { /// A handle to running event loop. Dropping the handle will cause event loop to finish. #[derive(Debug)] pub struct RpcEventLoop { - executor: tokio_compat::runtime::TaskExecutor, + executor: TaskExecutor, close: Option>, - runtime: Option, + runtime: Option, } impl Drop for RpcEventLoop { @@ -92,16 +113,35 @@ impl RpcEventLoop { pub fn with_name(name: Option) -> io::Result { let (stop, stopped) = futures03::channel::oneshot::channel(); - let mut tb = tokio_compat::runtime::Builder::new(); + let mut tb = runtime::Builder::new(); tb.core_threads(1); - if let Some(name) = name { - tb.name_prefix(name); + #[cfg(feature = "tokio02")] + { + tb.threaded_scheduler(); + tb.enable_io(); } let runtime = tb.build()?; - let executor = runtime.executor(); - runtime.spawn_std(async { let _ = stopped.await; }); + let executor; + #[cfg(feature = "tokio-compat")] + { + executor = runtime.executor(); + if let Some(name) = name { + tb.name_prefix(name); + } + + runtime.spawn_std(async { let _ = stopped.await; }); + } + #[cfg(feature = "tokio02")] + { + executor = runtime.handle().clone(); + if let Some(name) = name { + tb.thread_name(name); + } + + runtime.spawn(async { let _ = stopped.await; }); + } Ok(RpcEventLoop { executor, @@ -111,14 +151,34 @@ impl RpcEventLoop { } /// Get executor for this event loop. - pub fn executor(&self) -> tokio_compat::runtime::TaskExecutor { + #[cfg(feature = "tokio-compat")] + pub fn executor(&self) -> TaskExecutor { self.executor.clone() } + /// Get executor for this event loop. + #[cfg(feature = "tokio02")] + pub fn executor(&self) -> runtime::Handle { + self.runtime.as_ref() + .expect("Runtime is only None if we're being dropped; qed") + .handle() + .clone() + } + /// Blocks current thread and waits until the event loop is finished. pub fn wait(mut self) -> Result<(), ()> { - use futures01::Future; - self.runtime.take().ok_or(())?.shutdown_on_idle().wait() + #[cfg(feature = "tokio-compat")] + { + use futures01::Future; + self.runtime.take().ok_or(())?.shutdown_on_idle().wait() + } + #[cfg(feature = "tokio02")] + { + // Dropping Tokio 0.2 runtime waits for spawned task to shutdown by default + let runtime = self.runtime.take().ok_or(())?; + drop(runtime); + Ok(()) + } } /// Finishes this event loop. From ffc2fca7323e9e5f8114371c880b9a615c0e31a6 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Wed, 9 Dec 2020 17:17:49 +0100 Subject: [PATCH 18/73] Migrate IPC server to Tokio 0.2 --- ipc/Cargo.toml | 4 ++-- ipc/src/lib.rs | 2 +- ipc/src/server.rs | 31 ++++++++++++++----------------- 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/ipc/Cargo.toml b/ipc/Cargo.toml index 440c7deaf..7ff9292e1 100644 --- a/ipc/Cargo.toml +++ b/ipc/Cargo.toml @@ -15,7 +15,7 @@ futures03 = { version = "0.3", package = "futures", features = [ "compat" ] } log = "0.4" tower-service = "0.3" jsonrpc-core = { version = "16.0", path = "../core" } -jsonrpc-server-utils = { version = "16.0", path = "../server-utils", features = ["tokio"] } +jsonrpc-server-utils = { version = "16.0", path = "../server-utils", default-features = false, features = ["tokio02"] } parity-tokio-ipc = "0.7" parking_lot = "0.11.0" @@ -25,7 +25,7 @@ lazy_static = "1.0" [target.'cfg(not(windows))'.dev-dependencies] tokio-uds = "0.2" -tokio02 = { package = "tokio", version = "0.2", default-features = false, features = ["uds"] } +tokio02 = { package = "tokio", version = "0.2", default-features = false, features = ["uds", "time", "rt-threaded", "io-driver"] } [badges] travis-ci = { repository = "paritytech/jsonrpc", branch = "master"} diff --git a/ipc/src/lib.rs b/ipc/src/lib.rs index ab9e73130..5cfe374ad 100644 --- a/ipc/src/lib.rs +++ b/ipc/src/lib.rs @@ -26,4 +26,4 @@ pub use crate::meta::{MetaExtractor, NoopExtractor, RequestContext}; pub use crate::server::{CloseHandle, SecurityAttributes, Server, ServerBuilder}; pub use self::server_utils::session::{SessionId, SessionStats}; -pub use self::server_utils::{codecs::Separator, tokio}; +pub use self::server_utils::{codecs::Separator}; diff --git a/ipc/src/server.rs b/ipc/src/server.rs index 5e6aa104b..4566ca717 100644 --- a/ipc/src/server.rs +++ b/ipc/src/server.rs @@ -14,8 +14,7 @@ use tower_service::Service as _; use crate::server_utils::{ codecs, reactor, session, - tokio::reactor::Handle, - tokio_compat::runtime::TaskExecutor, + reactor::TaskExecutor, tokio_util, }; @@ -61,7 +60,6 @@ pub struct ServerBuilder = middleware::Noop> meta_extractor: Arc>, session_stats: Option>, executor: reactor::UninitializedExecutor, - reactor: Option, incoming_separator: codecs::Separator, outgoing_separator: codecs::Separator, security_attributes: SecurityAttributes, @@ -98,7 +96,6 @@ where meta_extractor: Arc::new(extractor), session_stats: None, executor: reactor::UninitializedExecutor::Unspawned, - reactor: None, incoming_separator: codecs::Separator::Empty, outgoing_separator: codecs::Separator::default(), security_attributes: SecurityAttributes::empty(), @@ -112,12 +109,6 @@ where self } - /// Sets different event loop I/O reactor. - pub fn event_loop_reactor(mut self, reactor: Handle) -> Self { - self.reactor = Some(reactor); - self - } - /// Sets session metadata extractor. pub fn session_meta_extractor(mut self, meta_extractor: X) -> Self where @@ -257,7 +248,7 @@ where use futures03::compat::Future01CompatExt; use futures03::FutureExt; let fut = Box::pin(fut.compat().map(drop)); - executor.executor().spawn_std(fut); + executor.executor().spawn(fut); let handle = InnerHandles { executor: Some(executor), @@ -343,9 +334,8 @@ mod tests { use futures01::{Future, Stream}; use jsonrpc_core::Value; - use jsonrpc_server_utils::tokio::{self, timer::Delay}; use std::thread; - use std::time::{self, Duration, Instant}; + use std::time::{self, Duration}; use tokio_uds::UnixStream; fn server_builder() -> ServerBuilder { @@ -617,9 +607,14 @@ mod tests { tx.send(true).expect("failed to report that the server has stopped"); }); - let delay = Delay::new(Instant::now() + Duration::from_millis(500)) - .map(|_| false) - .map_err(|err| panic!("{:?}", err)); + use futures03::FutureExt; + use futures03::TryFutureExt; + let delay = futures01::future::lazy(|| + // Lazily bound to Tokio timer instance + tokio02::time::delay_for(Duration::from_millis(500)) + .map(|_| Ok(false)) + .compat() + ); let result_fut = rx.map_err(|_| ()).select(delay).then(move |result| match result { Ok((result, _)) => { @@ -633,7 +628,9 @@ mod tests { Err(_) => Err(()), }); - tokio::run(result_fut); + use futures03::compat::Future01CompatExt; + let mut rt = tokio02::runtime::Runtime::new().unwrap(); + rt.block_on(result_fut.compat()).unwrap(); } #[test] From 9387f858bd271cb9b3fd724fdbd8e2b8643680f7 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Wed, 16 Dec 2020 13:12:55 +0100 Subject: [PATCH 19/73] Adapt ipc client transport to Tokio 0.2 --- core-client/transports/Cargo.toml | 10 +- core-client/transports/src/transports/ipc.rs | 118 ++++++++----------- 2 files changed, 54 insertions(+), 74 deletions(-) diff --git a/core-client/transports/Cargo.toml b/core-client/transports/Cargo.toml index 8d83e403a..58c7575d5 100644 --- a/core-client/transports/Cargo.toml +++ b/core-client/transports/Cargo.toml @@ -29,9 +29,7 @@ ws = [ ] ipc = [ "parity-tokio-ipc", - "jsonrpc-server-utils", - "tokio", - "futures01", + "jsonrpc-server-utils/tokio02", ] arbitrary_precision = ["serde_json/arbitrary_precision", "jsonrpc-core/arbitrary_precision"] @@ -48,17 +46,19 @@ url = "1.7" futures01 = { version = "0.1.26", package = "futures", optional = true } hyper = { version = "0.12", optional = true } hyper-tls = { version = "0.3.2", optional = true } -jsonrpc-server-utils = { version = "16.0", path = "../../server-utils", optional = true } +jsonrpc-server-utils = { version = "16.0", path = "../../server-utils", optional = true, default-features = false } parity-tokio-ipc = { version = "0.7", optional = true } tokio = { version = "0.1", optional = true } websocket = { version = "0.24", optional = true } [dev-dependencies] assert_matches = "1.1" -jsonrpc-http-server = { version = "16.0", path = "../../http" } +# FIXME: Uncomment once http migrates to Tokio 0.2 +# jsonrpc-http-server = { version = "16.0", path = "../../http" } jsonrpc-ipc-server = { version = "16.0", path = "../../ipc" } lazy_static = "1.0" env_logger = "0.7" +tokio02 = { package = "tokio", version = "0.2", features = ["macros"] } [badges] travis-ci = { repository = "paritytech/jsonrpc", branch = "master" } diff --git a/core-client/transports/src/transports/ipc.rs b/core-client/transports/src/transports/ipc.rs index 4ad2ff429..ba608bc7d 100644 --- a/core-client/transports/src/transports/ipc.rs +++ b/core-client/transports/src/transports/ipc.rs @@ -3,42 +3,34 @@ use crate::transports::duplex::duplex; use crate::{RpcChannel, RpcError}; -use futures::compat::{Sink01CompatExt, Stream01CompatExt}; -use futures::StreamExt; -use futures01::prelude::*; +use futures::{SinkExt, StreamExt, TryStreamExt}; use jsonrpc_server_utils::codecs::StreamCodec; -use parity_tokio_ipc::IpcConnection; +use jsonrpc_server_utils::tokio_util::codec::Decoder as _; +use jsonrpc_server_utils::tokio02 as tokio02; +use parity_tokio_ipc::Endpoint; use std::path::Path; -use tokio::codec::Decoder; -use tokio::runtime::Runtime; /// Connect to a JSON-RPC IPC server. -pub fn connect, Client: From>( +pub async fn connect, Client: From>( path: P, -) -> impl futures::Future> { - let rt = Runtime::new().unwrap(); - #[allow(deprecated)] - let reactor = rt.reactor().clone(); - async move { - let connection = IpcConnection::connect(path, &reactor).map_err(|e| RpcError::Other(e.into()))?; +) -> Result { + let connection = Endpoint::connect(path).await.map_err(|e| RpcError::Other(Box::new(e)))?; let (sink, stream) = StreamCodec::stream_incoming().framed(connection).split(); - let sink = sink.sink_map_err(|e| RpcError::Other(e.into())); + let sink = sink.sink_map_err(|e| RpcError::Other(Box::new(e))); let stream = stream.map_err(|e| log::error!("IPC stream error: {}", e)); let (client, sender) = duplex( - Box::pin(sink.sink_compat()), + Box::pin(sink), Box::pin( stream - .compat() .take_while(|x| futures::future::ready(x.is_ok())) .map(|x| x.expect("Stream is closed upon first error.")), ), ); - tokio::spawn(futures::compat::Compat::new(client).map_err(|_| unreachable!())); + tokio02::spawn(client); Ok(sender.into()) - } } #[cfg(test)] @@ -52,13 +44,10 @@ mod tests { #[test] fn should_call_one() { - let mut rt = Runtime::new().unwrap(); - #[allow(deprecated)] - let reactor = rt.reactor().clone(); let sock_path = dummy_endpoint(); let mut io = IoHandler::new(); - io.add_method("greeting", |params| { + io.add_method("greeting", |params| async { let map_obj = match params { Params::Map(obj) => obj, _ => return Err(Error::invalid_params("missing object")), @@ -69,66 +58,57 @@ mod tests { }; Ok(Value::String(format!("Hello {}!", name))) }); - let builder = ServerBuilder::new(io).event_loop_executor(rt.executor()); - let server = builder.start(&sock_path).expect("Couldn't open socket"); - - let client: RawClient = rt.block_on(connect(sock_path, &reactor).unwrap()).unwrap(); - let mut map = Map::new(); - map.insert("name".to_string(), "Jeffry".into()); - let fut = client.call_method("greeting", Params::Map(map)); - - // FIXME: it seems that IPC server on Windows won't be polled with - // default I/O reactor, work around with sending stop signal which polls - // the server (https://github.com/paritytech/jsonrpc/pull/459) - server.close(); - - match rt.block_on(fut) { - Ok(val) => assert_eq!(&val, "Hello Jeffry!"), - Err(err) => panic!("IPC RPC call failed: {}", err), - } - rt.shutdown_now().wait().unwrap(); + let builder = ServerBuilder::new(io); + let _server = builder.start(&sock_path).expect("Couldn't open socket"); + + let client_fut = async move { + let client: RawClient = connect(sock_path).await.unwrap(); + let mut map = Map::new(); + map.insert("name".to_string(), "Jeffry".into()); + let fut = client.call_method("greeting", Params::Map(map)); + + match fut.await { + Ok(val) => assert_eq!(&val, "Hello Jeffry!"), + Err(err) => panic!("IPC RPC call failed: {}", err), + } + }; + tokio02::runtime::Runtime::new().unwrap().block_on(client_fut); } #[test] fn should_fail_without_server() { - let rt = Runtime::new().unwrap(); - #[allow(deprecated)] - let reactor = rt.reactor(); - - match connect::<_, RawClient>(dummy_endpoint(), reactor) { + let test_fut = async move { + match connect::<_, RawClient>(dummy_endpoint()).await { Err(..) => {} Ok(..) => panic!("Should not be able to connect to an IPC socket that's not open"), } - rt.shutdown_now().wait().unwrap(); + }; + + tokio02::runtime::Runtime::new().unwrap().block_on(test_fut); } #[test] fn should_handle_server_error() { - let mut rt = Runtime::new().unwrap(); - #[allow(deprecated)] - let reactor = rt.reactor().clone(); let sock_path = dummy_endpoint(); let mut io = IoHandler::new(); - io.add_method("greeting", |_params| Err(Error::invalid_params("test error"))); - let builder = ServerBuilder::new(io).event_loop_executor(rt.executor()); - let server = builder.start(&sock_path).expect("Couldn't open socket"); - - let client: RawClient = rt.block_on(connect(sock_path, &reactor).unwrap()).unwrap(); - let mut map = Map::new(); - map.insert("name".to_string(), "Jeffry".into()); - let fut = client.call_method("greeting", Params::Map(map)); - - // FIXME: it seems that IPC server on Windows won't be polled with - // default I/O reactor, work around with sending stop signal which polls - // the server (https://github.com/paritytech/jsonrpc/pull/459) - server.close(); - - match rt.block_on(fut) { - Err(RpcError::JsonRpcError(err)) => assert_eq!(err.code, ErrorCode::InvalidParams), - Ok(_) => panic!("Expected the call to fail"), - _ => panic!("Unexpected error type"), - } - rt.shutdown_now().wait().unwrap(); + io.add_method("greeting", |_params| async { Err(Error::invalid_params("test error")) }); + let builder = ServerBuilder::new(io); + let _server = builder.start(&sock_path).expect("Couldn't open socket"); + + let client_fut = async move { + let client: RawClient = connect(sock_path).await.unwrap(); + let mut map = Map::new(); + map.insert("name".to_string(), "Jeffry".into()); + let fut = client.call_method("greeting", Params::Map(map)); + + match fut.await { + Err(RpcError::JsonRpcError(err)) => assert_eq!(err.code, ErrorCode::InvalidParams), + Ok(_) => panic!("Expected the call to fail"), + _ => panic!("Unexpected error type"), + } + }; + + tokio02::runtime::Runtime::new().unwrap().block_on(client_fut); } } From 7ceda6926101e42e1938925988f9135fac91ca8a Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Thu, 17 Dec 2020 17:59:40 +0100 Subject: [PATCH 20/73] Adapt ws client to Tokio 0.2 --- core-client/transports/Cargo.toml | 3 ++- core-client/transports/src/transports/ws.rs | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core-client/transports/Cargo.toml b/core-client/transports/Cargo.toml index 58c7575d5..f40b3760d 100644 --- a/core-client/transports/Cargo.toml +++ b/core-client/transports/Cargo.toml @@ -24,7 +24,7 @@ tls = ["hyper-tls", "http"] http = ["hyper", "futures01"] ws = [ "websocket", - "tokio", + "tokio02", "futures01", ] ipc = [ @@ -50,6 +50,7 @@ jsonrpc-server-utils = { version = "16.0", path = "../../server-utils", optional parity-tokio-ipc = { version = "0.7", optional = true } tokio = { version = "0.1", optional = true } websocket = { version = "0.24", optional = true } +tokio02 = { package = "tokio", version = "0.2", features = ["macros"], optional = true } [dev-dependencies] assert_matches = "1.1" diff --git a/core-client/transports/src/transports/ws.rs b/core-client/transports/src/transports/ws.rs index d74d5df44..e424b938e 100644 --- a/core-client/transports/src/transports/ws.rs +++ b/core-client/transports/src/transports/ws.rs @@ -49,8 +49,8 @@ where ), ); let (rpc_client, sender) = super::duplex(sink, stream); - let rpc_client = rpc_client.compat().map_err(|error| log::error!("{:?}", error)); - tokio::spawn(rpc_client); + let rpc_client = rpc_client.map_err(|error| log::error!("{:?}", error)); + tokio02::spawn(rpc_client); sender.into() }) .map_err(|error| RpcError::Other(Box::new(error))) From 6009b185ab6a6033381d4aa8877d9c4d5bad2ef4 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Thu, 17 Dec 2020 20:09:54 +0100 Subject: [PATCH 21/73] WIP: Migrate http server to Tokio 0.2 part 1 --- http/Cargo.toml | 4 +- http/src/handler.rs | 40 +++++++-- http/src/lib.rs | 75 +++++++++++------ server-utils/Cargo.toml | 4 +- server-utils/src/lib.rs | 4 +- server-utils/src/suspendable_stream.rs | 109 +++++++++++++++++++------ 6 files changed, 178 insertions(+), 58 deletions(-) diff --git a/http/Cargo.toml b/http/Cargo.toml index 1afb6f0a4..d4d146077 100644 --- a/http/Cargo.toml +++ b/http/Cargo.toml @@ -13,9 +13,9 @@ version = "16.0.0" [dependencies] futures01 = { version = "0.1", package = "futures" } futures03 = { version = "0.3", package = "futures", features = ["compat"] } -hyper = "0.12" +hyper = "0.13" jsonrpc-core = { version = "16.0", path = "../core" } -jsonrpc-server-utils = { version = "16.0", path = "../server-utils", features = ["tokio"] } +jsonrpc-server-utils = { version = "16.0", path = "../server-utils", default-features = false, features = ["tokio02"] } log = "0.4" net2 = "0.2" parking_lot = "0.11.0" diff --git a/http/src/handler.rs b/http/src/handler.rs index b6abb796f..15613f9a8 100644 --- a/http/src/handler.rs +++ b/http/src/handler.rs @@ -2,6 +2,7 @@ use crate::WeakRpc; use std::sync::Arc; use std::{fmt, mem, str}; +use std::{pin::Pin, task}; use hyper::header::{self, HeaderMap, HeaderValue}; use hyper::{self, service::Service, Body, Method}; @@ -10,7 +11,7 @@ use crate::jsonrpc::serde_json; use crate::jsonrpc::{self as core, middleware, Metadata, Middleware}; use crate::response::Response; use crate::server_utils::cors; -use futures01::{Async, Future, Poll, Stream}; +use futures01::{Async, Future, Poll}; use crate::{utils, AllowedHosts, CorsDomains, RequestMiddleware, RequestMiddlewareAction, RestApi}; @@ -57,17 +58,22 @@ impl> ServerHandler { } } -impl> Service for ServerHandler +impl> Service> for ServerHandler where S::Future: Unpin, S::CallFuture: Unpin, { - type ReqBody = Body; - type ResBody = Body; + type Response = hyper::Response; + // type ReqBody = Body; + // type ResBody = Body; type Error = hyper::Error; type Future = Handler; - fn call(&mut self, request: hyper::Request) -> Self::Future { + fn poll_ready(&mut self, cx: &mut task::Context<'_>) -> task::Poll> { + todo!() + } + + fn call(&mut self, request: hyper::Request) -> Self::Future { let is_host_allowed = utils::is_host_allowed(&request, &self.allowed_hosts); let action = self.middleware.on_request(request); @@ -143,6 +149,30 @@ where } } +impl> std::future::Future for Handler +where + S::Future: Unpin, + S::CallFuture: Unpin, +{ + type Output = hyper::Result>; + // type Error = hyper::Error; + + fn poll(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> task::Poll { + use std::future::Future as StdFuture; + use futures03::compat::Future01CompatExt; + match *self { + Handler::Rpc(ref mut handler) => StdFuture::poll(Pin::new(&mut handler.compat()), cx), + Handler::Middleware(ref mut middleware) => StdFuture::poll(Pin::new(&mut middleware.compat()), cx), + Handler::Err(ref mut response) => std::task::Poll::Ready(Ok( + response + .take() + .expect("Response always Some initialy. Returning `Ready` so will never be polled again; qed") + .into(), + )), + } + } +} + enum RpcPollState { Ready(RpcHandlerState), NotReady(RpcHandlerState), diff --git a/http/src/lib.rs b/http/src/lib.rs index 114129209..5a6f2e1dc 100644 --- a/http/src/lib.rs +++ b/http/src/lib.rs @@ -45,7 +45,9 @@ use parking_lot::Mutex; use crate::jsonrpc::MetaIoHandler; use crate::server_utils::reactor::{Executor, UninitializedExecutor}; use futures01::sync::oneshot; -use futures01::{future, Future, Stream}; +use futures01::{future, Future, + // Stream +}; use hyper::{server, Body}; use jsonrpc_core as jsonrpc; @@ -53,7 +55,8 @@ pub use crate::handler::ServerHandler; pub use crate::response::Response; pub use crate::server_utils::cors::{self, AccessControlAllowOrigin, AllowCors, Origin}; pub use crate::server_utils::hosts::{DomainsValidation, Host}; -pub use crate::server_utils::{tokio, tokio_compat, SuspendableStream}; +pub use crate::server_utils::{tokio02, SuspendableStream}; +pub use crate::server_utils::{reactor::TaskExecutor}; pub use crate::utils::{cors_allow_headers, cors_allow_origin, is_host_allowed}; /// Action undertaken by a middleware. @@ -300,7 +303,7 @@ where /// Utilize existing event loop executor to poll RPC results. /// /// Applies only to 1 of the threads. Other threads will spawn their own Event Loops. - pub fn event_loop_executor(mut self, executor: tokio_compat::runtime::TaskExecutor) -> Self { + pub fn event_loop_executor(mut self, executor: TaskExecutor) -> Self { self.executor = UninitializedExecutor::Shared(executor); self } @@ -519,7 +522,7 @@ fn serve>( mpsc::Sender>, oneshot::Sender<()>, ), - executor: tokio_compat::runtime::TaskExecutor, + executor: TaskExecutor, addr: SocketAddr, cors_domains: CorsDomains, cors_max_age: Option, @@ -536,10 +539,10 @@ fn serve>( S::Future: Unpin, S::CallFuture: Unpin, { + use futures03::compat::Future01CompatExt; + let (shutdown_signal, local_addr_tx, done_tx) = signals; executor.spawn({ - let handle = tokio::reactor::Handle::default(); - let bind = move || { let listener = match addr { SocketAddr::V4(_) => net2::TcpBuilder::new_v4()?, @@ -549,7 +552,7 @@ fn serve>( listener.reuse_address(true)?; listener.bind(&addr)?; let listener = listener.listen(1024)?; - let listener = tokio::net::TcpListener::from_std(listener, &handle)?; + let listener = tokio02::net::TcpListener::from_std(listener)?; // Add current host to allowed headers. // NOTE: we need to use `l.local_addr()` instead of `addr` // it might be different! @@ -587,8 +590,9 @@ fn serve>( let mut http = server::conn::Http::new(); http.keep_alive(keep_alive); let tcp_stream = SuspendableStream::new(listener.incoming()); + use futures03::StreamExt; - tcp_stream + let server = tcp_stream .map(move |socket| { let service = ServerHandler::new( jsonrpc_handler.downgrade(), @@ -603,27 +607,52 @@ fn serve>( keep_alive, ); - tokio::spawn( - http.serve_connection(socket, service) - .map_err(|e| error!("Error serving connection: {:?}", e)) - .then(|_| Ok(())), - ) - }) - .for_each(|_| Ok(())) - .map_err(|e| { - warn!("Incoming streams error, closing sever: {:?}", e); + let connection = http.serve_connection(socket, service) + .map(|res| { + res.map_err(|e| error!("Error serving connection: {:?}", e)) + }); + + tokio02::spawn(connection); }) - .select(shutdown_signal.map_err(|e| { - debug!("Shutdown signaller dropped, closing server: {:?}", e); - })) - .map_err(|_| ()) + .for_each(|_| async { () }); + let shutdown_signal = shutdown_signal.map_err(|e| { + debug!("Shutdown signaller dropped, closing server: {:?}", e); + }); + + use futures03::FutureExt; + use std::future::Future as StdFuture; + + let server: std::pin::Pin + Send>> = Box::pin(server); + let shutdown_signal: std::pin::Pin + Send>> = Box::pin(shutdown_signal.compat().map(|_| ())); + let select = futures03::future::select(server, shutdown_signal); + + futures03::compat::Compat::new(select.map(|x| Ok(x))) + // futures03::future::select(server, shutdown_signal).compat() + // .for_each(|_| async { () }); + // .into_future() + // .map_err(|e| { + // warn!("Incoming streams error, closing sever: {:?}", e); + // }) + // .select(shutdown_signal.map_err(|e| { + // debug!("Shutdown signaller dropped, closing server: {:?}", e); + // })) + // .map_err(|_| ()) }) - .and_then(|(_, server)| { + .and_then(|either| { // We drop the server first to prevent a situation where main thread terminates // before the server is properly dropped (see #504 for more details) - drop(server); + match either { + futures03::future::Either::Left((server_done, shutdown_future)) => { + drop(shutdown_future); + }, + futures03::future::Either::Right((shutdown_signalled, server_future)) => { + drop(server_future); + }, + } + // drop(server); done_tx.send(()) }) + .compat() }); } diff --git a/server-utils/Cargo.toml b/server-utils/Cargo.toml index 8b481e3fe..5e174fc90 100644 --- a/server-utils/Cargo.toml +++ b/server-utils/Cargo.toml @@ -14,7 +14,7 @@ version = "16.0.0" bytes04 = { version = "0.4", package = "bytes" } bytes05 = { version = "0.5", package = "bytes" } futures01 = { version = "0.1", package = "futures" } -futures03 = { version = "0.3", package = "futures" } +futures03 = { version = "0.3", package = "futures", features = ["compat"] } globset = "0.4" jsonrpc-core = { version = "16.0", path = "../core" } lazy_static = "1.1.0" @@ -23,7 +23,7 @@ tokio = { version = "0.1.15", default-features = false, features = ["timer", "tc tokio-codec = { version = "0.1", optional = true } tokio-util = { version = "0.3", features = ["codec", "compat"] } tokio-compat = { version = "0.1", optional = true } -tokio02 = { package = "tokio", version = "0.2", optional = true, features = ["rt-threaded", "io-driver"] } +tokio02 = { package = "tokio", version = "0.2", optional = true, features = ["rt-threaded", "io-driver", "time"] } unicase = "2.0" diff --git a/server-utils/src/lib.rs b/server-utils/src/lib.rs index 1596afc44..60ba0208d 100644 --- a/server-utils/src/lib.rs +++ b/server-utils/src/lib.rs @@ -24,11 +24,11 @@ mod matcher; pub mod reactor; pub mod session; mod stream_codec; -#[cfg(feature = "tokio")] +#[cfg(feature = "tokio02")] mod suspendable_stream; pub use crate::matcher::Pattern; -#[cfg(feature = "tokio")] +#[cfg(feature = "tokio02")] pub use crate::suspendable_stream::SuspendableStream; /// Codecs utilities diff --git a/server-utils/src/suspendable_stream.rs b/server-utils/src/suspendable_stream.rs index f563cdebe..a43dcea9f 100644 --- a/server-utils/src/suspendable_stream.rs +++ b/server-utils/src/suspendable_stream.rs @@ -1,7 +1,7 @@ use std::io; -use std::time::{Duration, Instant}; -use tokio::prelude::*; -use tokio::timer::Delay; +use std::time::Duration; +// use futures01::prelude::*; +use tokio02::time::Delay; /// `Incoming` is a stream of incoming sockets /// Polling the stream may return a temporary io::Error (for instance if we can't open the connection because of "too many open files" limit) @@ -33,38 +33,99 @@ impl SuspendableStream { } } -impl Stream for SuspendableStream +// impl futures01::Stream for SuspendableStream +// where +// S: futures01::Stream, +// { +// type Item = I; +// type Error = (); + +// fn poll(&mut self) -> Result>, ()> { +// loop { +// if let Some(timeout) = self.timeout.take() { +// let deadline = timeout.deadline(); +// use futures03::FutureExt; +// let mut compat = futures03::compat::Compat::new(timeout.map(Ok::<(), ()>)); +// match futures01::Future::poll(&mut compat) { +// Ok(Async::Ready(_)) => {} +// Ok(Async::NotReady) => { +// self.timeout = Some(tokio02::time::delay_until(deadline)); +// return Ok(Async::NotReady); +// } +// Err(err) => { +// warn!("Timeout error {:?}", err); +// futures01::task::current().notify(); +// return Ok(Async::NotReady); +// } +// } +// } + +// match self.stream.poll() { +// Ok(item) => { +// if self.next_delay > self.initial_delay { +// self.next_delay = self.initial_delay; +// } +// return Ok(item); +// } +// Err(ref err) => { +// if connection_error(err) { +// warn!("Connection Error: {:?}", err); +// continue; +// } +// self.next_delay = if self.next_delay < self.max_delay { +// self.next_delay * 2 +// } else { +// self.next_delay +// }; +// debug!("Error accepting connection: {}", err); +// debug!("The server will stop accepting connections for {:?}", self.next_delay); +// self.timeout = Some(tokio02::time::delay_for(self.next_delay)); +// } +// } +// } +// } +// } + +use std::future::Future; +use std::pin::Pin; +use std::task::Poll; + +impl futures03::Stream for SuspendableStream where - S: Stream, + S: futures03::Stream> + Unpin, { type Item = I; - type Error = (); - fn poll(&mut self) -> Result>, ()> { + fn poll_next(mut self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll> { loop { - if let Some(mut timeout) = self.timeout.take() { - match timeout.poll() { - Ok(Async::Ready(_)) => {} - Ok(Async::NotReady) => { - self.timeout = Some(timeout); - return Ok(Async::NotReady); - } - Err(err) => { - warn!("Timeout error {:?}", err); - task::current().notify(); - return Ok(Async::NotReady); - } + if let Some(timeout) = self.timeout.as_mut() { + match Pin::new(timeout).poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(()) => {} } } - match self.stream.poll() { - Ok(item) => { + match Pin::new(&mut self.stream).poll_next(cx) { + Poll::Pending => { + if self.next_delay > self.initial_delay { + self.next_delay = self.initial_delay; + } + return Poll::Pending; + }, + Poll::Ready(None) => { if self.next_delay > self.initial_delay { self.next_delay = self.initial_delay; } - return Ok(item); + return Poll::Ready(None); + } + Poll::Ready(Some(Ok(item))) => { + if self.next_delay > self.initial_delay { + self.next_delay = self.initial_delay; + } + + return Poll::Ready(Some(item)); } - Err(ref err) => { + Poll::Ready(Some(Err(ref err))) => { if connection_error(err) { warn!("Connection Error: {:?}", err); continue; @@ -76,7 +137,7 @@ where }; debug!("Error accepting connection: {}", err); debug!("The server will stop accepting connections for {:?}", self.next_delay); - self.timeout = Some(Delay::new(Instant::now() + self.next_delay)); + self.timeout = Some(tokio02::time::delay_for(self.next_delay)); } } } From fb49817aea862b0e8d7a6abb0f1b6b575aa7a4bd Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Thu, 17 Dec 2020 20:28:50 +0100 Subject: [PATCH 22/73] WIP: Migrate http server to Tokio 0.2 part 2 --- http/src/handler.rs | 70 ++++++++++++++++++++++++--------------------- http/src/lib.rs | 14 ++------- 2 files changed, 41 insertions(+), 43 deletions(-) diff --git a/http/src/handler.rs b/http/src/handler.rs index 15613f9a8..d5402ed30 100644 --- a/http/src/handler.rs +++ b/http/src/handler.rs @@ -67,13 +67,14 @@ where // type ReqBody = Body; // type ResBody = Body; type Error = hyper::Error; - type Future = Handler; + type Future = futures03::compat::Compat01As03>; - fn poll_ready(&mut self, cx: &mut task::Context<'_>) -> task::Poll> { - todo!() + fn poll_ready(&mut self, _cx: &mut task::Context<'_>) -> task::Poll> { + task::Poll::Ready(Ok(())) } fn call(&mut self, request: hyper::Request) -> Self::Future { + use futures03::compat::Future01CompatExt; let is_host_allowed = utils::is_host_allowed(&request, &self.allowed_hosts); let action = self.middleware.on_request(request); @@ -90,12 +91,12 @@ where // Validate host if should_validate_hosts && !is_host_allowed { - return Handler::Err(Some(Response::host_not_allowed())); + return Handler::Err(Some(Response::host_not_allowed())).compat(); } // Replace response with the one returned by middleware. match response { - Ok(response) => Handler::Middleware(response), + Ok(response) => Handler::Middleware(response).compat(), Err(request) => { Handler::Rpc(RpcHandler { jsonrpc_handler: self.jsonrpc_handler.clone(), @@ -115,7 +116,7 @@ where max_request_body_size: self.max_request_body_size, // initial value, overwritten when reading client headers keep_alive: true, - }) + }).compat() } } } @@ -149,29 +150,30 @@ where } } -impl> std::future::Future for Handler -where - S::Future: Unpin, - S::CallFuture: Unpin, -{ - type Output = hyper::Result>; - // type Error = hyper::Error; - - fn poll(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> task::Poll { - use std::future::Future as StdFuture; - use futures03::compat::Future01CompatExt; - match *self { - Handler::Rpc(ref mut handler) => StdFuture::poll(Pin::new(&mut handler.compat()), cx), - Handler::Middleware(ref mut middleware) => StdFuture::poll(Pin::new(&mut middleware.compat()), cx), - Handler::Err(ref mut response) => std::task::Poll::Ready(Ok( - response - .take() - .expect("Response always Some initialy. Returning `Ready` so will never be polled again; qed") - .into(), - )), - } - } -} +// impl> std::future::Future for Handler +// where +// S::Future: Unpin, +// S::CallFuture: Unpin, +// { +// type Output = hyper::Result>; +// // type Error = hyper::Error; + +// fn poll(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> task::Poll { +// let lol = Pin::into_inner(self); +// use std::future::Future as StdFuture; +// use futures03::compat::Future01CompatExt; +// match *self { +// Handler::Rpc(ref mut handler) => StdFuture::poll(Pin::new(&mut handler.compat()), cx), +// Handler::Middleware(ref mut middleware) => StdFuture::poll(Pin::new(&mut middleware.compat()), cx), +// Handler::Err(ref mut response) => std::task::Poll::Ready(Ok( +// response +// .take() +// .expect("Response always Some initialy. Returning `Ready` so will never be polled again; qed") +// .into(), +// )), +// } +// } +// } enum RpcPollState { Ready(RpcHandlerState), @@ -495,13 +497,17 @@ where fn process_body( &self, - mut body: hyper::Body, + body: hyper::Body, mut request: Vec, uri: Option, metadata: M, ) -> Result, BodyError> { + use futures03::TryStreamExt; + use futures01::Stream; + let mut compat = body.compat(); + loop { - match body.poll()? { + match compat.poll()? { Async::Ready(Some(chunk)) => { if request .len() @@ -538,7 +544,7 @@ where } Async::NotReady => { return Ok(RpcPollState::NotReady(RpcHandlerState::ReadingBody { - body, + body: compat.into_inner(), request, metadata, uri, diff --git a/http/src/lib.rs b/http/src/lib.rs index 5a6f2e1dc..1d2cdc0d5 100644 --- a/http/src/lib.rs +++ b/http/src/lib.rs @@ -589,7 +589,7 @@ fn serve>( let mut http = server::conn::Http::new(); http.keep_alive(keep_alive); - let tcp_stream = SuspendableStream::new(listener.incoming()); + let tcp_stream = SuspendableStream::new(listener); use futures03::StreamExt; let server = tcp_stream @@ -638,18 +638,10 @@ fn serve>( // })) // .map_err(|_| ()) }) - .and_then(|either| { + .and_then(|either_fut| { // We drop the server first to prevent a situation where main thread terminates // before the server is properly dropped (see #504 for more details) - match either { - futures03::future::Either::Left((server_done, shutdown_future)) => { - drop(shutdown_future); - }, - futures03::future::Either::Right((shutdown_signalled, server_future)) => { - drop(server_future); - }, - } - // drop(server); + drop(either_fut); done_tx.send(()) }) .compat() From 68e1475c9e6fd127717d6bd8ce8aedf9bd864f56 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Thu, 17 Dec 2020 21:54:09 +0100 Subject: [PATCH 23/73] WIP: Migrate http server to Tokio 0.2 part 3: tests pass --- http/src/lib.rs | 107 +++++++++++++++++----------------------- server-utils/Cargo.toml | 2 +- 2 files changed, 46 insertions(+), 63 deletions(-) diff --git a/http/src/lib.rs b/http/src/lib.rs index 1d2cdc0d5..a4ead7f91 100644 --- a/http/src/lib.rs +++ b/http/src/lib.rs @@ -540,9 +540,8 @@ fn serve>( S::CallFuture: Unpin, { use futures03::compat::Future01CompatExt; - let (shutdown_signal, local_addr_tx, done_tx) = signals; - executor.spawn({ + executor.spawn(async move { let bind = move || { let listener = match addr { SocketAddr::V4(_) => net2::TcpBuilder::new_v4()?, @@ -583,68 +582,52 @@ fn serve>( } }; - bind_result - .and_then(move |(listener, local_addr)| { - let allowed_hosts = server_utils::hosts::update(allowed_hosts, &local_addr); - - let mut http = server::conn::Http::new(); - http.keep_alive(keep_alive); - let tcp_stream = SuspendableStream::new(listener); - use futures03::StreamExt; - - let server = tcp_stream - .map(move |socket| { - let service = ServerHandler::new( - jsonrpc_handler.downgrade(), - cors_domains.clone(), - cors_max_age, - allowed_headers.clone(), - allowed_hosts.clone(), - request_middleware.clone(), - rest_api, - health_api.clone(), - max_request_body_size, - keep_alive, - ); + let (listener, local_addr) = bind_result.compat().await?; - let connection = http.serve_connection(socket, service) - .map(|res| { - res.map_err(|e| error!("Error serving connection: {:?}", e)) - }); - - tokio02::spawn(connection); - }) - .for_each(|_| async { () }); - let shutdown_signal = shutdown_signal.map_err(|e| { - debug!("Shutdown signaller dropped, closing server: {:?}", e); - }); - - use futures03::FutureExt; - use std::future::Future as StdFuture; - - let server: std::pin::Pin + Send>> = Box::pin(server); - let shutdown_signal: std::pin::Pin + Send>> = Box::pin(shutdown_signal.compat().map(|_| ())); - let select = futures03::future::select(server, shutdown_signal); - - futures03::compat::Compat::new(select.map(|x| Ok(x))) - // futures03::future::select(server, shutdown_signal).compat() - // .for_each(|_| async { () }); - // .into_future() - // .map_err(|e| { - // warn!("Incoming streams error, closing sever: {:?}", e); - // }) - // .select(shutdown_signal.map_err(|e| { - // debug!("Shutdown signaller dropped, closing server: {:?}", e); - // })) - // .map_err(|_| ()) - }) - .and_then(|either_fut| { - // We drop the server first to prevent a situation where main thread terminates - // before the server is properly dropped (see #504 for more details) - drop(either_fut); - done_tx.send(()) + let allowed_hosts = server_utils::hosts::update(allowed_hosts, &local_addr); + + let mut http = server::conn::Http::new(); + http.keep_alive(keep_alive); + let tcp_stream = SuspendableStream::new(listener); + use futures03::StreamExt; + + let server = tcp_stream + .map(move |socket| { + let service = ServerHandler::new( + jsonrpc_handler.downgrade(), + cors_domains.clone(), + cors_max_age, + allowed_headers.clone(), + allowed_hosts.clone(), + request_middleware.clone(), + rest_api, + health_api.clone(), + max_request_body_size, + keep_alive, + ); + + let connection = http.serve_connection(socket, service) + .map(|res| { + res.map_err(|e| error!("Error serving connection: {:?}", e)) + }); + + tokio02::spawn(connection); }) - .compat() + .for_each(|_| async { () }); + let shutdown_signal = shutdown_signal.map_err(|e| { + debug!("Shutdown signaller dropped, closing server: {:?}", e); + }); + + use futures03::FutureExt; + + let select = futures03::future::select(Box::pin(server), shutdown_signal.compat().map(drop)); + + let res = select.await; + // We drop the server (possibly unresolved future) first to prevent a situation where + // main thread terminates before the server is properly dropped (see #504 for more details) + drop(res); + + done_tx.send(()) }); } diff --git a/server-utils/Cargo.toml b/server-utils/Cargo.toml index 5e174fc90..e3e868f1d 100644 --- a/server-utils/Cargo.toml +++ b/server-utils/Cargo.toml @@ -23,7 +23,7 @@ tokio = { version = "0.1.15", default-features = false, features = ["timer", "tc tokio-codec = { version = "0.1", optional = true } tokio-util = { version = "0.3", features = ["codec", "compat"] } tokio-compat = { version = "0.1", optional = true } -tokio02 = { package = "tokio", version = "0.2", optional = true, features = ["rt-threaded", "io-driver", "time"] } +tokio02 = { package = "tokio", version = "0.2", optional = true, features = ["rt-threaded", "io-driver", "time", "tcp"] } unicase = "2.0" From 67360db04d26f3e7ebab3546892193c0f78238c5 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Thu, 17 Dec 2020 21:55:09 +0100 Subject: [PATCH 24/73] http: Silence warnings --- http/src/handler.rs | 2 +- http/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/http/src/handler.rs b/http/src/handler.rs index d5402ed30..b032e701f 100644 --- a/http/src/handler.rs +++ b/http/src/handler.rs @@ -2,7 +2,7 @@ use crate::WeakRpc; use std::sync::Arc; use std::{fmt, mem, str}; -use std::{pin::Pin, task}; +use std::task; use hyper::header::{self, HeaderMap, HeaderValue}; use hyper::{self, service::Service, Body, Method}; diff --git a/http/src/lib.rs b/http/src/lib.rs index a4ead7f91..223477a59 100644 --- a/http/src/lib.rs +++ b/http/src/lib.rs @@ -587,7 +587,7 @@ fn serve>( let allowed_hosts = server_utils::hosts::update(allowed_hosts, &local_addr); let mut http = server::conn::Http::new(); - http.keep_alive(keep_alive); + http.http1_keep_alive(keep_alive); let tcp_stream = SuspendableStream::new(listener); use futures03::StreamExt; From 3b5d965f98d5469641ccf13e65b4572f985d650d Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Thu, 17 Dec 2020 22:30:43 +0100 Subject: [PATCH 25/73] WIP: Adapt tcp server: part 1 --- server-utils/Cargo.toml | 2 +- tcp/Cargo.toml | 2 +- tcp/src/lib.rs | 2 +- tcp/src/server.rs | 38 ++++--- tcp/src/tests.rs | 228 +++++++++++++++++++++++++++------------- 5 files changed, 182 insertions(+), 90 deletions(-) diff --git a/server-utils/Cargo.toml b/server-utils/Cargo.toml index e3e868f1d..6943fe97a 100644 --- a/server-utils/Cargo.toml +++ b/server-utils/Cargo.toml @@ -23,7 +23,7 @@ tokio = { version = "0.1.15", default-features = false, features = ["timer", "tc tokio-codec = { version = "0.1", optional = true } tokio-util = { version = "0.3", features = ["codec", "compat"] } tokio-compat = { version = "0.1", optional = true } -tokio02 = { package = "tokio", version = "0.2", optional = true, features = ["rt-threaded", "io-driver", "time", "tcp"] } +tokio02 = { package = "tokio", version = "0.2", optional = true, features = ["rt-threaded", "io-driver", "io-util", "time", "tcp"] } unicase = "2.0" diff --git a/tcp/Cargo.toml b/tcp/Cargo.toml index 66b5d280d..04da54236 100644 --- a/tcp/Cargo.toml +++ b/tcp/Cargo.toml @@ -14,7 +14,7 @@ futures01 = { version = "0.1", package = "futures" } # TODO remove when we no longer need compat (use jsonrpc-core re-export instead) futures03 = { version = "0.3", features = ["compat"], package = "futures" } jsonrpc-core = { version = "16.0", path = "../core" } -jsonrpc-server-utils = { version = "16.0", path = "../server-utils", features = ["tokio", "tokio-codec"] } +jsonrpc-server-utils = { version = "16.0", path = "../server-utils", default-features = false, features = ["tokio02"] } log = "0.4" parking_lot = "0.11.0" tower-service = "0.3" diff --git a/tcp/src/lib.rs b/tcp/src/lib.rs index 5eda8e7e1..c90af9469 100644 --- a/tcp/src/lib.rs +++ b/tcp/src/lib.rs @@ -42,7 +42,7 @@ mod tests; use jsonrpc_core as jsonrpc; -pub use self::server_utils::{codecs::Separator, tokio}; +pub use self::server_utils::{codecs::Separator, tokio02}; pub use crate::dispatch::{Dispatcher, PushMessageError}; pub use crate::meta::{MetaExtractor, RequestContext}; pub use crate::server::{Server, ServerBuilder}; diff --git a/tcp/src/server.rs b/tcp/src/server.rs index 157c12c7c..4a3033121 100644 --- a/tcp/src/server.rs +++ b/tcp/src/server.rs @@ -8,7 +8,7 @@ use futures01::sync::oneshot; use futures01::{future, Future, Sink, Stream}; use crate::jsonrpc::{middleware, MetaIoHandler, Metadata, Middleware}; -use crate::server_utils::{codecs, reactor, tokio, tokio_compat, tokio_codec::Framed, SuspendableStream}; +use crate::server_utils::{codecs, reactor, tokio02, tokio_util::codec::Framed, SuspendableStream}; use crate::dispatch::{Dispatcher, PeerMessageQueue, SenderChannels}; use crate::meta::{MetaExtractor, NoopExtractor, RequestContext}; @@ -59,11 +59,11 @@ where } } - /// Utilize existing event loop executor. - pub fn event_loop_executor(mut self, handle: tokio_compat::runtime::TaskExecutor) -> Self { - self.executor = reactor::UninitializedExecutor::Shared(handle); - self - } + // /// Utilize existing event loop executor. + // pub fn event_loop_executor(mut self, handle: tokio_compat::runtime::TaskExecutor) -> Self { + // self.executor = reactor::UninitializedExecutor::Shared(handle); + // self + // } /// Sets session meta extractor pub fn session_meta_extractor + 'static>(mut self, meta_extractor: T) -> Self { @@ -91,12 +91,15 @@ where let executor = self.executor.initialize()?; - executor.executor().spawn(future::lazy(move || { - let start = move || { - let listener = tokio::net::TcpListener::bind(&address)?; - let connections = SuspendableStream::new(listener.incoming()); + use futures03::StreamExt; + executor.executor().spawn(async move { + let start = async move { + let listener = tokio02::net::TcpListener::bind(&address).await?; + let connections = SuspendableStream::new(listener); + let connections = connections.map(Ok); - let server = connections.map(move |socket| { + use futures03::TryStreamExt; + let server = connections.compat().map(move |socket| { let peer_addr = match socket.peer_addr() { Ok(addr) => addr, Err(e) => { @@ -120,6 +123,10 @@ where ) .split(); + use futures03::TryStreamExt; + use futures03::SinkExt; + let reader = reader.compat(); + let writer = writer.compat(); use futures03::TryFutureExt; let responses = reader.and_then(move |req| { service.call(req).compat().then(|response| match response { @@ -156,11 +163,14 @@ where future::Either::B(writer) }); + use futures03::compat::Stream01CompatExt; Ok(server) }; + use futures03::compat::Future01CompatExt; + let stop = stop_rx.map_err(|_| ()); - match start() { + match start.await { Ok(server) => { tx.send(Ok(())).expect("Rx is blocking parent thread."); future::Either::A( @@ -181,7 +191,9 @@ where })) } } - })); + .compat() + .await + }); let res = rx.recv().expect("Response is always sent before tx is dropped."); diff --git a/tcp/src/tests.rs b/tcp/src/tests.rs index f06293847..7067becd8 100644 --- a/tcp/src/tests.rs +++ b/tcp/src/tests.rs @@ -6,7 +6,9 @@ use std::time::{Duration, Instant}; use futures01::{future, Future}; use jsonrpc_core::{MetaIoHandler, Metadata, Value}; -use crate::server_utils::tokio::{self, io, net::TcpStream, timer::Delay}; +use crate::server_utils::tokio02::{self as tokio, io, net::TcpStream, time::Delay}; + use tokio::io::AsyncWriteExt; + use tokio::io::AsyncReadExt; use parking_lot::Mutex; @@ -20,6 +22,11 @@ fn casual_server() -> ServerBuilder { ServerBuilder::new(io) } +fn run_future(fut: impl std::future::Future + Send) -> O { + let mut rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(fut) +} + #[test] fn doc_test() { crate::logger::init_log(); @@ -41,11 +48,14 @@ fn doc_test_connect() { let server = casual_server(); let _server = server.start(&addr).expect("Server must run with no issues"); - let stream = TcpStream::connect(&addr) - .and_then(move |_stream| Ok(())) - .map_err(|err| panic!("Server connection error: {:?}", err)); + run_future(async move { + TcpStream::connect(&addr).await + }).unwrap(); + // let stream = TcpStream::connect(&addr) + // .and_then(move |_stream| Ok(())) + // .map_err(|err| panic!("Server connection error: {:?}", err)); - tokio::run(stream); + // tokio::run(stream); } #[test] @@ -56,14 +66,19 @@ fn disconnect() { let dispatcher = server.dispatcher(); let _server = server.start(&addr).expect("Server must run with no issues"); - let stream = TcpStream::connect(&addr) - .and_then(move |stream| { - assert_eq!(stream.peer_addr().unwrap(), addr); - stream.shutdown(::std::net::Shutdown::Both) - }) - .map_err(|err| panic!("Error disconnecting: {:?}", err)); + run_future(async move { + let stream = TcpStream::connect(&addr).await.unwrap(); + assert_eq!(stream.peer_addr().unwrap(), addr); + stream.shutdown(::std::net::Shutdown::Both).unwrap(); + }); + // let stream = TcpStream::connect(&addr) + // .and_then(move |stream| { + // assert_eq!(stream.peer_addr().unwrap(), addr); + // stream.shutdown(::std::net::Shutdown::Both) + // }) + // .map_err(|err| panic!("Error disconnecting: {:?}", err)); - tokio::run(stream); + // tokio::run(stream); ::std::thread::sleep(::std::time::Duration::from_millis(50)); @@ -73,17 +88,34 @@ fn disconnect() { fn dummy_request(addr: &SocketAddr, data: Vec) -> Vec { let (ret_tx, ret_rx) = futures01::sync::oneshot::channel(); - let stream = TcpStream::connect(addr) - .and_then(move |stream| io::write_all(stream, data)) - .and_then(|(stream, _data)| { - stream.shutdown(Shutdown::Write).unwrap(); - io::read_to_end(stream, vec![]) - }) - .and_then(move |(_stream, read_buf)| ret_tx.send(read_buf).map_err(|err| panic!("Unable to send {:?}", err))) - .map_err(|err| panic!("Error connecting or closing connection: {:?}", err)); + use futures03::TryFutureExt; + + let stream = async move { + let mut stream = TcpStream::connect(addr).await?; + stream.write_all(&data).await?; + stream.shutdown(Shutdown::Write)?; + let mut read_buf = vec![]; + let _ = stream.read_to_end(&mut read_buf).await; - tokio::run(stream); + let _ = ret_tx.send(read_buf).map_err(|err| panic!("Unable to send {:?}", err)); + + Ok::<(), Box>(()) + }; + + run_future(stream).unwrap(); ret_rx.wait().expect("Unable to receive result") + + // let stream = TcpStream::connect(addr) + // .and_then(move |stream| io::write_all(stream, data)) + // .and_then(|(stream, _data)| { + // stream.shutdown(Shutdown::Write).unwrap(); + // io::read_to_end(stream, vec![]) + // }) + // .and_then(move |(_stream, read_buf)| ret_tx.send(read_buf).map_err(|err| panic!("Unable to send {:?}", err))) + // .map_err(|err| panic!("Error connecting or closing connection: {:?}", err)); + + // tokio::run(stream); + // ret_rx.wait().expect("Unable to receive result") } fn dummy_request_str(addr: &SocketAddr, data: Vec) -> String { @@ -232,7 +264,8 @@ fn message() { let _server = server.start(&addr).expect("Server must run with no issues"); - let delay = Delay::new(Instant::now() + Duration::from_millis(500)).map_err(|err| panic!("{:?}", err)); + // let delay = futures03::future::lazy(|_| tokio::time::delay_for(Duration::from_millis(500))); + // let delay = Delay::new(Instant::now() + Duration::from_millis(500)).map_err(|err| panic!("{:?}", err)); let message = "ping"; let executed_dispatch = Arc::new(Mutex::new(false)); @@ -241,58 +274,105 @@ fn message() { let executed_request_move = executed_request.clone(); // CLIENT RUN - let stream = TcpStream::connect(&addr) - .and_then(|stream| future::ok(stream).join(delay)) - .and_then(move |stream| { - let peer_addr = peer_list.lock()[0].clone(); - dispatcher - .push_message(&peer_addr, message.to_owned()) - .expect("Should be sent with no errors"); - trace!(target: "tcp", "Dispatched message for {}", peer_addr); - future::ok(stream) - }) - .and_then(move |(stream, _)| { - // Read message plus newline appended by codec. - io::read_exact(stream, vec![0u8; message.len() + 1]) - }) - .and_then(move |(stream, read_buf)| { - trace!(target: "tcp", "Read ping message"); - let ping_signal = read_buf[..].to_vec(); - - assert_eq!( - format!("{}\n", message), - String::from_utf8(ping_signal).expect("String should be utf-8"), - "Sent request does not match received by the peer", - ); - // ensure that the above assert was actually triggered - *executed_dispatch_move.lock() = true; - - future::ok(stream) - }) - .and_then(|stream| { - // make request AFTER message dispatches - let data = b"{\"jsonrpc\": \"2.0\", \"method\": \"say_hello\", \"params\": [42, 23], \"id\": 1}\n"; - io::write_all(stream, &data[..]) - }) - .and_then(|(stream, _)| { - stream.shutdown(Shutdown::Write).unwrap(); - io::read_to_end(stream, Vec::new()) - }) - .and_then(move |(_, read_buf)| { - trace!(target: "tcp", "Read response message"); - let response_signal = read_buf[..].to_vec(); - assert_eq!( - "{\"jsonrpc\":\"2.0\",\"result\":\"hello\",\"id\":1}\n", - String::from_utf8(response_signal).expect("String should be utf-8"), - "Response does not match the expected handling", - ); - *executed_request_move.lock() = true; - - future::ok(()) - }) - .map_err(|err| panic!("Dispach message error: {:?}", err)); - - tokio::run(stream); + let stream = async move { + let mut stream = TcpStream::connect(&addr).await?; + tokio::time::delay_for(Duration::from_millis(500)).await; + // TODO: delay + + let peer_addr = peer_list.lock()[0].clone(); + dispatcher + .push_message(&peer_addr, message.to_owned()) + .expect("Should be sent with no errors"); + trace!(target: "tcp", "Dispatched message for {}", peer_addr); + + // Read message plus newline appended by codec. + let mut read_buf = vec![0u8; message.len() + 1]; + let _ = stream.read_exact(&mut read_buf).await?; + + trace!(target: "tcp", "Read ping message"); + let ping_signal = read_buf[..].to_vec(); + + assert_eq!( + format!("{}\n", message), + String::from_utf8(ping_signal).expect("String should be utf-8"), + "Sent request does not match received by the peer", + ); + // ensure that the above assert was actually triggered + *executed_dispatch_move.lock() = true; + + // make request AFTER message dispatches + let data = b"{\"jsonrpc\": \"2.0\", \"method\": \"say_hello\", \"params\": [42, 23], \"id\": 1}\n"; + stream.write_all(&data[..]).await?; + + stream.shutdown(Shutdown::Write).unwrap(); + let _ = stream.read_to_end(&mut Vec::new()).await?; + + trace!(target: "tcp", "Read response message"); + let response_signal = read_buf[..].to_vec(); + assert_eq!( + "{\"jsonrpc\":\"2.0\",\"result\":\"hello\",\"id\":1}\n", + String::from_utf8(response_signal).expect("String should be utf-8"), + "Response does not match the expected handling", + ); + *executed_request_move.lock() = true; + + // delay + Ok::<(), Box>(()) + }; + + run_future(stream).unwrap(); + // let stream = TcpStream::connect(&addr) + // .and_then(|stream| future::ok(stream).join(delay)) + // .and_then(move |stream| { + // let peer_addr = peer_list.lock()[0].clone(); + // dispatcher + // .push_message(&peer_addr, message.to_owned()) + // .expect("Should be sent with no errors"); + // trace!(target: "tcp", "Dispatched message for {}", peer_addr); + // future::ok(stream) + // }) + // .and_then(move |(stream, _)| { + // // Read message plus newline appended by codec. + // io::read_exact(stream, vec![0u8; message.len() + 1]) + // }) + // .and_then(move |(stream, read_buf)| { + // trace!(target: "tcp", "Read ping message"); + // let ping_signal = read_buf[..].to_vec(); + + // assert_eq!( + // format!("{}\n", message), + // String::from_utf8(ping_signal).expect("String should be utf-8"), + // "Sent request does not match received by the peer", + // ); + // // ensure that the above assert was actually triggered + // *executed_dispatch_move.lock() = true; + + // future::ok(stream) + // }) + // .and_then(|stream| { + // // make request AFTER message dispatches + // let data = b"{\"jsonrpc\": \"2.0\", \"method\": \"say_hello\", \"params\": [42, 23], \"id\": 1}\n"; + // io::write_all(stream, &data[..]) + // }) + // .and_then(|(stream, _)| { + // stream.shutdown(Shutdown::Write).unwrap(); + // io::read_to_end(stream, Vec::new()) + // }) + // .and_then(move |(_, read_buf)| { + // trace!(target: "tcp", "Read response message"); + // let response_signal = read_buf[..].to_vec(); + // assert_eq!( + // "{\"jsonrpc\":\"2.0\",\"result\":\"hello\",\"id\":1}\n", + // String::from_utf8(response_signal).expect("String should be utf-8"), + // "Response does not match the expected handling", + // ); + // *executed_request_move.lock() = true; + + // future::ok(()) + // }) + // .map_err(|err| panic!("Dispach message error: {:?}", err)); + + // tokio::run(stream); assert!(*executed_dispatch.lock()); assert!(*executed_request.lock()); } From 0be5b868c9b67056623e235d933dc3d297e53248 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Thu, 17 Dec 2020 23:13:20 +0100 Subject: [PATCH 26/73] Argh yet another attempt at one type is more general --- tcp/src/server.rs | 52 ++++++++++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/tcp/src/server.rs b/tcp/src/server.rs index 4a3033121..4a514c828 100644 --- a/tcp/src/server.rs +++ b/tcp/src/server.rs @@ -96,15 +96,15 @@ where let start = async move { let listener = tokio02::net::TcpListener::bind(&address).await?; let connections = SuspendableStream::new(listener); - let connections = connections.map(Ok); + // let connections = connections.map(Ok); use futures03::TryStreamExt; - let server = connections.compat().map(move |socket| { + let server = connections.map(move |socket| { let peer_addr = match socket.peer_addr() { Ok(addr) => addr, Err(e) => { warn!(target: "tcp", "Unable to determine socket peer address, ignoring connection {}", e); - return future::Either::A(future::ok(())); + return future::Either::A(future::ok::<(), ()>(())).compat(); } }; trace!(target: "tcp", "Accepted incoming connection from {}", &peer_addr); @@ -157,10 +157,10 @@ where trace!(target: "tcp", "Peer {}: service finished", peer_addr); let mut channels = shared_channels.lock(); channels.remove(&peer_addr); - Ok(()) + Ok::<(), ()>(()) }); - future::Either::B(writer) + future::Either::B(writer).compat() }); use futures03::compat::Stream01CompatExt; @@ -169,30 +169,40 @@ where use futures03::compat::Future01CompatExt; - let stop = stop_rx.map_err(|_| ()); + let stop = stop_rx.map(drop).map_err(drop).compat(); match start.await { Ok(server) => { tx.send(Ok(())).expect("Rx is blocking parent thread."); - future::Either::A( - server - .buffer_unordered(1024) - .for_each(|_| Ok(())) - .select(stop) - .map(|_| ()) - .map_err(|(e, _)| { - error!("Error while executing the server: {:?}", e); - }), - ) + let server = server + .buffer_unordered(1024) + .for_each(|_| async { () }) + ; + // let stop = stop; + + let select = futures03::future::select(Box::pin(server), stop); + select.await; + + // future::Either::A( + // server + // .buffer_unordered(1024) + // // .for_each(|_| async { Ok(()) }) + // .select(stop) + // .map(|_| ()) + // .map_err(|(e, _)| { + // error!("Error while executing the server: {:?}", e); + // }), + // ) } Err(e) => { tx.send(Err(e)).expect("Rx is blocking parent thread."); - future::Either::B(stop.map_err(|e| { - error!("Error while executing the server: {:?}", e); - })) + stop.await; + // future::Either::B(stop.map_err(|e| { + // error!("Error while executing the server: {:?}", e); + // })) } } - .compat() - .await + // .compat() + // .await }); let res = rx.recv().expect("Response is always sent before tx is dropped."); From 91d48ae83783a49c7982d9e7b9a18ac96ac36ab2 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Thu, 17 Dec 2020 23:43:50 +0100 Subject: [PATCH 27/73] Prune futures01 from tcp server --- tcp/src/dispatch.rs | 71 ++++++++++++++++++++++++++++++--------------- tcp/src/server.rs | 50 ++++++++++++++++--------------- 2 files changed, 74 insertions(+), 47 deletions(-) diff --git a/tcp/src/dispatch.rs b/tcp/src/dispatch.rs index f12121a51..ad2bfef07 100644 --- a/tcp/src/dispatch.rs +++ b/tcp/src/dispatch.rs @@ -4,24 +4,31 @@ use std::net::SocketAddr; use std::sync::Arc; use crate::jsonrpc::futures::{self as futures03, channel::mpsc, StreamExt}; -use futures01::{Async, Poll, Stream}; +// use futures01::{Async, + // Poll, + // Stream +// }; +use futures03::Stream; +use std::task::Poll; use parking_lot::Mutex; pub type SenderChannels = Mutex>>; -pub struct PeerMessageQueue { +pub struct PeerMessageQueue { up: S, - receiver: Option + Send>>, + receiver: Option>, + // receiver: Option + Send>>, _addr: SocketAddr, } -impl PeerMessageQueue { +impl PeerMessageQueue { pub fn new(response_stream: S, receiver: mpsc::UnboundedReceiver, addr: SocketAddr) -> Self { - let receiver = futures03::compat::Compat::new(receiver.map(|v| Ok(v))); + // let receiver = futures03::compat::Compat::new(receiver.map(|v| Ok(v))); PeerMessageQueue { up: response_stream, - receiver: Some(Box::new(receiver)), + receiver: Some(receiver), + // receiver: Some(Box::new(receiver)), _addr: addr, } } @@ -78,9 +85,11 @@ impl Dispatcher { } } -impl> Stream for PeerMessageQueue { - type Item = String; - type Error = std::io::Error; +use std::pin::Pin; + +impl>> Stream for PeerMessageQueue { + type Item = std::io::Result; + // type Error = std::io::Error; // The receiver will never return `Ok(Async::Ready(None))` // Because the sender is kept in `SenderChannels` and it will never be dropped until `the stream` is resolved. @@ -90,32 +99,46 @@ impl> Stream for PeerMessageQue // However, it is possible to have a race between `poll` and `push_work` if the connection is dropped. // Therefore, the receiver is then dropped when the connection is dropped and an error is propagated when // a `send` attempt is made on that channel. - fn poll(&mut self) -> Poll, std::io::Error> { + fn poll_next(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll> { // check if we have response pending + let this = Pin::into_inner(self); - let up_closed = match self.up.poll() { - Ok(Async::Ready(Some(item))) => return Ok(Async::Ready(Some(item))), - Ok(Async::Ready(None)) => true, - Ok(Async::NotReady) => false, - err => return err, + let up_closed = match Pin::new(&mut this.up).poll_next(cx) { + Poll::Pending => false, + Poll::Ready(None) => true, + Poll::Ready(Some(Ok(item))) => return Poll::Ready(Some(Ok(item))), + Poll::Ready(Some(Err(err))) => return Poll::Ready(Some(Err(err))), }; + // Ok(Async::Ready(Some(item))) => return Ok(Async::Ready(Some(item))), + // Ok(Async::Ready(None)) => true, + // Ok(Async::NotReady) => false, + // err => return err, + // }; - let rx = match &mut self.receiver { + let rx = match &mut this.receiver { None => { debug_assert!(up_closed); - return Ok(Async::Ready(None)); + return Poll::Ready(None); } Some(rx) => rx, }; - match rx.poll() { - Ok(Async::Ready(Some(item))) => Ok(Async::Ready(Some(item))), - Ok(Async::Ready(None)) | Ok(Async::NotReady) if up_closed => { - self.receiver = None; - Ok(Async::Ready(None)) + match Pin::new(&mut rx).poll_next(cx) { + Poll::Ready(Some(item)) => Poll::Ready(Some(Ok(item))), + Poll::Pending | Poll::Ready(None) if up_closed => { + this.receiver = None; + Poll::Ready(None) } - Ok(Async::Ready(None)) | Ok(Async::NotReady) => Ok(Async::NotReady), - Err(_) => Err(std::io::Error::new(std::io::ErrorKind::Other, "MPSC error")), + Poll::Pending | Poll::Ready(None) => Poll::Pending, } + // match rx.poll() { + // Ok(Async::Ready(Some(item))) => Ok(Async::Ready(Some(item))), + // Ok(Async::Ready(None)) | Ok(Async::NotReady) if up_closed => { + // self.receiver = None; + // Ok(Async::Ready(None)) + // } + // Ok(Async::Ready(None)) | Ok(Async::NotReady) => Ok(Async::NotReady), + // Err(_) => Err(std::io::Error::new(std::io::ErrorKind::Other, "MPSC error")), + // } } } diff --git a/tcp/src/server.rs b/tcp/src/server.rs index 4a514c828..c2d1fa94f 100644 --- a/tcp/src/server.rs +++ b/tcp/src/server.rs @@ -4,8 +4,12 @@ use std::sync::Arc; use tower_service::Service as _; -use futures01::sync::oneshot; -use futures01::{future, Future, Sink, Stream}; +use futures03::channel::oneshot; +// use futures01::sync::oneshot; +use futures01::{ + // future, + // Future, + Sink, Stream}; use crate::jsonrpc::{middleware, MetaIoHandler, Metadata, Middleware}; use crate::server_utils::{codecs, reactor, tokio02, tokio_util::codec::Framed, SuspendableStream}; @@ -87,7 +91,7 @@ where let outgoing_separator = self.outgoing_separator; let address = addr.to_owned(); let (tx, rx) = std::sync::mpsc::channel(); - let (stop_tx, stop_rx) = oneshot::channel(); + let (stop_tx, stop_rx) = futures03::channel::oneshot::channel(); let executor = self.executor.initialize()?; @@ -104,7 +108,10 @@ where Ok(addr) => addr, Err(e) => { warn!(target: "tcp", "Unable to determine socket peer address, ignoring connection {}", e); - return future::Either::A(future::ok::<(), ()>(())).compat(); + let fut = Box::pin(async move { }); + // return future::Either::A(future::ok::<(), ()>(())).compat(); + return futures03::future::Either::Left(fut); + // return future::Either::A(future::ok::<(), ()>(())).compat(); } }; trace!(target: "tcp", "Accepted incoming connection from {}", &peer_addr); @@ -125,22 +132,21 @@ where use futures03::TryStreamExt; use futures03::SinkExt; - let reader = reader.compat(); - let writer = writer.compat(); use futures03::TryFutureExt; - let responses = reader.and_then(move |req| { - service.call(req).compat().then(|response| match response { + use futures03::FutureExt; + let responses = TryStreamExt::and_then(reader, move |req| { + service.call(req).then(|response| match response { Err(e) => { warn!(target: "tcp", "Error while processing request: {:?}", e); - future::ok(String::new()) + futures03::future::ok(String::new()) } Ok(None) => { trace!(target: "tcp", "JSON RPC request produced no response"); - future::ok(String::new()) + futures03::future::ok(String::new()) } Ok(Some(response_data)) => { trace!(target: "tcp", "Sent response: {}", &response_data); - future::ok(response_data) + futures03::future::ok(response_data) } }) }); @@ -153,23 +159,24 @@ where }; let shared_channels = channels.clone(); - let writer = writer.send_all(peer_message_queue).then(move |_| { + let writer = Box::pin(async move { + let _ = writer.send_all(&mut peer_message_queue).await; + // writer.send_all(peer_message_queue).then(move |_| { trace!(target: "tcp", "Peer {}: service finished", peer_addr); let mut channels = shared_channels.lock(); channels.remove(&peer_addr); - Ok::<(), ()>(()) - }); + // Ok::<(), ()>(()) + }); + // }); - future::Either::B(writer).compat() + futures03::future::Either::Right(writer) }); - use futures03::compat::Stream01CompatExt; Ok(server) }; - use futures03::compat::Future01CompatExt; + use futures03::TryFutureExt; - let stop = stop_rx.map(drop).map_err(drop).compat(); match start.await { Ok(server) => { tx.send(Ok(())).expect("Rx is blocking parent thread."); @@ -177,9 +184,8 @@ where .buffer_unordered(1024) .for_each(|_| async { () }) ; - // let stop = stop; - let select = futures03::future::select(Box::pin(server), stop); + let select = futures03::future::select(Box::pin(server), stop_rx); select.await; // future::Either::A( @@ -195,14 +201,12 @@ where } Err(e) => { tx.send(Err(e)).expect("Rx is blocking parent thread."); - stop.await; + stop_rx.await; // future::Either::B(stop.map_err(|e| { // error!("Error while executing the server: {:?}", e); // })) } } - // .compat() - // .await }); let res = rx.recv().expect("Response is always sent before tx is dropped."); From 029b2d0a7b7d3d61037407beeb9f10ef9df55275 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Fri, 18 Dec 2020 11:39:13 +0100 Subject: [PATCH 28/73] Fix stuff --- tcp/src/dispatch.rs | 2 +- tcp/src/server.rs | 9 +++++---- tcp/src/tests.rs | 3 ++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tcp/src/dispatch.rs b/tcp/src/dispatch.rs index ad2bfef07..a9a35c274 100644 --- a/tcp/src/dispatch.rs +++ b/tcp/src/dispatch.rs @@ -115,7 +115,7 @@ impl>> Stream for PeerMessageQu // err => return err, // }; - let rx = match &mut this.receiver { + let mut rx = match &mut this.receiver { None => { debug_assert!(up_closed); return Poll::Ready(None); diff --git a/tcp/src/server.rs b/tcp/src/server.rs index c2d1fa94f..a101e383b 100644 --- a/tcp/src/server.rs +++ b/tcp/src/server.rs @@ -124,7 +124,7 @@ where let meta = meta_extractor.extract(&context); let mut service = Service::new(peer_addr, rpc_handler.clone(), meta); - let (writer, reader) = Framed::new( + let (mut writer, reader) = Framed::new( socket, codecs::StreamCodec::new(incoming_separator.clone(), outgoing_separator.clone()), ) @@ -134,7 +134,8 @@ where use futures03::SinkExt; use futures03::TryFutureExt; use futures03::FutureExt; - let responses = TryStreamExt::and_then(reader, move |req| { + // Work around https://github.com/rust-lang/rust/issues/64552 by boxing the stream type + let responses: std::pin::Pin> + Send>> = Box::pin(TryStreamExt::and_then(reader, move |req| { service.call(req).then(|response| match response { Err(e) => { warn!(target: "tcp", "Error while processing request: {:?}", e); @@ -149,9 +150,9 @@ where futures03::future::ok(response_data) } }) - }); + })); - let peer_message_queue = { + let mut peer_message_queue = { let mut channels = channels.lock(); channels.insert(peer_addr, sender.clone()); diff --git a/tcp/src/tests.rs b/tcp/src/tests.rs index 7067becd8..f8e9eec7f 100644 --- a/tcp/src/tests.rs +++ b/tcp/src/tests.rs @@ -305,7 +305,8 @@ fn message() { stream.write_all(&data[..]).await?; stream.shutdown(Shutdown::Write).unwrap(); - let _ = stream.read_to_end(&mut Vec::new()).await?; + let mut read_buf = vec![]; + let _ = stream.read_to_end(&mut read_buf).await?; trace!(target: "tcp", "Read response message"); let response_signal = read_buf[..].to_vec(); From 094222edf6950ac78750f954c22e3a24e11722d4 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Fri, 18 Dec 2020 12:51:26 +0100 Subject: [PATCH 29/73] Clean up a bit TCP server impl --- tcp/Cargo.toml | 3 -- tcp/src/dispatch.rs | 29 ++---------- tcp/src/lib.rs | 2 + tcp/src/server.rs | 31 ++++--------- tcp/src/service.rs | 2 + tcp/src/tests.rs | 111 ++++++-------------------------------------- 6 files changed, 31 insertions(+), 147 deletions(-) diff --git a/tcp/Cargo.toml b/tcp/Cargo.toml index 04da54236..7c58991f1 100644 --- a/tcp/Cargo.toml +++ b/tcp/Cargo.toml @@ -10,9 +10,6 @@ repository = "https://github.com/paritytech/jsonrpc" version = "16.0.0" [dependencies] -futures01 = { version = "0.1", package = "futures" } -# TODO remove when we no longer need compat (use jsonrpc-core re-export instead) -futures03 = { version = "0.3", features = ["compat"], package = "futures" } jsonrpc-core = { version = "16.0", path = "../core" } jsonrpc-server-utils = { version = "16.0", path = "../server-utils", default-features = false, features = ["tokio02"] } log = "0.4" diff --git a/tcp/src/dispatch.rs b/tcp/src/dispatch.rs index a9a35c274..ae3a2c5c5 100644 --- a/tcp/src/dispatch.rs +++ b/tcp/src/dispatch.rs @@ -3,11 +3,8 @@ use std::collections::HashMap; use std::net::SocketAddr; use std::sync::Arc; -use crate::jsonrpc::futures::{self as futures03, channel::mpsc, StreamExt}; -// use futures01::{Async, - // Poll, - // Stream -// }; +use crate::jsonrpc::futures::{self as futures03, channel::mpsc}; + use futures03::Stream; use std::task::Poll; @@ -18,17 +15,14 @@ pub type SenderChannels = Mutex { up: S, receiver: Option>, - // receiver: Option + Send>>, _addr: SocketAddr, } impl PeerMessageQueue { pub fn new(response_stream: S, receiver: mpsc::UnboundedReceiver, addr: SocketAddr) -> Self { - // let receiver = futures03::compat::Compat::new(receiver.map(|v| Ok(v))); PeerMessageQueue { up: response_stream, receiver: Some(receiver), - // receiver: Some(Box::new(receiver)), _addr: addr, } } @@ -89,7 +83,6 @@ use std::pin::Pin; impl>> Stream for PeerMessageQueue { type Item = std::io::Result; - // type Error = std::io::Error; // The receiver will never return `Ok(Async::Ready(None))` // Because the sender is kept in `SenderChannels` and it will never be dropped until `the stream` is resolved. @@ -109,11 +102,6 @@ impl>> Stream for PeerMessageQu Poll::Ready(Some(Ok(item))) => return Poll::Ready(Some(Ok(item))), Poll::Ready(Some(Err(err))) => return Poll::Ready(Some(Err(err))), }; - // Ok(Async::Ready(Some(item))) => return Ok(Async::Ready(Some(item))), - // Ok(Async::Ready(None)) => true, - // Ok(Async::NotReady) => false, - // err => return err, - // }; let mut rx = match &mut this.receiver { None => { @@ -125,20 +113,11 @@ impl>> Stream for PeerMessageQu match Pin::new(&mut rx).poll_next(cx) { Poll::Ready(Some(item)) => Poll::Ready(Some(Ok(item))), - Poll::Pending | Poll::Ready(None) if up_closed => { + Poll::Ready(None) | Poll::Pending if up_closed => { this.receiver = None; Poll::Ready(None) } - Poll::Pending | Poll::Ready(None) => Poll::Pending, + Poll::Ready(None) | Poll::Pending => Poll::Pending, } - // match rx.poll() { - // Ok(Async::Ready(Some(item))) => Ok(Async::Ready(Some(item))), - // Ok(Async::Ready(None)) | Ok(Async::NotReady) if up_closed => { - // self.receiver = None; - // Ok(Async::Ready(None)) - // } - // Ok(Async::Ready(None)) | Ok(Async::NotReady) => Ok(Async::NotReady), - // Err(_) => Err(std::io::Error::new(std::io::ErrorKind::Other, "MPSC error")), - // } } } diff --git a/tcp/src/lib.rs b/tcp/src/lib.rs index c90af9469..8537619aa 100644 --- a/tcp/src/lib.rs +++ b/tcp/src/lib.rs @@ -42,6 +42,8 @@ mod tests; use jsonrpc_core as jsonrpc; +pub(crate) use crate::jsonrpc::futures as futures03; + pub use self::server_utils::{codecs::Separator, tokio02}; pub use crate::dispatch::{Dispatcher, PushMessageError}; pub use crate::meta::{MetaExtractor, RequestContext}; diff --git a/tcp/src/server.rs b/tcp/src/server.rs index a101e383b..131b27106 100644 --- a/tcp/src/server.rs +++ b/tcp/src/server.rs @@ -4,15 +4,9 @@ use std::sync::Arc; use tower_service::Service as _; -use futures03::channel::oneshot; -// use futures01::sync::oneshot; -use futures01::{ - // future, - // Future, - Sink, Stream}; - use crate::jsonrpc::{middleware, MetaIoHandler, Metadata, Middleware}; use crate::server_utils::{codecs, reactor, tokio02, tokio_util::codec::Framed, SuspendableStream}; +use crate::futures03; use crate::dispatch::{Dispatcher, PeerMessageQueue, SenderChannels}; use crate::meta::{MetaExtractor, NoopExtractor, RequestContext}; @@ -100,18 +94,14 @@ where let start = async move { let listener = tokio02::net::TcpListener::bind(&address).await?; let connections = SuspendableStream::new(listener); - // let connections = connections.map(Ok); - use futures03::TryStreamExt; let server = connections.map(move |socket| { let peer_addr = match socket.peer_addr() { Ok(addr) => addr, Err(e) => { warn!(target: "tcp", "Unable to determine socket peer address, ignoring connection {}", e); let fut = Box::pin(async move { }); - // return future::Either::A(future::ok::<(), ()>(())).compat(); return futures03::future::Either::Left(fut); - // return future::Either::A(future::ok::<(), ()>(())).compat(); } }; trace!(target: "tcp", "Accepted incoming connection from {}", &peer_addr); @@ -132,10 +122,10 @@ where use futures03::TryStreamExt; use futures03::SinkExt; - use futures03::TryFutureExt; use futures03::FutureExt; // Work around https://github.com/rust-lang/rust/issues/64552 by boxing the stream type - let responses: std::pin::Pin> + Send>> = Box::pin(TryStreamExt::and_then(reader, move |req| { + let responses: std::pin::Pin> + Send>> = + Box::pin(reader.and_then(move |req| { service.call(req).then(|response| match response { Err(e) => { warn!(target: "tcp", "Error while processing request: {:?}", e); @@ -162,13 +152,10 @@ where let shared_channels = channels.clone(); let writer = Box::pin(async move { let _ = writer.send_all(&mut peer_message_queue).await; - // writer.send_all(peer_message_queue).then(move |_| { - trace!(target: "tcp", "Peer {}: service finished", peer_addr); - let mut channels = shared_channels.lock(); - channels.remove(&peer_addr); - // Ok::<(), ()>(()) + trace!(target: "tcp", "Peer {}: service finished", peer_addr); + let mut channels = shared_channels.lock(); + channels.remove(&peer_addr); }); - // }); futures03::future::Either::Right(writer) }); @@ -176,8 +163,6 @@ where Ok(server) }; - use futures03::TryFutureExt; - match start.await { Ok(server) => { tx.send(Ok(())).expect("Rx is blocking parent thread."); @@ -202,7 +187,7 @@ where } Err(e) => { tx.send(Err(e)).expect("Rx is blocking parent thread."); - stop_rx.await; + let _ = stop_rx.await; // future::Either::B(stop.map_err(|e| { // error!("Error while executing the server: {:?}", e); // })) @@ -227,7 +212,7 @@ where /// TCP Server handle pub struct Server { executor: Option, - stop: Option>, + stop: Option>, } impl Server { diff --git a/tcp/src/service.rs b/tcp/src/service.rs index fc07403fc..472839903 100644 --- a/tcp/src/service.rs +++ b/tcp/src/service.rs @@ -5,6 +5,8 @@ use std::sync::Arc; use std::task::{Context, Poll}; use crate::jsonrpc::{middleware, MetaIoHandler, Metadata, Middleware}; +use crate::futures03; + pub struct Service = middleware::Noop> { handler: Arc>, diff --git a/tcp/src/tests.rs b/tcp/src/tests.rs index f8e9eec7f..93a19484f 100644 --- a/tcp/src/tests.rs +++ b/tcp/src/tests.rs @@ -1,14 +1,14 @@ use std::net::{Shutdown, SocketAddr}; use std::str::FromStr; use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::time::Duration; -use futures01::{future, Future}; use jsonrpc_core::{MetaIoHandler, Metadata, Value}; +use tokio::io::AsyncWriteExt; +use tokio::io::AsyncReadExt; -use crate::server_utils::tokio02::{self as tokio, io, net::TcpStream, time::Delay}; - use tokio::io::AsyncWriteExt; - use tokio::io::AsyncReadExt; +use crate::server_utils::tokio02::{self as tokio, net::TcpStream}; +use crate::futures03; use parking_lot::Mutex; @@ -50,12 +50,7 @@ fn doc_test_connect() { run_future(async move { TcpStream::connect(&addr).await - }).unwrap(); - // let stream = TcpStream::connect(&addr) - // .and_then(move |_stream| Ok(())) - // .map_err(|err| panic!("Server connection error: {:?}", err)); - - // tokio::run(stream); + }).expect("Server connection error"); } #[test] @@ -71,14 +66,6 @@ fn disconnect() { assert_eq!(stream.peer_addr().unwrap(), addr); stream.shutdown(::std::net::Shutdown::Both).unwrap(); }); - // let stream = TcpStream::connect(&addr) - // .and_then(move |stream| { - // assert_eq!(stream.peer_addr().unwrap(), addr); - // stream.shutdown(::std::net::Shutdown::Both) - // }) - // .map_err(|err| panic!("Error disconnecting: {:?}", err)); - - // tokio::run(stream); ::std::thread::sleep(::std::time::Duration::from_millis(50)); @@ -86,9 +73,7 @@ fn disconnect() { } fn dummy_request(addr: &SocketAddr, data: Vec) -> Vec { - let (ret_tx, ret_rx) = futures01::sync::oneshot::channel(); - - use futures03::TryFutureExt; + let (ret_tx, ret_rx) = std::sync::mpsc::channel(); let stream = async move { let mut stream = TcpStream::connect(addr).await?; @@ -103,19 +88,7 @@ fn dummy_request(addr: &SocketAddr, data: Vec) -> Vec { }; run_future(stream).unwrap(); - ret_rx.wait().expect("Unable to receive result") - - // let stream = TcpStream::connect(addr) - // .and_then(move |stream| io::write_all(stream, data)) - // .and_then(|(stream, _data)| { - // stream.shutdown(Shutdown::Write).unwrap(); - // io::read_to_end(stream, vec![]) - // }) - // .and_then(move |(_stream, read_buf)| ret_tx.send(read_buf).map_err(|err| panic!("Unable to send {:?}", err))) - // .map_err(|err| panic!("Error connecting or closing connection: {:?}", err)); - - // tokio::run(stream); - // ret_rx.wait().expect("Unable to receive result") + ret_rx.recv().expect("Unable to receive result") } fn dummy_request_str(addr: &SocketAddr, data: Vec) -> String { @@ -264,20 +237,17 @@ fn message() { let _server = server.start(&addr).expect("Server must run with no issues"); - // let delay = futures03::future::lazy(|_| tokio::time::delay_for(Duration::from_millis(500))); - // let delay = Delay::new(Instant::now() + Duration::from_millis(500)).map_err(|err| panic!("{:?}", err)); - let message = "ping"; let executed_dispatch = Arc::new(Mutex::new(false)); let executed_request = Arc::new(Mutex::new(false)); let executed_dispatch_move = executed_dispatch.clone(); let executed_request_move = executed_request.clone(); - // CLIENT RUN - let stream = async move { - let mut stream = TcpStream::connect(&addr).await?; - tokio::time::delay_for(Duration::from_millis(500)).await; - // TODO: delay + let client = async move { + let stream = TcpStream::connect(&addr); + let delay = tokio::time::delay_for(Duration::from_millis(500)); + let (stream, _) = futures03::join!(stream, delay); + let mut stream = stream?; let peer_addr = peer_list.lock()[0].clone(); dispatcher @@ -321,59 +291,8 @@ fn message() { Ok::<(), Box>(()) }; - run_future(stream).unwrap(); - // let stream = TcpStream::connect(&addr) - // .and_then(|stream| future::ok(stream).join(delay)) - // .and_then(move |stream| { - // let peer_addr = peer_list.lock()[0].clone(); - // dispatcher - // .push_message(&peer_addr, message.to_owned()) - // .expect("Should be sent with no errors"); - // trace!(target: "tcp", "Dispatched message for {}", peer_addr); - // future::ok(stream) - // }) - // .and_then(move |(stream, _)| { - // // Read message plus newline appended by codec. - // io::read_exact(stream, vec![0u8; message.len() + 1]) - // }) - // .and_then(move |(stream, read_buf)| { - // trace!(target: "tcp", "Read ping message"); - // let ping_signal = read_buf[..].to_vec(); - - // assert_eq!( - // format!("{}\n", message), - // String::from_utf8(ping_signal).expect("String should be utf-8"), - // "Sent request does not match received by the peer", - // ); - // // ensure that the above assert was actually triggered - // *executed_dispatch_move.lock() = true; - - // future::ok(stream) - // }) - // .and_then(|stream| { - // // make request AFTER message dispatches - // let data = b"{\"jsonrpc\": \"2.0\", \"method\": \"say_hello\", \"params\": [42, 23], \"id\": 1}\n"; - // io::write_all(stream, &data[..]) - // }) - // .and_then(|(stream, _)| { - // stream.shutdown(Shutdown::Write).unwrap(); - // io::read_to_end(stream, Vec::new()) - // }) - // .and_then(move |(_, read_buf)| { - // trace!(target: "tcp", "Read response message"); - // let response_signal = read_buf[..].to_vec(); - // assert_eq!( - // "{\"jsonrpc\":\"2.0\",\"result\":\"hello\",\"id\":1}\n", - // String::from_utf8(response_signal).expect("String should be utf-8"), - // "Response does not match the expected handling", - // ); - // *executed_request_move.lock() = true; - - // future::ok(()) - // }) - // .map_err(|err| panic!("Dispach message error: {:?}", err)); - - // tokio::run(stream); + run_future(client).unwrap(); + assert!(*executed_dispatch.lock()); assert!(*executed_request.lock()); } From ab00637c3f22d6c1c6c7db6ad2f0614ec9338015 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Fri, 18 Dec 2020 13:16:33 +0100 Subject: [PATCH 30/73] Clean up TCP server a bit more --- tcp/src/dispatch.rs | 2 +- tcp/src/server.rs | 69 ++++++++++++++++++--------------------------- 2 files changed, 28 insertions(+), 43 deletions(-) diff --git a/tcp/src/dispatch.rs b/tcp/src/dispatch.rs index ae3a2c5c5..2d6c4293a 100644 --- a/tcp/src/dispatch.rs +++ b/tcp/src/dispatch.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use std::net::SocketAddr; use std::sync::Arc; -use crate::jsonrpc::futures::{self as futures03, channel::mpsc}; +use crate::futures03::{self, channel::mpsc}; use futures03::Stream; use std::task::Poll; diff --git a/tcp/src/server.rs b/tcp/src/server.rs index 131b27106..0036f117a 100644 --- a/tcp/src/server.rs +++ b/tcp/src/server.rs @@ -1,12 +1,14 @@ use std; +use std::io; use std::net::SocketAddr; use std::sync::Arc; +use std::pin::Pin; use tower_service::Service as _; use crate::jsonrpc::{middleware, MetaIoHandler, Metadata, Middleware}; use crate::server_utils::{codecs, reactor, tokio02, tokio_util::codec::Framed, SuspendableStream}; -use crate::futures03; +use crate::futures03::{self, future}; use crate::dispatch::{Dispatcher, PeerMessageQueue, SenderChannels}; use crate::meta::{MetaExtractor, NoopExtractor, RequestContext}; @@ -77,7 +79,7 @@ where } /// Starts a new server - pub fn start(self, addr: &SocketAddr) -> std::io::Result { + pub fn start(self, addr: &SocketAddr) -> io::Result { let meta_extractor = self.meta_extractor.clone(); let rpc_handler = self.handler.clone(); let channels = self.channels.clone(); @@ -89,23 +91,24 @@ where let executor = self.executor.initialize()?; - use futures03::StreamExt; + use futures03::{FutureExt, SinkExt, StreamExt, TryStreamExt}; executor.executor().spawn(async move { - let start = async move { + let start = async { let listener = tokio02::net::TcpListener::bind(&address).await?; let connections = SuspendableStream::new(listener); - let server = connections.map(move |socket| { + let server = connections.map(|socket| { let peer_addr = match socket.peer_addr() { Ok(addr) => addr, Err(e) => { warn!(target: "tcp", "Unable to determine socket peer address, ignoring connection {}", e); - let fut = Box::pin(async move { }); - return futures03::future::Either::Left(fut); + return future::Either::Left(async { + io::Result::Ok(()) + }); } }; trace!(target: "tcp", "Accepted incoming connection from {}", &peer_addr); - let (sender, receiver) = crate::jsonrpc::futures::channel::mpsc::unbounded(); + let (sender, receiver) = futures03::channel::mpsc::unbounded(); let context = RequestContext { peer_addr, @@ -120,24 +123,21 @@ where ) .split(); - use futures03::TryStreamExt; - use futures03::SinkExt; - use futures03::FutureExt; // Work around https://github.com/rust-lang/rust/issues/64552 by boxing the stream type - let responses: std::pin::Pin> + Send>> = + let responses: Pin> + Send>> = Box::pin(reader.and_then(move |req| { service.call(req).then(|response| match response { Err(e) => { warn!(target: "tcp", "Error while processing request: {:?}", e); - futures03::future::ok(String::new()) + future::ok(String::new()) } Ok(None) => { trace!(target: "tcp", "JSON RPC request produced no response"); - futures03::future::ok(String::new()) + future::ok(String::new()) } Ok(Some(response_data)) => { trace!(target: "tcp", "Sent response: {}", &response_data); - futures03::future::ok(response_data) + future::ok(response_data) } }) })); @@ -150,14 +150,15 @@ where }; let shared_channels = channels.clone(); - let writer = Box::pin(async move { - let _ = writer.send_all(&mut peer_message_queue).await; - trace!(target: "tcp", "Peer {}: service finished", peer_addr); - let mut channels = shared_channels.lock(); - channels.remove(&peer_addr); - }); - - futures03::future::Either::Right(writer) + let writer = async move { + writer.send_all(&mut peer_message_queue).await?; + trace!(target: "tcp", "Peer {}: service finished", peer_addr); + let mut channels = shared_channels.lock(); + channels.remove(&peer_addr); + Ok(()) + }; + + future::Either::Right(writer) }); Ok(server) @@ -168,29 +169,13 @@ where tx.send(Ok(())).expect("Rx is blocking parent thread."); let server = server .buffer_unordered(1024) - .for_each(|_| async { () }) - ; - - let select = futures03::future::select(Box::pin(server), stop_rx); - select.await; - - // future::Either::A( - // server - // .buffer_unordered(1024) - // // .for_each(|_| async { Ok(()) }) - // .select(stop) - // .map(|_| ()) - // .map_err(|(e, _)| { - // error!("Error while executing the server: {:?}", e); - // }), - // ) + .for_each(|_| async { () }); + + future::select(Box::pin(server), stop_rx).await; } Err(e) => { tx.send(Err(e)).expect("Rx is blocking parent thread."); let _ = stop_rx.await; - // future::Either::B(stop.map_err(|e| { - // error!("Error while executing the server: {:?}", e); - // })) } } }); From 0ce3ac3d5067d9b58484bb7e89a5fb81544d46ca Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Fri, 18 Dec 2020 13:23:09 +0100 Subject: [PATCH 31/73] Migrate to Tokio 0.2 for WS server --- ws/Cargo.toml | 2 +- ws/src/lib.rs | 2 +- ws/src/metadata.rs | 5 +++-- ws/src/server_builder.rs | 11 +++++------ ws/src/session.rs | 5 +++-- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/ws/Cargo.toml b/ws/Cargo.toml index 2247d1098..9a4775020 100644 --- a/ws/Cargo.toml +++ b/ws/Cargo.toml @@ -13,7 +13,7 @@ version = "16.0.0" futures01 = { version = "0.1", package = "futures" } futures03 = { version = "0.3", package = "futures", features = [ "compat" ] } jsonrpc-core = { version = "16.0", path = "../core" } -jsonrpc-server-utils = { version = "16.0", path = "../server-utils" } +jsonrpc-server-utils = { version = "16.0", path = "../server-utils", default-features = false, features = ["tokio02"] } log = "0.4" parking_lot = "0.11.0" slab = "0.4" diff --git a/ws/src/lib.rs b/ws/src/lib.rs index 5c1cfc3ca..9f41cb651 100644 --- a/ws/src/lib.rs +++ b/ws/src/lib.rs @@ -27,5 +27,5 @@ pub use self::server_builder::ServerBuilder; pub use self::server_utils::cors::Origin; pub use self::server_utils::hosts::{DomainsValidation, Host}; pub use self::server_utils::session::{SessionId, SessionStats}; -pub use self::server_utils::tokio; +pub use self::server_utils::tokio02 as tokio; pub use self::session::{MiddlewareAction, RequestMiddleware}; diff --git a/ws/src/metadata.rs b/ws/src/metadata.rs index 95b555481..4b642f45a 100644 --- a/ws/src/metadata.rs +++ b/ws/src/metadata.rs @@ -3,7 +3,7 @@ use std::sync::{atomic, Arc}; use crate::core; use crate::core::futures::channel::mpsc; -use crate::server_utils::{session, tokio_compat::runtime::TaskExecutor}; +use crate::server_utils::{session, reactor::TaskExecutor}; use crate::ws; use crate::error; @@ -79,11 +79,12 @@ impl RequestContext { /// Get this session as a `Sink` spawning a new future /// in the underlying event loop. pub fn sender(&self) -> mpsc::UnboundedSender { + use futures03::compat::Future01CompatExt; use futures03::{StreamExt, TryStreamExt}; let out = self.out.clone(); let (sender, receiver) = mpsc::unbounded(); let receiver = receiver.map(Ok).compat(); - self.executor.spawn(SenderFuture(out, Box::new(receiver))); + self.executor.spawn(SenderFuture(out, Box::new(receiver)).compat()); sender } } diff --git a/ws/src/server_builder.rs b/ws/src/server_builder.rs index 10124462b..d2b6bcd69 100644 --- a/ws/src/server_builder.rs +++ b/ws/src/server_builder.rs @@ -2,7 +2,6 @@ use std::net::SocketAddr; use std::sync::Arc; use crate::core; -use crate::server_utils; use crate::server_utils::cors::Origin; use crate::server_utils::hosts::{DomainsValidation, Host}; use crate::server_utils::reactor::UninitializedExecutor; @@ -68,11 +67,11 @@ where } } - /// Utilize existing event loop executor to poll RPC results. - pub fn event_loop_executor(mut self, executor: server_utils::tokio_compat::runtime::TaskExecutor) -> Self { - self.executor = UninitializedExecutor::Shared(executor); - self - } + // /// Utilize existing event loop executor to poll RPC results. + // pub fn event_loop_executor(mut self, executor: server_utils::tokio_compat::runtime::TaskExecutor) -> Self { + // self.executor = UninitializedExecutor::Shared(executor); + // self + // } /// Sets a meta extractor. pub fn session_meta_extractor>(mut self, extractor: T) -> Self { diff --git a/ws/src/session.rs b/ws/src/session.rs index 43fd4be77..3855fddfd 100644 --- a/ws/src/session.rs +++ b/ws/src/session.rs @@ -11,7 +11,7 @@ use slab::Slab; use crate::server_utils::cors::Origin; use crate::server_utils::hosts::Host; use crate::server_utils::session::{SessionId, SessionStats}; -use crate::server_utils::tokio_compat::runtime::TaskExecutor; +use crate::server_utils::reactor::TaskExecutor; use crate::server_utils::Pattern; use crate::ws; @@ -294,7 +294,8 @@ where .map(|_| ()) .map_err(|_| ()); - self.executor.spawn(future); + use futures03::compat::Future01CompatExt; + self.executor.spawn(future.compat()); Ok(()) } From ce0874b9628dddfa19a470ed04464dcc9316ee83 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Fri, 18 Dec 2020 19:37:28 +0100 Subject: [PATCH 32/73] WIP: Adapt HTTP client to use Tokio 0.2 --- core-client/transports/Cargo.toml | 11 ++--- core-client/transports/src/transports/http.rs | 40 +++++++++++++------ 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/core-client/transports/Cargo.toml b/core-client/transports/Cargo.toml index f40b3760d..7713a034c 100644 --- a/core-client/transports/Cargo.toml +++ b/core-client/transports/Cargo.toml @@ -21,7 +21,7 @@ categories = [ [features] default = ["http", "tls", "ws"] tls = ["hyper-tls", "http"] -http = ["hyper", "futures01"] +http = ["hyper", "futures01", "tokio02/full"] ws = [ "websocket", "tokio02", @@ -44,22 +44,19 @@ serde_json = "1.0" url = "1.7" futures01 = { version = "0.1.26", package = "futures", optional = true } -hyper = { version = "0.12", optional = true } -hyper-tls = { version = "0.3.2", optional = true } +hyper = { version = "0.13", optional = true } +hyper-tls = { version = "0.4", optional = true } jsonrpc-server-utils = { version = "16.0", path = "../../server-utils", optional = true, default-features = false } parity-tokio-ipc = { version = "0.7", optional = true } -tokio = { version = "0.1", optional = true } websocket = { version = "0.24", optional = true } tokio02 = { package = "tokio", version = "0.2", features = ["macros"], optional = true } [dev-dependencies] assert_matches = "1.1" -# FIXME: Uncomment once http migrates to Tokio 0.2 -# jsonrpc-http-server = { version = "16.0", path = "../../http" } +jsonrpc-http-server = { version = "16.0", path = "../../http" } jsonrpc-ipc-server = { version = "16.0", path = "../../ipc" } lazy_static = "1.0" env_logger = "0.7" -tokio02 = { package = "tokio", version = "0.2", features = ["macros"] } [badges] travis-ci = { repository = "paritytech/jsonrpc", branch = "master" } diff --git a/core-client/transports/src/transports/http.rs b/core-client/transports/src/transports/http.rs index b4b7cb9ae..4fda3c0a3 100644 --- a/core-client/transports/src/transports/http.rs +++ b/core-client/transports/src/transports/http.rs @@ -5,7 +5,7 @@ use super::RequestBuilder; use crate::{RpcChannel, RpcError, RpcMessage, RpcResult}; use futures::{Future, FutureExt, StreamExt, TryFutureExt, TryStreamExt}; -use hyper::{http, rt, Client, Request, Uri}; +use hyper::{http, Client, Request, Uri}; /// Create a HTTP Client pub fn connect(url: &str) -> impl Future> @@ -16,17 +16,27 @@ where let url = url.to_owned(); std::thread::spawn(move || { - let connect = rt::lazy(move || { + let connect = futures01::lazy(move || { do_connect(&url) .map(|client| { if sender.send(client).is_err() { panic!("The caller did not wait for the server."); } - Ok(()) + Ok::<(), ()>(()) }) .compat() }); - rt::run(connect); + use futures::compat::Future01CompatExt; + let mut rt = tokio02::runtime::Builder::new() + .basic_scheduler() + .enable_io() + .enable_time() + .build() + .unwrap(); + let fut = connect.compat().map(drop); + rt.block_on(fut); + // FIXME: This keeps Tokio 0.2 runtime alive + let _ = rt.block_on(futures01::future::empty::<(), ()>().compat()); }); receiver.map(|res| res.expect("Server closed prematurely.").map(TClient::from)) @@ -42,10 +52,7 @@ fn do_connect(url: &str) -> impl Future> { }; #[cfg(feature = "tls")] - let connector = match hyper_tls::HttpsConnector::new(4) { - Ok(connector) => connector, - Err(e) => return ready(Err(RpcError::Other(Box::new(e)))), - }; + let connector = hyper_tls::HttpsConnector::new(); #[cfg(feature = "tls")] let client = Client::builder().build::<_, hyper::Body>(connector); @@ -85,7 +92,7 @@ fn do_connect(url: &str) -> impl Future> { .body(request.into()) .expect("Uri and request headers are valid; qed"); - Some(client.request(request).then(move |response| Ok((response, sender)))) + Some(client.request(request).compat().then(move |response| Ok((response, sender)))) }) .buffer_unordered(max_parallel) .for_each(|(result, sender)| { @@ -101,10 +108,14 @@ fn do_connect(url: &str) -> impl Future> { res.status() )))) } - Ok(res) => B(res - .into_body() + Ok(res) => B( + Box::pin(hyper::body::to_bytes(res.into_body())).compat() .map_err(|e| RpcError::ParseError(e.to_string(), Box::new(e))) - .concat2()), + // res + // .into_body() + // .map_err(|e| RpcError::ParseError(e.to_string(), Box::new(e))) + // .concat2() + ), Err(err) => A(future::err(RpcError::Other(Box::new(err)))), }; future.then(|result| { @@ -123,7 +134,10 @@ fn do_connect(url: &str) -> impl Future> { }) }); - rt::spawn(fut.map_err(|e: RpcError| log::error!("RPC Client error: {:?}", e))); + let fut = fut.map_err(|e: RpcError| log::error!("RPC Client error: {:?}", e)); + use futures::compat::Future01CompatExt; + tokio02::spawn(Box::pin(fut.compat())); + // tokio::spawn(fut.map_err(|e: RpcError| log::error!("RPC Client error: {:?}", e))); ready(Ok(sender.into())) } From 188f09beccbfcf0c158a725837be90b43f61988e Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Fri, 18 Dec 2020 20:14:04 +0100 Subject: [PATCH 33/73] Prune tokio 0.1 from server-utils entirely --- server-utils/Cargo.toml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/server-utils/Cargo.toml b/server-utils/Cargo.toml index 6943fe97a..2695e76c0 100644 --- a/server-utils/Cargo.toml +++ b/server-utils/Cargo.toml @@ -11,7 +11,6 @@ repository = "https://github.com/paritytech/jsonrpc" version = "16.0.0" [dependencies] -bytes04 = { version = "0.4", package = "bytes" } bytes05 = { version = "0.5", package = "bytes" } futures01 = { version = "0.1", package = "futures" } futures03 = { version = "0.3", package = "futures", features = ["compat"] } @@ -19,17 +18,13 @@ globset = "0.4" jsonrpc-core = { version = "16.0", path = "../core" } lazy_static = "1.1.0" log = "0.4" -tokio = { version = "0.1.15", default-features = false, features = ["timer", "tcp", "reactor", "rt-full"], optional = true } -tokio-codec = { version = "0.1", optional = true } tokio-util = { version = "0.3", features = ["codec", "compat"] } -tokio-compat = { version = "0.1", optional = true } tokio02 = { package = "tokio", version = "0.2", optional = true, features = ["rt-threaded", "io-driver", "io-util", "time", "tcp"] } unicase = "2.0" [features] -default = ["tokio-compat"] -# tokio-compat and tokio02 are mutually-exclusive +default = ["tokio02"] [badges] travis-ci = { repository = "paritytech/jsonrpc", branch = "master"} From 90d28eed1979f272a1850c2840458ef88438022c Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Fri, 18 Dec 2020 20:51:54 +0100 Subject: [PATCH 34/73] Migrate stdio server to Tokio 0.2 --- stdio/Cargo.toml | 6 ++---- stdio/src/lib.rs | 53 +++++++++++++++++++++++++++++++----------------- 2 files changed, 36 insertions(+), 23 deletions(-) diff --git a/stdio/Cargo.toml b/stdio/Cargo.toml index ef90c2093..e74e28867 100644 --- a/stdio/Cargo.toml +++ b/stdio/Cargo.toml @@ -13,10 +13,8 @@ version = "16.0.0" futures = { version = "0.3", features = [ "compat" ] } jsonrpc-core = { version = "16.0", path = "../core" } log = "0.4" -tokio = "0.1.7" -tokio-codec = "0.1.0" -tokio-io = "0.1.7" -tokio-stdin-stdout = "0.1.4" +tokio = { version = "0.2", features = ["io-std", "rt-core", "io-driver", "io-util"] } +tokio-util = { version = "0.3", features = ["codec"] } [dev-dependencies] lazy_static = "1.0" diff --git a/stdio/src/lib.rs b/stdio/src/lib.rs index 9d118f563..a31dbf6b2 100644 --- a/stdio/src/lib.rs +++ b/stdio/src/lib.rs @@ -17,17 +17,15 @@ #![deny(missing_docs)] -use tokio; -use tokio_stdin_stdout; +use std::sync::Arc; + #[macro_use] extern crate log; pub use jsonrpc_core; use jsonrpc_core::{MetaIoHandler, Metadata, Middleware}; -use std::sync::Arc; -use tokio::prelude::{Future, Stream}; -use tokio_codec::{FramedRead, FramedWrite, LinesCodec}; +use tokio_util::codec::{FramedRead, LinesCodec}; /// Stdio server builder pub struct ServerBuilder = jsonrpc_core::NoopMiddleware> { @@ -51,27 +49,44 @@ where /// The server reads from STDIN line-by-line, one request is taken /// per line and each response is written to STDOUT on a new line. pub fn build(&self) { - let stdin = tokio_stdin_stdout::stdin(0); - let stdout = tokio_stdin_stdout::stdout(0).make_sendable(); + let stdin = tokio::io::stdin(); + let mut stdout = tokio::io::stdout(); - let framed_stdin = FramedRead::new(stdin, LinesCodec::new()); - let framed_stdout = FramedWrite::new(stdout, LinesCodec::new()); + let mut framed_stdin = FramedRead::new(stdin, LinesCodec::new()); let handler = self.handler.clone(); - let future = framed_stdin - .and_then(move |line| Self::process(&handler, line).map_err(|_| unreachable!())) - .forward(framed_stdout) - .map(|_| ()) - .map_err(|e| panic!("{:?}", e)); - - tokio::run(future); + use futures::StreamExt; + let future = async { + while let Some(request) = framed_stdin.next().await { + match request { + Ok(line) => { + let res = Self::process(&handler, line).await; + let mut sanitized = res.replace('\n', ""); + sanitized.push('\n'); + use tokio::io::AsyncWriteExt; + if let Err(e) = stdout.write_all(sanitized.as_bytes()).await { + log::warn!("Error writing response: {:?}", e); + } + } + Err(e) => { + log::warn!("Error reading line: {:?}", e); + } + } + } + }; + let mut rt = tokio::runtime::Builder::new() + .basic_scheduler() + .enable_io() + .build() + .unwrap(); + rt.block_on(future); } /// Process a request asynchronously - fn process(io: &Arc>, input: String) -> impl Future + Send { - use jsonrpc_core::futures::{FutureExt, TryFutureExt}; + fn process(io: &Arc>, input: String) -> impl std::future::Future + Send { + use jsonrpc_core::futures::{FutureExt}; let f = io.handle_request(&input, Default::default()); - f.map(Ok).compat().map(move |result| match result { + f.map(move |result| match result { Some(res) => res, None => { info!("JSON RPC request produced no response: {:?}", input); From def4a9884ae48af3ddaca43ae5b4132380fb3980 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Fri, 18 Dec 2020 21:08:04 +0100 Subject: [PATCH 35/73] Remove tokio-uds from ipc server --- ipc/Cargo.toml | 1 - ipc/src/server.rs | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ipc/Cargo.toml b/ipc/Cargo.toml index 7ff9292e1..866ef839f 100644 --- a/ipc/Cargo.toml +++ b/ipc/Cargo.toml @@ -24,7 +24,6 @@ env_logger = "0.7" lazy_static = "1.0" [target.'cfg(not(windows))'.dev-dependencies] -tokio-uds = "0.2" tokio02 = { package = "tokio", version = "0.2", default-features = false, features = ["uds", "time", "rt-threaded", "io-driver"] } [badges] diff --git a/ipc/src/server.rs b/ipc/src/server.rs index 4566ca717..435aea4e4 100644 --- a/ipc/src/server.rs +++ b/ipc/src/server.rs @@ -336,7 +336,7 @@ mod tests { use jsonrpc_core::Value; use std::thread; use std::time::{self, Duration}; - use tokio_uds::UnixStream; + use std::os::unix::net::UnixStream; fn server_builder() -> ServerBuilder { let mut io = MetaIoHandler::<()>::default(); @@ -388,7 +388,7 @@ mod tests { let path = "/tmp/test-ipc-30000"; let _server = run(path); - UnixStream::connect(path).wait().expect("Socket should connect"); + UnixStream::connect(path).expect("Socket should connect"); } #[test] @@ -469,7 +469,7 @@ mod tests { "There should be no socket file left" ); assert!( - UnixStream::connect(path).wait().is_err(), + UnixStream::connect(path).is_err(), "Connection to the closed socket should fail" ); } @@ -565,7 +565,7 @@ mod tests { let builder = ServerBuilder::with_meta_extractor(io, session_metadata_extractor); let server = builder.start(path).expect("Server must run with no issues"); { - let _ = UnixStream::connect(path).wait().expect("Socket should connect"); + let _ = UnixStream::connect(path).expect("Socket should connect"); } receiver @@ -585,7 +585,7 @@ mod tests { let handle = server.close_handle(); handle.close(); assert!( - UnixStream::connect(path).wait().is_err(), + UnixStream::connect(path).is_err(), "Connection to the closed socket should fail" ); } @@ -620,7 +620,7 @@ mod tests { Ok((result, _)) => { assert_eq!(result, true, "Wait timeout exceeded"); assert!( - UnixStream::connect(path).wait().is_err(), + UnixStream::connect(path).is_err(), "Connection to the closed socket should fail" ); Ok(()) From c0d80fb3b64bccca3442c2f7134de01adadc5752 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Fri, 18 Dec 2020 21:16:57 +0100 Subject: [PATCH 36/73] Remove some Tokio 0.1-related code --- server-utils/src/lib.rs | 4 -- server-utils/src/stream_codec.rs | 76 -------------------------- server-utils/src/suspendable_stream.rs | 62 ++------------------- 3 files changed, 4 insertions(+), 138 deletions(-) diff --git a/server-utils/src/lib.rs b/server-utils/src/lib.rs index 60ba0208d..a7dbbc237 100644 --- a/server-utils/src/lib.rs +++ b/server-utils/src/lib.rs @@ -8,15 +8,11 @@ extern crate log; #[macro_use] extern crate lazy_static; -#[cfg(feature = "tokio")] -pub use tokio; pub use tokio_util; #[cfg(feature = "tokio-compat")] pub use tokio_compat; #[cfg(feature = "tokio02")] pub use tokio02; -#[cfg(feature = "tokio-codec")] -pub use tokio_codec; pub mod cors; pub mod hosts; diff --git a/server-utils/src/stream_codec.rs b/server-utils/src/stream_codec.rs index 448ecfc0f..76f8e08fc 100644 --- a/server-utils/src/stream_codec.rs +++ b/server-utils/src/stream_codec.rs @@ -106,82 +106,6 @@ impl tokio_util::codec::Decoder for StreamCodec { } } -#[cfg(feature = "tokio-codec")] -impl tokio_codec::Decoder for StreamCodec { - type Item = String; - type Error = io::Error; - - fn decode(&mut self, buf: &mut bytes04::BytesMut) -> io::Result> { - if let Separator::Byte(separator) = self.incoming_separator { - if let Some(i) = buf.as_ref().iter().position(|&b| b == separator) { - let line = buf.split_to(i); - buf.split_to(1); - - match str::from_utf8(&line.as_ref()) { - Ok(s) => Ok(Some(s.to_string())), - Err(_) => Err(io::Error::new(io::ErrorKind::Other, "invalid UTF-8")), - } - } else { - Ok(None) - } - } else { - let mut depth = 0; - let mut in_str = false; - let mut is_escaped = false; - let mut start_idx = 0; - let mut whitespaces = 0; - - for idx in 0..buf.as_ref().len() { - let byte = buf.as_ref()[idx]; - - if (byte == b'{' || byte == b'[') && !in_str { - if depth == 0 { - start_idx = idx; - } - depth += 1; - } else if (byte == b'}' || byte == b']') && !in_str { - depth -= 1; - } else if byte == b'"' && !is_escaped { - in_str = !in_str; - } else if is_whitespace(byte) { - whitespaces += 1; - } - if byte == b'\\' && !is_escaped && in_str { - is_escaped = true; - } else { - is_escaped = false; - } - - if depth == 0 && idx != start_idx && idx - start_idx + 1 > whitespaces { - let bts = buf.split_to(idx + 1); - match String::from_utf8(bts.as_ref().to_vec()) { - Ok(val) => return Ok(Some(val)), - Err(_) => { - return Ok(None); - } // skip non-utf requests (TODO: log error?) - }; - } - } - Ok(None) - } - } -} - -#[cfg(feature = "tokio-codec")] -impl tokio_codec::Encoder for StreamCodec { - type Item = String; - type Error = io::Error; - - fn encode(&mut self, msg: String, buf: &mut bytes04::BytesMut) -> io::Result<()> { - let mut payload = msg.into_bytes(); - if let Separator::Byte(separator) = self.outgoing_separator { - payload.push(separator); - } - buf.extend_from_slice(&payload); - Ok(()) - } -} - impl tokio_util::codec::Encoder for StreamCodec { type Error = io::Error; diff --git a/server-utils/src/suspendable_stream.rs b/server-utils/src/suspendable_stream.rs index a43dcea9f..7a79335a0 100644 --- a/server-utils/src/suspendable_stream.rs +++ b/server-utils/src/suspendable_stream.rs @@ -1,6 +1,9 @@ use std::io; +use std::future::Future; +use std::pin::Pin; +use std::task::Poll; use std::time::Duration; -// use futures01::prelude::*; + use tokio02::time::Delay; /// `Incoming` is a stream of incoming sockets @@ -33,63 +36,6 @@ impl SuspendableStream { } } -// impl futures01::Stream for SuspendableStream -// where -// S: futures01::Stream, -// { -// type Item = I; -// type Error = (); - -// fn poll(&mut self) -> Result>, ()> { -// loop { -// if let Some(timeout) = self.timeout.take() { -// let deadline = timeout.deadline(); -// use futures03::FutureExt; -// let mut compat = futures03::compat::Compat::new(timeout.map(Ok::<(), ()>)); -// match futures01::Future::poll(&mut compat) { -// Ok(Async::Ready(_)) => {} -// Ok(Async::NotReady) => { -// self.timeout = Some(tokio02::time::delay_until(deadline)); -// return Ok(Async::NotReady); -// } -// Err(err) => { -// warn!("Timeout error {:?}", err); -// futures01::task::current().notify(); -// return Ok(Async::NotReady); -// } -// } -// } - -// match self.stream.poll() { -// Ok(item) => { -// if self.next_delay > self.initial_delay { -// self.next_delay = self.initial_delay; -// } -// return Ok(item); -// } -// Err(ref err) => { -// if connection_error(err) { -// warn!("Connection Error: {:?}", err); -// continue; -// } -// self.next_delay = if self.next_delay < self.max_delay { -// self.next_delay * 2 -// } else { -// self.next_delay -// }; -// debug!("Error accepting connection: {}", err); -// debug!("The server will stop accepting connections for {:?}", self.next_delay); -// self.timeout = Some(tokio02::time::delay_for(self.next_delay)); -// } -// } -// } -// } -// } - -use std::future::Future; -use std::pin::Pin; -use std::task::Poll; - impl futures03::Stream for SuspendableStream where S: futures03::Stream> + Unpin, From 7f420a32b4aed15080a49dd59eb4c642623ba772 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Fri, 18 Dec 2020 21:19:36 +0100 Subject: [PATCH 37/73] Remove some commented code --- core-client/transports/src/transports/http.rs | 16 +++++------ http/src/handler.rs | 27 ------------------- 2 files changed, 6 insertions(+), 37 deletions(-) diff --git a/core-client/transports/src/transports/http.rs b/core-client/transports/src/transports/http.rs index 4fda3c0a3..9105266c1 100644 --- a/core-client/transports/src/transports/http.rs +++ b/core-client/transports/src/transports/http.rs @@ -28,11 +28,11 @@ where }); use futures::compat::Future01CompatExt; let mut rt = tokio02::runtime::Builder::new() - .basic_scheduler() - .enable_io() - .enable_time() - .build() - .unwrap(); + .basic_scheduler() + .enable_io() + .enable_time() + .build() + .unwrap(); let fut = connect.compat().map(drop); rt.block_on(fut); // FIXME: This keeps Tokio 0.2 runtime alive @@ -110,11 +110,7 @@ fn do_connect(url: &str) -> impl Future> { } Ok(res) => B( Box::pin(hyper::body::to_bytes(res.into_body())).compat() - .map_err(|e| RpcError::ParseError(e.to_string(), Box::new(e))) - // res - // .into_body() - // .map_err(|e| RpcError::ParseError(e.to_string(), Box::new(e))) - // .concat2() + .map_err(|e| RpcError::ParseError(e.to_string(), Box::new(e))) ), Err(err) => A(future::err(RpcError::Other(Box::new(err)))), }; diff --git a/http/src/handler.rs b/http/src/handler.rs index b032e701f..9205732f6 100644 --- a/http/src/handler.rs +++ b/http/src/handler.rs @@ -64,8 +64,6 @@ where S::CallFuture: Unpin, { type Response = hyper::Response; - // type ReqBody = Body; - // type ResBody = Body; type Error = hyper::Error; type Future = futures03::compat::Compat01As03>; @@ -150,31 +148,6 @@ where } } -// impl> std::future::Future for Handler -// where -// S::Future: Unpin, -// S::CallFuture: Unpin, -// { -// type Output = hyper::Result>; -// // type Error = hyper::Error; - -// fn poll(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> task::Poll { -// let lol = Pin::into_inner(self); -// use std::future::Future as StdFuture; -// use futures03::compat::Future01CompatExt; -// match *self { -// Handler::Rpc(ref mut handler) => StdFuture::poll(Pin::new(&mut handler.compat()), cx), -// Handler::Middleware(ref mut middleware) => StdFuture::poll(Pin::new(&mut middleware.compat()), cx), -// Handler::Err(ref mut response) => std::task::Poll::Ready(Ok( -// response -// .take() -// .expect("Response always Some initialy. Returning `Ready` so will never be polled again; qed") -// .into(), -// )), -// } -// } -// } - enum RpcPollState { Ready(RpcHandlerState), NotReady(RpcHandlerState), From 9a9680c59176bff446bd7d2f6d236770f8e35961 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Fri, 18 Dec 2020 21:21:09 +0100 Subject: [PATCH 38/73] Remove some tokio-compat-related code --- server-utils/src/lib.rs | 2 -- server-utils/src/reactor.rs | 34 ---------------------------------- 2 files changed, 36 deletions(-) diff --git a/server-utils/src/lib.rs b/server-utils/src/lib.rs index a7dbbc237..fe9bfb7dd 100644 --- a/server-utils/src/lib.rs +++ b/server-utils/src/lib.rs @@ -9,8 +9,6 @@ extern crate log; extern crate lazy_static; pub use tokio_util; -#[cfg(feature = "tokio-compat")] -pub use tokio_compat; #[cfg(feature = "tokio02")] pub use tokio02; diff --git a/server-utils/src/reactor.rs b/server-utils/src/reactor.rs index 7e98eea0d..6aa2e4007 100644 --- a/server-utils/src/reactor.rs +++ b/server-utils/src/reactor.rs @@ -7,11 +7,6 @@ use std::io; -#[cfg(feature = "tokio-compat")] -use tokio_compat::runtime; -#[cfg(feature = "tokio-compat")] -/// Task executor for Tokio-compat runtime. -pub type TaskExecutor = tokio_compat::runtime::TaskExecutor; #[cfg(feature = "tokio02")] use tokio02::runtime; #[cfg(feature = "tokio02")] @@ -56,15 +51,6 @@ pub enum Executor { } impl Executor { - #[cfg(feature = "tokio-compat")] - /// Get tokio executor associated with this event loop. - pub fn executor(&self) -> TaskExecutor { - match self { - Executor::Shared(ref executor) => executor.clone(), - Executor::Spawned(ref eloop) => eloop.executor(), - } - } - #[cfg(feature = "tokio02")] /// Get tokio executor associated with this event loop. pub fn executor(&self) -> TaskExecutor { @@ -124,15 +110,6 @@ impl RpcEventLoop { let runtime = tb.build()?; let executor; - #[cfg(feature = "tokio-compat")] - { - executor = runtime.executor(); - if let Some(name) = name { - tb.name_prefix(name); - } - - runtime.spawn_std(async { let _ = stopped.await; }); - } #[cfg(feature = "tokio02")] { executor = runtime.handle().clone(); @@ -150,12 +127,6 @@ impl RpcEventLoop { }) } - /// Get executor for this event loop. - #[cfg(feature = "tokio-compat")] - pub fn executor(&self) -> TaskExecutor { - self.executor.clone() - } - /// Get executor for this event loop. #[cfg(feature = "tokio02")] pub fn executor(&self) -> runtime::Handle { @@ -167,11 +138,6 @@ impl RpcEventLoop { /// Blocks current thread and waits until the event loop is finished. pub fn wait(mut self) -> Result<(), ()> { - #[cfg(feature = "tokio-compat")] - { - use futures01::Future; - self.runtime.take().ok_or(())?.shutdown_on_idle().wait() - } #[cfg(feature = "tokio02")] { // Dropping Tokio 0.2 runtime waits for spawned task to shutdown by default From a9be6628bc967ef521a9f85252ad4a92ae20de98 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Fri, 18 Dec 2020 21:22:43 +0100 Subject: [PATCH 39/73] Match previous poll match order in tcp server --- tcp/src/dispatch.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tcp/src/dispatch.rs b/tcp/src/dispatch.rs index 2d6c4293a..04a8f02d1 100644 --- a/tcp/src/dispatch.rs +++ b/tcp/src/dispatch.rs @@ -97,10 +97,10 @@ impl>> Stream for PeerMessageQu let this = Pin::into_inner(self); let up_closed = match Pin::new(&mut this.up).poll_next(cx) { - Poll::Pending => false, - Poll::Ready(None) => true, Poll::Ready(Some(Ok(item))) => return Poll::Ready(Some(Ok(item))), Poll::Ready(Some(Err(err))) => return Poll::Ready(Some(Err(err))), + Poll::Ready(None) => true, + Poll::Pending => false, }; let mut rx = match &mut this.receiver { From ac60b45b9d54ae471c2ea876c6950528f84e931b Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Fri, 18 Dec 2020 21:25:06 +0100 Subject: [PATCH 40/73] Bring back event_loop_executors --- tcp/src/server.rs | 10 +++++----- ws/src/server_builder.rs | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tcp/src/server.rs b/tcp/src/server.rs index 0036f117a..9361b7203 100644 --- a/tcp/src/server.rs +++ b/tcp/src/server.rs @@ -59,11 +59,11 @@ where } } - // /// Utilize existing event loop executor. - // pub fn event_loop_executor(mut self, handle: tokio_compat::runtime::TaskExecutor) -> Self { - // self.executor = reactor::UninitializedExecutor::Shared(handle); - // self - // } + /// Utilize existing event loop executor. + pub fn event_loop_executor(mut self, handle: reactor::TaskExecutor) -> Self { + self.executor = reactor::UninitializedExecutor::Shared(handle); + self + } /// Sets session meta extractor pub fn session_meta_extractor + 'static>(mut self, meta_extractor: T) -> Self { diff --git a/ws/src/server_builder.rs b/ws/src/server_builder.rs index d2b6bcd69..2bfdcaadc 100644 --- a/ws/src/server_builder.rs +++ b/ws/src/server_builder.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use crate::core; use crate::server_utils::cors::Origin; use crate::server_utils::hosts::{DomainsValidation, Host}; -use crate::server_utils::reactor::UninitializedExecutor; +use crate::server_utils::reactor::{self, UninitializedExecutor}; use crate::server_utils::session::SessionStats; use crate::error::Result; @@ -67,11 +67,11 @@ where } } - // /// Utilize existing event loop executor to poll RPC results. - // pub fn event_loop_executor(mut self, executor: server_utils::tokio_compat::runtime::TaskExecutor) -> Self { - // self.executor = UninitializedExecutor::Shared(executor); - // self - // } + /// Utilize existing event loop executor to poll RPC results. + pub fn event_loop_executor(mut self, executor: reactor::TaskExecutor) -> Self { + self.executor = UninitializedExecutor::Shared(executor); + self + } /// Sets a meta extractor. pub fn session_meta_extractor>(mut self, extractor: T) -> Self { From 2642ad411ab58986a78d96b30fbc904c64fc5f95 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Fri, 18 Dec 2020 21:30:35 +0100 Subject: [PATCH 41/73] Minimize differences in server-utils/src/stream_codec.rs --- server-utils/Cargo.toml | 2 +- server-utils/src/stream_codec.rs | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/server-utils/Cargo.toml b/server-utils/Cargo.toml index 2695e76c0..3c8752971 100644 --- a/server-utils/Cargo.toml +++ b/server-utils/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/paritytech/jsonrpc" version = "16.0.0" [dependencies] -bytes05 = { version = "0.5", package = "bytes" } +bytes = { version = "0.5", package = "bytes" } futures01 = { version = "0.1", package = "futures" } futures03 = { version = "0.3", package = "futures", features = ["compat"] } globset = "0.4" diff --git a/server-utils/src/stream_codec.rs b/server-utils/src/stream_codec.rs index 76f8e08fc..4edef5add 100644 --- a/server-utils/src/stream_codec.rs +++ b/server-utils/src/stream_codec.rs @@ -1,3 +1,4 @@ +use bytes::BytesMut; use std::{io, str}; /// Separator for enveloping messages in streaming codecs @@ -50,7 +51,7 @@ impl tokio_util::codec::Decoder for StreamCodec { type Item = String; type Error = io::Error; - fn decode(&mut self, buf: &mut bytes05::BytesMut) -> io::Result> { + fn decode(&mut self, buf: &mut BytesMut) -> io::Result> { if let Separator::Byte(separator) = self.incoming_separator { if let Some(i) = buf.as_ref().iter().position(|&b| b == separator) { let line = buf.split_to(i); @@ -109,7 +110,7 @@ impl tokio_util::codec::Decoder for StreamCodec { impl tokio_util::codec::Encoder for StreamCodec { type Error = io::Error; - fn encode(&mut self, msg: String, buf: &mut bytes05::BytesMut) -> io::Result<()> { + fn encode(&mut self, msg: String, buf: &mut BytesMut) -> io::Result<()> { let mut payload = msg.into_bytes(); if let Separator::Byte(separator) = self.outgoing_separator { payload.push(separator); @@ -123,7 +124,7 @@ impl tokio_util::codec::Encoder for StreamCodec { mod tests { use super::StreamCodec; - use bytes05::{BufMut, BytesMut}; + use bytes::{BufMut, BytesMut}; use tokio_util::codec::Decoder; #[test] From 3f90c267f78373a6a4eb30a16d245f7098daecf5 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Fri, 18 Dec 2020 21:32:52 +0100 Subject: [PATCH 42/73] Minor improvements to tcp/src/dispatch.rs --- tcp/src/dispatch.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tcp/src/dispatch.rs b/tcp/src/dispatch.rs index 04a8f02d1..69fe9632d 100644 --- a/tcp/src/dispatch.rs +++ b/tcp/src/dispatch.rs @@ -2,11 +2,12 @@ use std; use std::collections::HashMap; use std::net::SocketAddr; use std::sync::Arc; +use std::pin::Pin; +use std::task::Poll; use crate::futures03::{self, channel::mpsc}; use futures03::Stream; -use std::task::Poll; use parking_lot::Mutex; @@ -79,9 +80,7 @@ impl Dispatcher { } } -use std::pin::Pin; - -impl>> Stream for PeerMessageQueue { +impl> + Unpin> Stream for PeerMessageQueue { type Item = std::io::Result; // The receiver will never return `Ok(Async::Ready(None))` From 3c6693c1a9f6bc0485f3bfe3f59cd5fabb09a08b Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Fri, 18 Dec 2020 21:39:54 +0100 Subject: [PATCH 43/73] Remove commented code in http client --- core-client/transports/src/transports/http.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-client/transports/src/transports/http.rs b/core-client/transports/src/transports/http.rs index 9105266c1..d9367d02b 100644 --- a/core-client/transports/src/transports/http.rs +++ b/core-client/transports/src/transports/http.rs @@ -133,7 +133,7 @@ fn do_connect(url: &str) -> impl Future> { let fut = fut.map_err(|e: RpcError| log::error!("RPC Client error: {:?}", e)); use futures::compat::Future01CompatExt; tokio02::spawn(Box::pin(fut.compat())); - // tokio::spawn(fut.map_err(|e: RpcError| log::error!("RPC Client error: {:?}", e))); + ready(Ok(sender.into())) } From 2e2e3d2ea7bb36b2c59f7bbf1378f306954c0161 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 21 Dec 2020 12:45:07 +0100 Subject: [PATCH 44/73] Polish and minimize the diff --- core-client/transports/Cargo.toml | 5 +++-- http/Cargo.toml | 2 +- ipc/Cargo.toml | 2 +- server-utils/Cargo.toml | 8 ++----- server-utils/src/lib.rs | 3 --- server-utils/src/reactor.rs | 36 ++++++++++--------------------- tcp/Cargo.toml | 2 +- ws/Cargo.toml | 2 +- 8 files changed, 20 insertions(+), 40 deletions(-) diff --git a/core-client/transports/Cargo.toml b/core-client/transports/Cargo.toml index 7713a034c..5186cc0a3 100644 --- a/core-client/transports/Cargo.toml +++ b/core-client/transports/Cargo.toml @@ -29,7 +29,8 @@ ws = [ ] ipc = [ "parity-tokio-ipc", - "jsonrpc-server-utils/tokio02", + "jsonrpc-server-utils", + "tokio02", ] arbitrary_precision = ["serde_json/arbitrary_precision", "jsonrpc-core/arbitrary_precision"] @@ -49,7 +50,7 @@ hyper-tls = { version = "0.4", optional = true } jsonrpc-server-utils = { version = "16.0", path = "../../server-utils", optional = true, default-features = false } parity-tokio-ipc = { version = "0.7", optional = true } websocket = { version = "0.24", optional = true } -tokio02 = { package = "tokio", version = "0.2", features = ["macros"], optional = true } +tokio02 = { package = "tokio", version = "0.2", optional = true, features = ["macros"] } [dev-dependencies] assert_matches = "1.1" diff --git a/http/Cargo.toml b/http/Cargo.toml index d4d146077..910dd4b5e 100644 --- a/http/Cargo.toml +++ b/http/Cargo.toml @@ -15,7 +15,7 @@ futures01 = { version = "0.1", package = "futures" } futures03 = { version = "0.3", package = "futures", features = ["compat"] } hyper = "0.13" jsonrpc-core = { version = "16.0", path = "../core" } -jsonrpc-server-utils = { version = "16.0", path = "../server-utils", default-features = false, features = ["tokio02"] } +jsonrpc-server-utils = { version = "16.0", path = "../server-utils" } log = "0.4" net2 = "0.2" parking_lot = "0.11.0" diff --git a/ipc/Cargo.toml b/ipc/Cargo.toml index 866ef839f..13bd4c381 100644 --- a/ipc/Cargo.toml +++ b/ipc/Cargo.toml @@ -15,7 +15,7 @@ futures03 = { version = "0.3", package = "futures", features = [ "compat" ] } log = "0.4" tower-service = "0.3" jsonrpc-core = { version = "16.0", path = "../core" } -jsonrpc-server-utils = { version = "16.0", path = "../server-utils", default-features = false, features = ["tokio02"] } +jsonrpc-server-utils = { version = "16.0", path = "../server-utils", default-features = false } parity-tokio-ipc = "0.7" parking_lot = "0.11.0" diff --git a/server-utils/Cargo.toml b/server-utils/Cargo.toml index 3c8752971..6398221a0 100644 --- a/server-utils/Cargo.toml +++ b/server-utils/Cargo.toml @@ -12,19 +12,15 @@ version = "16.0.0" [dependencies] bytes = { version = "0.5", package = "bytes" } -futures01 = { version = "0.1", package = "futures" } -futures03 = { version = "0.3", package = "futures", features = ["compat"] } +futures03 = { version = "0.3", package = "futures" } globset = "0.4" jsonrpc-core = { version = "16.0", path = "../core" } lazy_static = "1.1.0" log = "0.4" tokio-util = { version = "0.3", features = ["codec", "compat"] } -tokio02 = { package = "tokio", version = "0.2", optional = true, features = ["rt-threaded", "io-driver", "io-util", "time", "tcp"] } +tokio02 = { package = "tokio", version = "0.2", features = ["rt-threaded", "io-driver", "io-util", "time", "tcp"] } unicase = "2.0" -[features] -default = ["tokio02"] - [badges] travis-ci = { repository = "paritytech/jsonrpc", branch = "master"} diff --git a/server-utils/src/lib.rs b/server-utils/src/lib.rs index fe9bfb7dd..0fde97169 100644 --- a/server-utils/src/lib.rs +++ b/server-utils/src/lib.rs @@ -9,7 +9,6 @@ extern crate log; extern crate lazy_static; pub use tokio_util; -#[cfg(feature = "tokio02")] pub use tokio02; pub mod cors; @@ -18,11 +17,9 @@ mod matcher; pub mod reactor; pub mod session; mod stream_codec; -#[cfg(feature = "tokio02")] mod suspendable_stream; pub use crate::matcher::Pattern; -#[cfg(feature = "tokio02")] pub use crate::suspendable_stream::SuspendableStream; /// Codecs utilities diff --git a/server-utils/src/reactor.rs b/server-utils/src/reactor.rs index 6aa2e4007..d83bf9cb3 100644 --- a/server-utils/src/reactor.rs +++ b/server-utils/src/reactor.rs @@ -7,9 +7,7 @@ use std::io; -#[cfg(feature = "tokio02")] use tokio02::runtime; -#[cfg(feature = "tokio02")] /// Task executor for Tokio 0.2 runtime. pub type TaskExecutor = tokio02::runtime::Handle; @@ -51,7 +49,6 @@ pub enum Executor { } impl Executor { - #[cfg(feature = "tokio02")] /// Get tokio executor associated with this event loop. pub fn executor(&self) -> TaskExecutor { match self { @@ -101,24 +98,17 @@ impl RpcEventLoop { let mut tb = runtime::Builder::new(); tb.core_threads(1); + tb.threaded_scheduler(); + tb.enable_all(); - #[cfg(feature = "tokio02")] - { - tb.threaded_scheduler(); - tb.enable_io(); + if let Some(name) = name { + tb.thread_name(name); } let runtime = tb.build()?; - let executor; - #[cfg(feature = "tokio02")] - { - executor = runtime.handle().clone(); - if let Some(name) = name { - tb.thread_name(name); - } - - runtime.spawn(async { let _ = stopped.await; }); - } + let executor = runtime.handle().to_owned(); + + runtime.spawn(async { let _ = stopped.await; }); Ok(RpcEventLoop { executor, @@ -128,7 +118,6 @@ impl RpcEventLoop { } /// Get executor for this event loop. - #[cfg(feature = "tokio02")] pub fn executor(&self) -> runtime::Handle { self.runtime.as_ref() .expect("Runtime is only None if we're being dropped; qed") @@ -138,13 +127,10 @@ impl RpcEventLoop { /// Blocks current thread and waits until the event loop is finished. pub fn wait(mut self) -> Result<(), ()> { - #[cfg(feature = "tokio02")] - { - // Dropping Tokio 0.2 runtime waits for spawned task to shutdown by default - let runtime = self.runtime.take().ok_or(())?; - drop(runtime); - Ok(()) - } + // Dropping Tokio 0.2 runtime waits for all spawned tasks to terminate + let runtime = self.runtime.take().ok_or(())?; + drop(runtime); + Ok(()) } /// Finishes this event loop. diff --git a/tcp/Cargo.toml b/tcp/Cargo.toml index 7c58991f1..cda8542d2 100644 --- a/tcp/Cargo.toml +++ b/tcp/Cargo.toml @@ -11,7 +11,7 @@ version = "16.0.0" [dependencies] jsonrpc-core = { version = "16.0", path = "../core" } -jsonrpc-server-utils = { version = "16.0", path = "../server-utils", default-features = false, features = ["tokio02"] } +jsonrpc-server-utils = { version = "16.0", path = "../server-utils" } log = "0.4" parking_lot = "0.11.0" tower-service = "0.3" diff --git a/ws/Cargo.toml b/ws/Cargo.toml index 9a4775020..2247d1098 100644 --- a/ws/Cargo.toml +++ b/ws/Cargo.toml @@ -13,7 +13,7 @@ version = "16.0.0" futures01 = { version = "0.1", package = "futures" } futures03 = { version = "0.3", package = "futures", features = [ "compat" ] } jsonrpc-core = { version = "16.0", path = "../core" } -jsonrpc-server-utils = { version = "16.0", path = "../server-utils", default-features = false, features = ["tokio02"] } +jsonrpc-server-utils = { version = "16.0", path = "../server-utils" } log = "0.4" parking_lot = "0.11.0" slab = "0.4" From 9aac40c9efe6b3e2bcded08b1ce7cd9b79f35139 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 21 Dec 2020 12:56:15 +0100 Subject: [PATCH 45/73] Polish and minimize the diff pt 2 --- core-client/transports/Cargo.toml | 10 +++++----- core-client/transports/src/transports/http.rs | 4 ++-- core-client/transports/src/transports/ipc.rs | 10 +++++----- core-client/transports/src/transports/ws.rs | 2 +- http/src/lib.rs | 6 +++--- server-utils/Cargo.toml | 2 +- server-utils/src/lib.rs | 2 +- server-utils/src/reactor.rs | 4 ++-- server-utils/src/suspendable_stream.rs | 4 ++-- tcp/src/lib.rs | 2 +- tcp/src/server.rs | 4 ++-- tcp/src/tests.rs | 2 +- ws/src/lib.rs | 2 +- 13 files changed, 27 insertions(+), 27 deletions(-) diff --git a/core-client/transports/Cargo.toml b/core-client/transports/Cargo.toml index 5186cc0a3..6f40b9951 100644 --- a/core-client/transports/Cargo.toml +++ b/core-client/transports/Cargo.toml @@ -21,16 +21,16 @@ categories = [ [features] default = ["http", "tls", "ws"] tls = ["hyper-tls", "http"] -http = ["hyper", "futures01", "tokio02/full"] +http = ["hyper", "futures01", "tokio/full"] ws = [ "websocket", - "tokio02", + "tokio", "futures01", ] ipc = [ "parity-tokio-ipc", "jsonrpc-server-utils", - "tokio02", + "tokio", ] arbitrary_precision = ["serde_json/arbitrary_precision", "jsonrpc-core/arbitrary_precision"] @@ -47,10 +47,10 @@ url = "1.7" futures01 = { version = "0.1.26", package = "futures", optional = true } hyper = { version = "0.13", optional = true } hyper-tls = { version = "0.4", optional = true } -jsonrpc-server-utils = { version = "16.0", path = "../../server-utils", optional = true, default-features = false } +jsonrpc-server-utils = { version = "16.0", path = "../../server-utils", optional = true } parity-tokio-ipc = { version = "0.7", optional = true } +tokio = { version = "0.2", optional = true } websocket = { version = "0.24", optional = true } -tokio02 = { package = "tokio", version = "0.2", optional = true, features = ["macros"] } [dev-dependencies] assert_matches = "1.1" diff --git a/core-client/transports/src/transports/http.rs b/core-client/transports/src/transports/http.rs index d9367d02b..f2502f1d0 100644 --- a/core-client/transports/src/transports/http.rs +++ b/core-client/transports/src/transports/http.rs @@ -27,7 +27,7 @@ where .compat() }); use futures::compat::Future01CompatExt; - let mut rt = tokio02::runtime::Builder::new() + let mut rt = tokio::runtime::Builder::new() .basic_scheduler() .enable_io() .enable_time() @@ -132,7 +132,7 @@ fn do_connect(url: &str) -> impl Future> { let fut = fut.map_err(|e: RpcError| log::error!("RPC Client error: {:?}", e)); use futures::compat::Future01CompatExt; - tokio02::spawn(Box::pin(fut.compat())); + tokio::spawn(Box::pin(fut.compat())); ready(Ok(sender.into())) } diff --git a/core-client/transports/src/transports/ipc.rs b/core-client/transports/src/transports/ipc.rs index ba608bc7d..1b475f7f0 100644 --- a/core-client/transports/src/transports/ipc.rs +++ b/core-client/transports/src/transports/ipc.rs @@ -6,7 +6,7 @@ use crate::{RpcChannel, RpcError}; use futures::{SinkExt, StreamExt, TryStreamExt}; use jsonrpc_server_utils::codecs::StreamCodec; use jsonrpc_server_utils::tokio_util::codec::Decoder as _; -use jsonrpc_server_utils::tokio02 as tokio02; +use jsonrpc_server_utils::tokio; use parity_tokio_ipc::Endpoint; use std::path::Path; @@ -28,7 +28,7 @@ pub async fn connect, Client: From>( ), ); - tokio02::spawn(client); + tokio::spawn(client); Ok(sender.into()) } @@ -72,7 +72,7 @@ mod tests { Err(err) => panic!("IPC RPC call failed: {}", err), } }; - tokio02::runtime::Runtime::new().unwrap().block_on(client_fut); + tokio::runtime::Runtime::new().unwrap().block_on(client_fut); } #[test] @@ -84,7 +84,7 @@ mod tests { } }; - tokio02::runtime::Runtime::new().unwrap().block_on(test_fut); + tokio::runtime::Runtime::new().unwrap().block_on(test_fut); } #[test] @@ -109,6 +109,6 @@ mod tests { } }; - tokio02::runtime::Runtime::new().unwrap().block_on(client_fut); + tokio::runtime::Runtime::new().unwrap().block_on(client_fut); } } diff --git a/core-client/transports/src/transports/ws.rs b/core-client/transports/src/transports/ws.rs index e424b938e..6cd82c78e 100644 --- a/core-client/transports/src/transports/ws.rs +++ b/core-client/transports/src/transports/ws.rs @@ -50,7 +50,7 @@ where ); let (rpc_client, sender) = super::duplex(sink, stream); let rpc_client = rpc_client.map_err(|error| log::error!("{:?}", error)); - tokio02::spawn(rpc_client); + tokio::spawn(rpc_client); sender.into() }) .map_err(|error| RpcError::Other(Box::new(error))) diff --git a/http/src/lib.rs b/http/src/lib.rs index 223477a59..ebcd7ba0e 100644 --- a/http/src/lib.rs +++ b/http/src/lib.rs @@ -55,7 +55,7 @@ pub use crate::handler::ServerHandler; pub use crate::response::Response; pub use crate::server_utils::cors::{self, AccessControlAllowOrigin, AllowCors, Origin}; pub use crate::server_utils::hosts::{DomainsValidation, Host}; -pub use crate::server_utils::{tokio02, SuspendableStream}; +pub use crate::server_utils::{tokio, SuspendableStream}; pub use crate::server_utils::{reactor::TaskExecutor}; pub use crate::utils::{cors_allow_headers, cors_allow_origin, is_host_allowed}; @@ -551,7 +551,7 @@ fn serve>( listener.reuse_address(true)?; listener.bind(&addr)?; let listener = listener.listen(1024)?; - let listener = tokio02::net::TcpListener::from_std(listener)?; + let listener = tokio::net::TcpListener::from_std(listener)?; // Add current host to allowed headers. // NOTE: we need to use `l.local_addr()` instead of `addr` // it might be different! @@ -611,7 +611,7 @@ fn serve>( res.map_err(|e| error!("Error serving connection: {:?}", e)) }); - tokio02::spawn(connection); + tokio::spawn(connection); }) .for_each(|_| async { () }); let shutdown_signal = shutdown_signal.map_err(|e| { diff --git a/server-utils/Cargo.toml b/server-utils/Cargo.toml index 6398221a0..f21b5ed88 100644 --- a/server-utils/Cargo.toml +++ b/server-utils/Cargo.toml @@ -17,8 +17,8 @@ globset = "0.4" jsonrpc-core = { version = "16.0", path = "../core" } lazy_static = "1.1.0" log = "0.4" +tokio = { version = "0.2", features = ["rt-threaded", "io-driver", "io-util", "time", "tcp"] } tokio-util = { version = "0.3", features = ["codec", "compat"] } -tokio02 = { package = "tokio", version = "0.2", features = ["rt-threaded", "io-driver", "io-util", "time", "tcp"] } unicase = "2.0" diff --git a/server-utils/src/lib.rs b/server-utils/src/lib.rs index 0fde97169..5c9f52d1a 100644 --- a/server-utils/src/lib.rs +++ b/server-utils/src/lib.rs @@ -8,8 +8,8 @@ extern crate log; #[macro_use] extern crate lazy_static; +pub use tokio; pub use tokio_util; -pub use tokio02; pub mod cors; pub mod hosts; diff --git a/server-utils/src/reactor.rs b/server-utils/src/reactor.rs index d83bf9cb3..d6f61a633 100644 --- a/server-utils/src/reactor.rs +++ b/server-utils/src/reactor.rs @@ -7,9 +7,9 @@ use std::io; -use tokio02::runtime; +use tokio::runtime; /// Task executor for Tokio 0.2 runtime. -pub type TaskExecutor = tokio02::runtime::Handle; +pub type TaskExecutor = tokio::runtime::Handle; /// Possibly uninitialized event loop executor. #[derive(Debug)] diff --git a/server-utils/src/suspendable_stream.rs b/server-utils/src/suspendable_stream.rs index 7a79335a0..fd0448985 100644 --- a/server-utils/src/suspendable_stream.rs +++ b/server-utils/src/suspendable_stream.rs @@ -4,7 +4,7 @@ use std::pin::Pin; use std::task::Poll; use std::time::Duration; -use tokio02::time::Delay; +use tokio::time::Delay; /// `Incoming` is a stream of incoming sockets /// Polling the stream may return a temporary io::Error (for instance if we can't open the connection because of "too many open files" limit) @@ -83,7 +83,7 @@ where }; debug!("Error accepting connection: {}", err); debug!("The server will stop accepting connections for {:?}", self.next_delay); - self.timeout = Some(tokio02::time::delay_for(self.next_delay)); + self.timeout = Some(tokio::time::delay_for(self.next_delay)); } } } diff --git a/tcp/src/lib.rs b/tcp/src/lib.rs index 8537619aa..d4c161e2e 100644 --- a/tcp/src/lib.rs +++ b/tcp/src/lib.rs @@ -44,7 +44,7 @@ use jsonrpc_core as jsonrpc; pub(crate) use crate::jsonrpc::futures as futures03; -pub use self::server_utils::{codecs::Separator, tokio02}; +pub use self::server_utils::{codecs::Separator, tokio}; pub use crate::dispatch::{Dispatcher, PushMessageError}; pub use crate::meta::{MetaExtractor, RequestContext}; pub use crate::server::{Server, ServerBuilder}; diff --git a/tcp/src/server.rs b/tcp/src/server.rs index 9361b7203..020e2486f 100644 --- a/tcp/src/server.rs +++ b/tcp/src/server.rs @@ -7,7 +7,7 @@ use std::pin::Pin; use tower_service::Service as _; use crate::jsonrpc::{middleware, MetaIoHandler, Metadata, Middleware}; -use crate::server_utils::{codecs, reactor, tokio02, tokio_util::codec::Framed, SuspendableStream}; +use crate::server_utils::{codecs, reactor, tokio, tokio_util::codec::Framed, SuspendableStream}; use crate::futures03::{self, future}; use crate::dispatch::{Dispatcher, PeerMessageQueue, SenderChannels}; @@ -94,7 +94,7 @@ where use futures03::{FutureExt, SinkExt, StreamExt, TryStreamExt}; executor.executor().spawn(async move { let start = async { - let listener = tokio02::net::TcpListener::bind(&address).await?; + let listener = tokio::net::TcpListener::bind(&address).await?; let connections = SuspendableStream::new(listener); let server = connections.map(|socket| { diff --git a/tcp/src/tests.rs b/tcp/src/tests.rs index 93a19484f..a3d3ee2c8 100644 --- a/tcp/src/tests.rs +++ b/tcp/src/tests.rs @@ -7,7 +7,7 @@ use jsonrpc_core::{MetaIoHandler, Metadata, Value}; use tokio::io::AsyncWriteExt; use tokio::io::AsyncReadExt; -use crate::server_utils::tokio02::{self as tokio, net::TcpStream}; +use crate::server_utils::tokio::{self, net::TcpStream}; use crate::futures03; use parking_lot::Mutex; diff --git a/ws/src/lib.rs b/ws/src/lib.rs index 9f41cb651..5c1cfc3ca 100644 --- a/ws/src/lib.rs +++ b/ws/src/lib.rs @@ -27,5 +27,5 @@ pub use self::server_builder::ServerBuilder; pub use self::server_utils::cors::Origin; pub use self::server_utils::hosts::{DomainsValidation, Host}; pub use self::server_utils::session::{SessionId, SessionStats}; -pub use self::server_utils::tokio02 as tokio; +pub use self::server_utils::tokio; pub use self::session::{MiddlewareAction, RequestMiddleware}; From 68e9be679782030efd2f55656eaefed66d7531c7 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 21 Dec 2020 14:16:22 +0100 Subject: [PATCH 46/73] Fix HTTP client Tokio runtime management --- core-client/transports/src/transports/http.rs | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/core-client/transports/src/transports/http.rs b/core-client/transports/src/transports/http.rs index f2502f1d0..8121a1580 100644 --- a/core-client/transports/src/transports/http.rs +++ b/core-client/transports/src/transports/http.rs @@ -12,44 +12,45 @@ pub fn connect(url: &str) -> impl Future> where TClient: From, { + use futures::future::Either; + let (sender, receiver) = futures::channel::oneshot::channel(); let url = url.to_owned(); + let url: Uri = match url.parse() { + Ok(url) => url, + Err(e) => return Either::Left(async { Err(RpcError::Other(Box::new(e))) }), + }; + std::thread::spawn(move || { - let connect = futures01::lazy(move || { - do_connect(&url) - .map(|client| { - if sender.send(client).is_err() { - panic!("The caller did not wait for the server."); - } - Ok::<(), ()>(()) - }) - .compat() - }); - use futures::compat::Future01CompatExt; let mut rt = tokio::runtime::Builder::new() .basic_scheduler() - .enable_io() - .enable_time() + .enable_all() .build() .unwrap(); - let fut = connect.compat().map(drop); - rt.block_on(fut); - // FIXME: This keeps Tokio 0.2 runtime alive - let _ = rt.block_on(futures01::future::empty::<(), ()>().compat()); + + rt.block_on(async { + let (api, worker) = do_connect(url).await; + + if sender.send(api).is_err() { + panic!("The caller did not wait for the server."); + } + + // NOTE: We need to explicitly wait on a returned worker task; + // idleness tracking in runtime was removed from Tokio 0.1 + worker.await; + }); }); - receiver.map(|res| res.expect("Server closed prematurely.").map(TClient::from)) -} + return Either::Right(async move { + let api = receiver.await.expect("Server closed prematurely"); -fn do_connect(url: &str) -> impl Future> { - use futures::future::ready; + Ok(TClient::from(api)) + }) +} +fn do_connect(url: Uri) -> impl Future)> { let max_parallel = 8; - let url: Uri = match url.parse() { - Ok(url) => url, - Err(e) => return ready(Err(RpcError::Other(Box::new(e)))), - }; #[cfg(feature = "tls")] let connector = hyper_tls::HttpsConnector::new(); @@ -132,9 +133,8 @@ fn do_connect(url: &str) -> impl Future> { let fut = fut.map_err(|e: RpcError| log::error!("RPC Client error: {:?}", e)); use futures::compat::Future01CompatExt; - tokio::spawn(Box::pin(fut.compat())); - ready(Ok(sender.into())) + async { (sender.into(), Box::pin(fut.compat().map(drop))) } } #[cfg(test)] From 3859116e033777ffd89239fe9f853851e850727f Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Sun, 27 Dec 2020 00:42:55 +0100 Subject: [PATCH 47/73] Prefer using hyper API for HTTP server --- core-client/transports/src/transports/http.rs | 10 +- http/src/lib.rs | 98 ++++++++++--------- 2 files changed, 55 insertions(+), 53 deletions(-) diff --git a/core-client/transports/src/transports/http.rs b/core-client/transports/src/transports/http.rs index 8121a1580..1915fac7c 100644 --- a/core-client/transports/src/transports/http.rs +++ b/core-client/transports/src/transports/http.rs @@ -30,15 +30,15 @@ where .unwrap(); rt.block_on(async { - let (api, worker) = do_connect(url).await; + let (client_api, client_worker) = do_connect(url).await; - if sender.send(api).is_err() { + if sender.send(client_api).is_err() { panic!("The caller did not wait for the server."); } // NOTE: We need to explicitly wait on a returned worker task; // idleness tracking in runtime was removed from Tokio 0.1 - worker.await; + client_worker.await; }); }); @@ -49,7 +49,7 @@ where }) } -fn do_connect(url: Uri) -> impl Future)> { +async fn do_connect(url: Uri) -> (RpcChannel, impl Future) { let max_parallel = 8; #[cfg(feature = "tls")] @@ -134,7 +134,7 @@ fn do_connect(url: Uri) -> impl Future>( listener.reuse_address(true)?; listener.bind(&addr)?; let listener = listener.listen(1024)?; - let listener = tokio::net::TcpListener::from_std(listener)?; + let local_addr = listener.local_addr()?; + + // NOTE: Future-proof by explicitly setting the listener socket to + // non-blocking mode of operation (future Tokio/Hyper versions + // require for the callers to do that manually) + listener.set_nonblocking(true)?; + let server_builder = hyper::Server::from_tcp(listener) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; // Add current host to allowed headers. // NOTE: we need to use `l.local_addr()` instead of `addr` // it might be different! - let local_addr = listener.local_addr()?; - - Ok((listener, local_addr)) + Ok((server_builder, local_addr)) }; let bind_result = match bind() { - Ok((listener, local_addr)) => { + Ok((server_builder, local_addr)) => { // Send local address match local_addr_tx.send(Ok(local_addr)) { - Ok(_) => future::ok((listener, local_addr)), + Ok(_) => Ok((server_builder, local_addr)), Err(_) => { warn!( "Thread {:?} unable to reach receiver, closing server", thread::current().name() ); - future::err(()) + Err(()) } } } @@ -578,54 +584,46 @@ fn serve>( // Send error let _send_result = local_addr_tx.send(Err(err)); - future::err(()) + Err(()) } }; - let (listener, local_addr) = bind_result.compat().await?; + let (server_builder, local_addr) = bind_result?; let allowed_hosts = server_utils::hosts::update(allowed_hosts, &local_addr); - let mut http = server::conn::Http::new(); - http.http1_keep_alive(keep_alive); - let tcp_stream = SuspendableStream::new(listener); - use futures03::StreamExt; - - let server = tcp_stream - .map(move |socket| { - let service = ServerHandler::new( - jsonrpc_handler.downgrade(), - cors_domains.clone(), - cors_max_age, - allowed_headers.clone(), - allowed_hosts.clone(), - request_middleware.clone(), - rest_api, - health_api.clone(), - max_request_body_size, - keep_alive, - ); - - let connection = http.serve_connection(socket, service) - .map(|res| { - res.map_err(|e| error!("Error serving connection: {:?}", e)) - }); - - tokio::spawn(connection); - }) - .for_each(|_| async { () }); - let shutdown_signal = shutdown_signal.map_err(|e| { - debug!("Shutdown signaller dropped, closing server: {:?}", e); + let server_builder = server_builder + .http1_keepalive(keep_alive) + .tcp_nodelay(true) + // Explicitly attempt to recover from accept errors (e.g. too many + // files opened) instead of erroring out the entire server. + .tcp_sleep_on_accept_errors(true); + + let service_fn = hyper::service::make_service_fn(move |_addr_stream| { + let service = ServerHandler::new( + jsonrpc_handler.downgrade(), + cors_domains.clone(), + cors_max_age, + allowed_headers.clone(), + allowed_hosts.clone(), + request_middleware.clone(), + rest_api, + health_api.clone(), + max_request_body_size, + keep_alive, + ); + async { Ok::<_, Infallible>(service) } }); - use futures03::FutureExt; - - let select = futures03::future::select(Box::pin(server), shutdown_signal.compat().map(drop)); + let server = server_builder.serve(service_fn).with_graceful_shutdown(async { + if let Err(err) = shutdown_signal.compat().await { + debug!("Shutdown signaller dropped, closing server: {:?}", err); + } + }); - let res = select.await; - // We drop the server (possibly unresolved future) first to prevent a situation where - // main thread terminates before the server is properly dropped (see #504 for more details) - drop(res); + if let Err(err) = server.await { + error!("Error running HTTP server: {:?}", err); + } done_tx.send(()) }); @@ -658,8 +656,12 @@ impl CloseHandle { pub fn close(self) { if let Some(executors) = self.0.lock().take() { for (executor, closer) in executors { - executor.close(); + // First send shutdown signal so we can proceed with select + eprintln!("Before sending shutdown signal"); let _ = closer.send(()); + eprintln!("After sending shutdown signal"); + executor.close(); + eprintln!("After closing executor"); } } } From e3b960326dc7be158a11fc768357e120e86d06e8 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Sun, 27 Dec 2020 01:37:20 +0100 Subject: [PATCH 48/73] Work around Windows TCP socket close bug in Tokio 0.2 --- http/src/lib.rs | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/http/src/lib.rs b/http/src/lib.rs index 6b2d51592..a3be220ac 100644 --- a/http/src/lib.rs +++ b/http/src/lib.rs @@ -558,19 +558,25 @@ fn serve>( // non-blocking mode of operation (future Tokio/Hyper versions // require for the callers to do that manually) listener.set_nonblocking(true)?; + // HACK: See below. + #[cfg(windows)] + let raw_socket = std::os::windows::io::AsRawSocket::as_raw_socket(&listener); + #[cfg(not(windows))] + let raw_socket = (); + let server_builder = hyper::Server::from_tcp(listener) .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; // Add current host to allowed headers. // NOTE: we need to use `l.local_addr()` instead of `addr` // it might be different! - Ok((server_builder, local_addr)) + Ok((server_builder, local_addr, raw_socket)) }; let bind_result = match bind() { - Ok((server_builder, local_addr)) => { + Ok((server_builder, local_addr, raw_socket)) => { // Send local address match local_addr_tx.send(Ok(local_addr)) { - Ok(_) => Ok((server_builder, local_addr)), + Ok(_) => Ok((server_builder, local_addr, raw_socket)), Err(_) => { warn!( "Thread {:?} unable to reach receiver, closing server", @@ -588,7 +594,7 @@ fn serve>( } }; - let (server_builder, local_addr) = bind_result?; + let (server_builder, local_addr, _raw_socket) = bind_result?; let allowed_hosts = server_utils::hosts::update(allowed_hosts, &local_addr); @@ -625,6 +631,16 @@ fn serve>( error!("Error running HTTP server: {:?}", err); } + // FIXME: Work around TCP listener socket not being properly closed + // in mio v0.6. This runs the std::net::TcpListener's destructor, + // which closes the underlying OS socket. + // Remove this once we migrate to Tokio 1.0. + #[cfg(windows)] + let _: std::net::TcpListener = unsafe { + std::os::windows::io::FromRawSocket::from_raw_socket(_raw_socket) + }; + + done_tx.send(()) }); } From a13cc9712d81b0139bfe5e42aa539a1429274ec4 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Sun, 27 Dec 2020 21:30:45 +0100 Subject: [PATCH 49/73] Migrate http to std::future --- http/Cargo.toml | 3 +- http/src/handler.rs | 140 ++++++++++++++++++++------------------------ http/src/lib.rs | 30 +++++----- 3 files changed, 81 insertions(+), 92 deletions(-) diff --git a/http/Cargo.toml b/http/Cargo.toml index 910dd4b5e..e4356f294 100644 --- a/http/Cargo.toml +++ b/http/Cargo.toml @@ -11,8 +11,7 @@ repository = "https://github.com/paritytech/jsonrpc" version = "16.0.0" [dependencies] -futures01 = { version = "0.1", package = "futures" } -futures03 = { version = "0.3", package = "futures", features = ["compat"] } +futures = "0.3" hyper = "0.13" jsonrpc-core = { version = "16.0", path = "../core" } jsonrpc-server-utils = { version = "16.0", path = "../server-utils" } diff --git a/http/src/handler.rs b/http/src/handler.rs index 9205732f6..cbe0efb42 100644 --- a/http/src/handler.rs +++ b/http/src/handler.rs @@ -1,8 +1,10 @@ use crate::WeakRpc; +use std::future::Future; use std::sync::Arc; use std::{fmt, mem, str}; -use std::task; +use std::pin::Pin; +use std::task::{self, Poll}; use hyper::header::{self, HeaderMap, HeaderValue}; use hyper::{self, service::Service, Body, Method}; @@ -11,7 +13,6 @@ use crate::jsonrpc::serde_json; use crate::jsonrpc::{self as core, middleware, Metadata, Middleware}; use crate::response::Response; use crate::server_utils::cors; -use futures01::{Async, Future, Poll}; use crate::{utils, AllowedHosts, CorsDomains, RequestMiddleware, RequestMiddlewareAction, RestApi}; @@ -61,18 +62,17 @@ impl> ServerHandler { impl> Service> for ServerHandler where S::Future: Unpin, - S::CallFuture: Unpin, + S::CallFuture: Unpin, M: std::marker::Unpin { type Response = hyper::Response; type Error = hyper::Error; - type Future = futures03::compat::Compat01As03>; + type Future = Handler; fn poll_ready(&mut self, _cx: &mut task::Context<'_>) -> task::Poll> { task::Poll::Ready(Ok(())) } fn call(&mut self, request: hyper::Request) -> Self::Future { - use futures03::compat::Future01CompatExt; let is_host_allowed = utils::is_host_allowed(&request, &self.allowed_hosts); let action = self.middleware.on_request(request); @@ -89,12 +89,12 @@ where // Validate host if should_validate_hosts && !is_host_allowed { - return Handler::Err(Some(Response::host_not_allowed())).compat(); + return Handler::Err(Some(Response::host_not_allowed())); } // Replace response with the one returned by middleware. match response { - Ok(response) => Handler::Middleware(response).compat(), + Ok(response) => Handler::Middleware(response), Err(request) => { Handler::Rpc(RpcHandler { jsonrpc_handler: self.jsonrpc_handler.clone(), @@ -114,7 +114,7 @@ where max_request_body_size: self.max_request_body_size, // initial value, overwritten when reading client headers keep_alive: true, - }).compat() + }) } } } @@ -123,22 +123,21 @@ where pub enum Handler> { Rpc(RpcHandler), Err(Option), - Middleware(Box, Error = hyper::Error> + Send>), + Middleware(Pin>> + Send>>), } impl> Future for Handler where S::Future: Unpin, - S::CallFuture: Unpin, + S::CallFuture: Unpin, M: Unpin { - type Item = hyper::Response; - type Error = hyper::Error; + type Output = hyper::Result>; - fn poll(&mut self) -> Poll { - match *self { - Handler::Rpc(ref mut handler) => handler.poll(), - Handler::Middleware(ref mut middleware) => middleware.poll(), - Handler::Err(ref mut response) => Ok(Async::Ready( + fn poll(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> Poll { + match Pin::into_inner(self) { + Handler::Rpc(ref mut handler) => Pin::new(handler).poll(cx), + Handler::Middleware(ref mut middleware) => Pin::new(middleware).poll(cx), + Handler::Err(ref mut response) => Poll::Ready(Ok( response .take() .expect("Response always Some initialy. Returning `Ready` so will never be polled again; qed") @@ -186,8 +185,8 @@ enum RpcHandlerState { metadata: M, }, Writing(Response), - Waiting(Box, Error = ()> + Send>), - WaitingForResponse(Box + Send>), + Waiting(Pin> + Send>>), + WaitingForResponse(Pin + Send>>), Done, } @@ -224,13 +223,14 @@ pub struct RpcHandler> { impl> Future for RpcHandler where S::Future: Unpin, - S::CallFuture: Unpin, + S::CallFuture: Unpin, M: std::marker::Unpin { - type Item = hyper::Response; - type Error = hyper::Error; + type Output = hyper::Result>; + + fn poll(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> Poll { + let this = Pin::into_inner(self); - fn poll(&mut self) -> Poll { - let new_state = match mem::replace(&mut self.state, RpcHandlerState::Done) { + let new_state = match mem::replace(&mut this.state, RpcHandlerState::Done) { RpcHandlerState::ReadingHeaders { request, cors_domains, @@ -239,19 +239,19 @@ where keep_alive, } => { // Read cors header - self.cors_allow_origin = utils::cors_allow_origin(&request, &cors_domains); - self.cors_allow_headers = utils::cors_allow_headers(&request, &cors_headers); - self.keep_alive = utils::keep_alive(&request, keep_alive); - self.is_options = *request.method() == Method::OPTIONS; + this.cors_allow_origin = utils::cors_allow_origin(&request, &cors_domains); + this.cors_allow_headers = utils::cors_allow_headers(&request, &cors_headers); + this.keep_alive = utils::keep_alive(&request, keep_alive); + this.is_options = *request.method() == Method::OPTIONS; // Read other headers - RpcPollState::Ready(self.read_headers(request, continue_on_invalid_cors)) + RpcPollState::Ready(this.read_headers(request, continue_on_invalid_cors)) } RpcHandlerState::ReadingBody { body, request, metadata, uri, - } => match self.process_body(body, request, uri, metadata) { + } => match this.process_body(body, request, uri, metadata, cx) { Err(BodyError::Utf8(ref e)) => { let mesg = format!("utf-8 encoding error at byte {} in request body", e.valid_up_to()); let resp = Response::bad_request(mesg); @@ -261,19 +261,18 @@ where let resp = Response::too_large("request body size exceeds allowed maximum"); RpcPollState::Ready(RpcHandlerState::Writing(resp)) } - Err(BodyError::Hyper(e)) => return Err(e), + Err(BodyError::Hyper(e)) => return Poll::Ready(Err(e)), Ok(state) => state, }, - RpcHandlerState::ProcessRest { uri, metadata } => self.process_rest(uri, metadata)?, - RpcHandlerState::ProcessHealth { method, metadata } => self.process_health(method, metadata)?, - RpcHandlerState::WaitingForResponse(mut waiting) => match waiting.poll() { - Ok(Async::Ready(response)) => RpcPollState::Ready(RpcHandlerState::Writing(response)), - Ok(Async::NotReady) => RpcPollState::NotReady(RpcHandlerState::WaitingForResponse(waiting)), - Err(e) => RpcPollState::Ready(RpcHandlerState::Writing(Response::internal_error(format!("{:?}", e)))), + RpcHandlerState::ProcessRest { uri, metadata } => this.process_rest(uri, metadata)?, + RpcHandlerState::ProcessHealth { method, metadata } => this.process_health(method, metadata)?, + RpcHandlerState::WaitingForResponse(mut waiting) => match Pin::new(&mut waiting).poll(cx) { + Poll::Ready(response) => RpcPollState::Ready(RpcHandlerState::Writing(response)), + Poll::Pending => RpcPollState::NotReady(RpcHandlerState::WaitingForResponse(waiting)), }, RpcHandlerState::Waiting(mut waiting) => { - match waiting.poll() { - Ok(Async::Ready(response)) => { + match Pin::new(&mut waiting).poll(cx) { + Poll::Ready(response) => { RpcPollState::Ready(RpcHandlerState::Writing(match response { // Notification, just return empty response. None => Response::ok(String::new()), @@ -281,10 +280,7 @@ where Some(result) => Response::ok(format!("{}\n", result)), })) } - Ok(Async::NotReady) => RpcPollState::NotReady(RpcHandlerState::Waiting(waiting)), - Err(e) => { - RpcPollState::Ready(RpcHandlerState::Writing(Response::internal_error(format!("{:?}", e)))) - } + Poll::Pending => RpcPollState::NotReady(RpcHandlerState::Waiting(waiting)), } } state => RpcPollState::NotReady(state), @@ -294,25 +290,25 @@ where match new_state { RpcHandlerState::Writing(res) => { let mut response: hyper::Response = res.into(); - let cors_allow_origin = mem::replace(&mut self.cors_allow_origin, cors::AllowCors::Invalid); - let cors_allow_headers = mem::replace(&mut self.cors_allow_headers, cors::AllowCors::Invalid); + let cors_allow_origin = mem::replace(&mut this.cors_allow_origin, cors::AllowCors::Invalid); + let cors_allow_headers = mem::replace(&mut this.cors_allow_headers, cors::AllowCors::Invalid); Self::set_response_headers( response.headers_mut(), - self.is_options, - self.cors_max_age, + this.is_options, + this.cors_max_age, cors_allow_origin.into(), cors_allow_headers.into(), - self.keep_alive, + this.keep_alive, ); - Ok(Async::Ready(response)) + Poll::Ready(Ok(response)) } state => { - self.state = state; + this.state = state; if is_ready { - self.poll() + Pin::new(this).poll(cx) } else { - Ok(Async::NotReady) + Poll::Pending } } } @@ -399,7 +395,6 @@ where fn process_health(&self, method: String, metadata: M) -> Result, hyper::Error> { use self::core::types::{Call, Failure, Id, MethodCall, Output, Params, Request, Success, Version}; - use futures03::{FutureExt, TryFutureExt}; // Create a request let call = Request::Single(Call::MethodCall(MethodCall { @@ -413,10 +408,9 @@ where Some(h) => h.handler.handle_rpc_request(call, metadata), None => return Ok(RpcPollState::Ready(RpcHandlerState::Writing(Response::closing()))), }; - let response = response.map(Ok).compat(); - Ok(RpcPollState::Ready(RpcHandlerState::WaitingForResponse(Box::new( - response.map(|res| match res { + Ok(RpcPollState::Ready(RpcHandlerState::WaitingForResponse(Box::pin(async { + match response.await { Some(core::Response::Single(Output::Success(Success { result, .. }))) => { let result = serde_json::to_string(&result).expect("Serialization of result is infallible;qed"); @@ -428,13 +422,12 @@ where Response::service_unavailable(result) } e => Response::internal_error(format!("Invalid response for health request: {:?}", e)), - }), - )))) + } + })))) } fn process_rest(&self, uri: hyper::Uri, metadata: M) -> Result, hyper::Error> { use self::core::types::{Call, Id, MethodCall, Params, Request, Value, Version}; - use futures03::{FutureExt, TryFutureExt}; // skip the initial / let mut it = uri.path().split('/').skip(1); @@ -461,27 +454,26 @@ where Some(h) => h.handler.handle_rpc_request(call, metadata), None => return Ok(RpcPollState::Ready(RpcHandlerState::Writing(Response::closing()))), }; - let response = response.map(Ok).compat(); - Ok(RpcPollState::Ready(RpcHandlerState::Waiting(Box::new(response.map( - |res| res.map(|x| serde_json::to_string(&x).expect("Serialization of response is infallible;qed")), - ))))) + Ok(RpcPollState::Ready(RpcHandlerState::Waiting(Box::pin(async { + response.await.map(|x| serde_json::to_string(&x).expect("Serialization of response is infallible;qed")) + })))) } fn process_body( &self, - body: hyper::Body, + mut body: hyper::Body, mut request: Vec, uri: Option, metadata: M, + cx: &mut task::Context<'_>, ) -> Result, BodyError> { - use futures03::TryStreamExt; - use futures01::Stream; - let mut compat = body.compat(); + use futures::Stream; loop { - match compat.poll()? { - Async::Ready(Some(chunk)) => { + let pinned_body = Pin::new(&mut body); + match pinned_body.poll_next(cx)? { + Poll::Ready(Some(chunk)) => { if request .len() .checked_add(chunk.len()) @@ -492,8 +484,7 @@ where } request.extend_from_slice(&*chunk) } - Async::Ready(None) => { - use futures03::{FutureExt, TryFutureExt}; + Poll::Ready(None) => { if let (Some(uri), true) = (uri, request.is_empty()) { return Ok(RpcPollState::Ready(RpcHandlerState::ProcessRest { uri, metadata })); } @@ -512,12 +503,11 @@ where }; // Content is ready - let response = response.map(Ok).compat(); - return Ok(RpcPollState::Ready(RpcHandlerState::Waiting(Box::new(response)))); + return Ok(RpcPollState::Ready(RpcHandlerState::Waiting(Box::pin(response)))); } - Async::NotReady => { + Poll::Pending => { return Ok(RpcPollState::NotReady(RpcHandlerState::ReadingBody { - body: compat.into_inner(), + body, request, metadata, uri, diff --git a/http/src/lib.rs b/http/src/lib.rs index a3be220ac..1d7fd5c92 100644 --- a/http/src/lib.rs +++ b/http/src/lib.rs @@ -37,7 +37,9 @@ mod utils; use std::io; use std::convert::Infallible; +use std::future::Future; use std::net::SocketAddr; +use std::pin::Pin; use std::sync::{mpsc, Arc, Weak}; use std::thread; @@ -45,10 +47,7 @@ use parking_lot::Mutex; use crate::jsonrpc::MetaIoHandler; use crate::server_utils::reactor::{Executor, UninitializedExecutor}; -use futures01::sync::oneshot; -use futures01::{future, Future, - // Stream -}; +use futures::{future, channel::oneshot}; use hyper::Body; use jsonrpc_core as jsonrpc; @@ -75,7 +74,7 @@ pub enum RequestMiddlewareAction { /// Should standard hosts validation be performed? should_validate_hosts: bool, /// a future for server response - response: Box, Error = hyper::Error> + Send>, + response: Pin>> + Send>>, }, } @@ -83,7 +82,7 @@ impl From for RequestMiddlewareAction { fn from(o: Response) -> Self { RequestMiddlewareAction::Respond { should_validate_hosts: true, - response: Box::new(future::ok(o.into())), + response: Box::pin(async { Ok(o.into()) }), } } } @@ -92,7 +91,7 @@ impl From> for RequestMiddlewareAction { fn from(response: hyper::Response) -> Self { RequestMiddlewareAction::Respond { should_validate_hosts: true, - response: Box::new(future::ok(response)), + response: Box::pin(async { Ok(response) }), } } } @@ -254,7 +253,7 @@ pub struct ServerBuilder = impl> ServerBuilder where S::Future: Unpin, - S::CallFuture: Unpin, + S::CallFuture: Unpin, M: Unpin { /// Creates new `ServerBuilder` for given `IoHandler`. /// @@ -272,7 +271,7 @@ where impl> ServerBuilder where S::Future: Unpin, - S::CallFuture: Unpin, + S::CallFuture: Unpin, M: std::marker::Unpin { /// Creates new `ServerBuilder` for given `IoHandler`. /// @@ -538,9 +537,8 @@ fn serve>( max_request_body_size: usize, ) where S::Future: Unpin, - S::CallFuture: Unpin, + S::CallFuture: Unpin, M: std::marker::Unpin { - use futures03::compat::Future01CompatExt; let (shutdown_signal, local_addr_tx, done_tx) = signals; executor.spawn(async move { let bind = move || { @@ -622,7 +620,7 @@ fn serve>( }); let server = server_builder.serve(service_fn).with_graceful_shutdown(async { - if let Err(err) = shutdown_signal.compat().await { + if let Err(err) = shutdown_signal.await { debug!("Shutdown signaller dropped, closing server: {:?}", err); } }); @@ -714,9 +712,11 @@ impl Server { fn wait_internal(&mut self) { if let Some(receivers) = self.done.take() { - for receiver in receivers { - let _ = receiver.wait(); - } + // NOTE: Gracefully handle the case where we may wait on a *nested* + // local task pool (for now, wait on a dedicated, spawned thread) + let _ = std::thread::spawn(move || { + futures::executor::block_on(future::try_join_all(receivers)) + }).join(); } } } From a4cd677441a1b2432e3ed988df696156b13676dc Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 28 Dec 2020 11:57:44 +0100 Subject: [PATCH 50/73] Remove futures01 from http core-client/http --- core-client/transports/Cargo.toml | 2 +- core-client/transports/src/transports/http.rs | 76 ++++++++----------- 2 files changed, 34 insertions(+), 44 deletions(-) diff --git a/core-client/transports/Cargo.toml b/core-client/transports/Cargo.toml index 6f40b9951..c61f9f898 100644 --- a/core-client/transports/Cargo.toml +++ b/core-client/transports/Cargo.toml @@ -21,7 +21,7 @@ categories = [ [features] default = ["http", "tls", "ws"] tls = ["hyper-tls", "http"] -http = ["hyper", "futures01", "tokio/full"] +http = ["hyper", "tokio/full"] ws = [ "websocket", "tokio", diff --git a/core-client/transports/src/transports/http.rs b/core-client/transports/src/transports/http.rs index 1915fac7c..8e2401305 100644 --- a/core-client/transports/src/transports/http.rs +++ b/core-client/transports/src/transports/http.rs @@ -4,7 +4,7 @@ use super::RequestBuilder; use crate::{RpcChannel, RpcError, RpcMessage, RpcResult}; -use futures::{Future, FutureExt, StreamExt, TryFutureExt, TryStreamExt}; +use futures::{future, Future, FutureExt, StreamExt, TryFutureExt}; use hyper::{http, Client, Request, Uri}; /// Create a HTTP Client @@ -59,28 +59,26 @@ async fn do_connect(url: Uri) -> (RpcChannel, impl Future) { #[cfg(not(feature = "tls"))] let client = Client::new(); - + // Keep track of internal request IDs when building subsequent requests let mut request_builder = RequestBuilder::new(); let (sender, receiver) = futures::channel::mpsc::unbounded(); - use futures01::{Future, Stream}; let fut = receiver - .map(Ok) - .compat() .filter_map(move |msg: RpcMessage| { - let (request, sender) = match msg { + future::ready(match msg { RpcMessage::Call(call) => { let (_, request) = request_builder.call_request(&call); - (request, Some(call.sender)) + Some((request, Some(call.sender))) } - RpcMessage::Notify(notify) => (request_builder.notification(¬ify), None), + RpcMessage::Notify(notify) => Some((request_builder.notification(¬ify), None)), RpcMessage::Subscribe(_) => { log::warn!("Unsupported `RpcMessage` type `Subscribe`."); - return None; + None } - }; - + }) + }) + .map(move |(request, sender)| { let request = Request::post(&url) .header( http::header::CONTENT_TYPE, @@ -93,48 +91,40 @@ async fn do_connect(url: Uri) -> (RpcChannel, impl Future) { .body(request.into()) .expect("Uri and request headers are valid; qed"); - Some(client.request(request).compat().then(move |response| Ok((response, sender)))) + client.request(request).then(|response| async move { + (response, sender) + }) }) .buffer_unordered(max_parallel) - .for_each(|(result, sender)| { - use futures01::future::{ - self, - Either::{A, B}, - }; - let future = match result { + .for_each(|(response, sender)| async { + let result = match response { Ok(ref res) if !res.status().is_success() => { log::trace!("http result status {}", res.status()); - A(future::err(RpcError::Client(format!( + Err(RpcError::Client(format!( "Unexpected response status code: {}", res.status() - )))) - } - Ok(res) => B( - Box::pin(hyper::body::to_bytes(res.into_body())).compat() - .map_err(|e| RpcError::ParseError(e.to_string(), Box::new(e))) - ), - Err(err) => A(future::err(RpcError::Other(Box::new(err)))), + ))) + }, + Err(err) => Err(RpcError::Other(Box::new(err))), + Ok(res) => hyper::body::to_bytes(res.into_body()) + .map_err(|e| RpcError::ParseError(e.to_string(), Box::new(e))) + .await, }; - future.then(|result| { - if let Some(sender) = sender { - let response = result - .and_then(|response| { - let response_str = String::from_utf8_lossy(response.as_ref()).into_owned(); - super::parse_response(&response_str) - }) - .and_then(|r| r.1); - if let Err(err) = sender.send(response) { - log::warn!("Error resuming asynchronous request: {:?}", err); - } + + if let Some(sender) = sender { + let response = result + .and_then(|response| { + let response_str = String::from_utf8_lossy(response.as_ref()).into_owned(); + super::parse_response(&response_str) + }) + .and_then(|r| r.1); + if let Err(err) = sender.send(response) { + log::warn!("Error resuming asynchronous request: {:?}", err); } - Ok(()) - }) + } }); - let fut = fut.map_err(|e: RpcError| log::error!("RPC Client error: {:?}", e)); - use futures::compat::Future01CompatExt; - - (sender.into(), Box::pin(fut.compat().map(drop))) + (sender.into(), fut) } #[cfg(test)] From 62263d97534cfa596ce640b938176922518acda9 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 28 Dec 2020 12:31:57 +0100 Subject: [PATCH 51/73] Migrate core-client/ws to std::future --- core-client/transports/Cargo.toml | 1 - core-client/transports/src/transports/ws.rs | 165 ++++++++++++-------- 2 files changed, 97 insertions(+), 69 deletions(-) diff --git a/core-client/transports/Cargo.toml b/core-client/transports/Cargo.toml index c61f9f898..21a423820 100644 --- a/core-client/transports/Cargo.toml +++ b/core-client/transports/Cargo.toml @@ -25,7 +25,6 @@ http = ["hyper", "tokio/full"] ws = [ "websocket", "tokio", - "futures01", ] ipc = [ "parity-tokio-ipc", diff --git a/core-client/transports/src/transports/ws.rs b/core-client/transports/src/transports/ws.rs index 6cd82c78e..29112e157 100644 --- a/core-client/transports/src/transports/ws.rs +++ b/core-client/transports/src/transports/ws.rs @@ -1,16 +1,18 @@ //! JSON-RPC websocket client implementation. -use crate::{RpcChannel, RpcError}; -use futures01::prelude::*; -use log::info; use std::collections::VecDeque; +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use crate::{RpcChannel, RpcError}; use websocket::{ClientBuilder, OwnedMessage}; /// Connect to a JSON-RPC websocket server. /// -/// Uses an unbuffered channel to queue outgoing rpc messages. +/// Uses an unbounded channel to queue outgoing rpc messages. /// /// Returns `Err` if the `url` is invalid. -pub fn try_connect(url: &str) -> Result, RpcError> +pub fn try_connect(url: &str) -> Result>, RpcError> where T: From, { @@ -20,40 +22,45 @@ where /// Connect to a JSON-RPC websocket server. /// -/// Uses an unbuffered channel to queue outgoing rpc messages. -pub fn connect(url: &url::Url) -> impl futures::Future> +/// Uses an unbounded channel to queue outgoing rpc messages. +pub fn connect(url: &url::Url) -> impl Future> where T: From, { let client_builder = ClientBuilder::from_url(url); - let fut = do_connect(client_builder); - futures::compat::Compat01As03::new(fut) + do_connect(client_builder) } -fn do_connect(client_builder: ClientBuilder) -> impl Future +fn do_connect(client_builder: ClientBuilder) -> impl Future> where T: From, { - client_builder - .async_connect(None) - .map(|(client, _)| { - use futures::{StreamExt, TryFutureExt}; - let (sink, stream) = client.split(); - let (sink, stream) = WebsocketClient::new(sink, stream).split(); - let (sink, stream) = ( - Box::pin(futures::compat::Compat01As03Sink::new(sink)), - Box::pin( - futures::compat::Compat01As03::new(stream) - .take_while(|x| futures::future::ready(x.is_ok())) - .map(|x| x.expect("Stream is closed upon first error.")), - ), - ); - let (rpc_client, sender) = super::duplex(sink, stream); - let rpc_client = rpc_client.map_err(|error| log::error!("{:?}", error)); - tokio::spawn(rpc_client); - sender.into() - }) - .map_err(|error| RpcError::Other(Box::new(error))) + use futures::compat::{Future01CompatExt, Sink01CompatExt, Stream01CompatExt}; + use futures::{SinkExt, StreamExt, TryFutureExt, TryStreamExt}; + use websocket::futures::Stream; + + client_builder.async_connect(None).compat() + .map_err(|error| RpcError::Other(Box::new(error))) + .map_ok(|(client, _)| { + let (sink, stream) = client.split(); + + let sink = sink.sink_compat().sink_map_err(|e| RpcError::Other(Box::new(e))); + let stream = stream.compat().map_err(|e| RpcError::Other(Box::new(e))); + let (sink, stream) = WebsocketClient::new(sink, stream).split(); + let (sink, stream) = ( + Box::pin(sink), + Box::pin( + stream + .take_while(|x| futures::future::ready(x.is_ok())) + .map(|x| x.expect("Stream is closed upon first error.")), + ), + ); + let (rpc_client, sender) = super::duplex(sink, stream); + let rpc_client = rpc_client.map_err(|error| log::error!("{:?}", error)); + tokio::spawn(rpc_client); + + sender.into() + }) } struct WebsocketClient { @@ -64,8 +71,8 @@ struct WebsocketClient { impl WebsocketClient where - TSink: Sink, - TStream: Stream, + TSink: futures::Sink, + TStream: futures::Stream>, TError: std::error::Error + Send + 'static, { pub fn new(sink: TSink, stream: TStream) -> Self { @@ -77,65 +84,87 @@ where } } -impl Sink for WebsocketClient +impl futures::Sink for WebsocketClient where - TSink: Sink, - TStream: Stream, - TError: std::error::Error + Send + 'static, + TSink: futures::Sink + Unpin, + TStream: futures::Stream> + Unpin, { - type SinkItem = String; - type SinkError = RpcError; + type Error = RpcError; - fn start_send(&mut self, request: Self::SinkItem) -> Result, Self::SinkError> { + fn start_send(mut self: Pin<&mut Self>, request: String) -> Result<(), Self::Error> { self.queue.push_back(OwnedMessage::Text(request)); - Ok(AsyncSink::Ready) + Ok(()) } - fn poll_complete(&mut self) -> Result, Self::SinkError> { + fn poll_ready(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = Pin::into_inner(self); + loop { - match self.queue.pop_front() { - Some(request) => match self.sink.start_send(request) { - Ok(AsyncSink::Ready) => continue, - Ok(AsyncSink::NotReady(request)) => { - self.queue.push_front(request); - break; + match this.queue.pop_front() { + Some(request) => match Pin::new(&mut this.sink).poll_ready(cx) { + Poll::Ready(Ok(())) => { + if let Err(err) = Pin::new(&mut this.sink).start_send(request) { + return Poll::Ready(Err(RpcError::Other(Box::new(err)))); + } + continue; } - Err(error) => return Err(RpcError::Other(Box::new(error))), - }, + poll => { + this.queue.push_front(request); + return poll; + } + } None => break, } } - self.sink - .poll_complete() + + Pin::new(&mut this.sink) + .poll_ready(cx) .map_err(|error| RpcError::Other(Box::new(error))) } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if !self.queue.is_empty() { + self.poll_ready(cx) + } else { + let this = Pin::into_inner(self); + Pin::new(&mut this.sink).poll_flush(cx) + } + } + + fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if !self.queue.is_empty() { + self.poll_ready(cx) + } else { + let this = Pin::into_inner(self); + Pin::new(&mut this.sink).poll_close(cx) + } + } } -impl Stream for WebsocketClient +impl futures::Stream for WebsocketClient where - TSink: Sink, - TStream: Stream, - TError: std::error::Error + Send + 'static, + TSink: futures::Sink + Unpin, + TStream: futures::Stream> + Unpin, { - type Item = String; - type Error = RpcError; + type Item = Result; - fn poll(&mut self) -> Result>, Self::Error> { + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = Pin::into_inner(self); loop { - match self.stream.poll() { - Ok(Async::Ready(Some(message))) => match message { - OwnedMessage::Text(data) => return Ok(Async::Ready(Some(data))), - OwnedMessage::Binary(data) => info!("server sent binary data {:?}", data), - OwnedMessage::Ping(p) => self.queue.push_front(OwnedMessage::Pong(p)), + match Pin::new(&mut this.stream).poll_next(cx) { + Poll::Ready(Some(Ok(message))) => match message { + OwnedMessage::Text(data) => return Poll::Ready(Some(Ok(data))), + OwnedMessage::Binary(data) => log::info!("server sent binary data {:?}", data), + OwnedMessage::Ping(p) => this.queue.push_front(OwnedMessage::Pong(p)), OwnedMessage::Pong(_) => {} - OwnedMessage::Close(c) => self.queue.push_front(OwnedMessage::Close(c)), + OwnedMessage::Close(c) => this.queue.push_front(OwnedMessage::Close(c)), }, - Ok(Async::Ready(None)) => { + Poll::Ready(None) => { // TODO try to reconnect (#411). - return Ok(Async::Ready(None)); + return Poll::Ready(None); } - Ok(Async::NotReady) => return Ok(Async::NotReady), - Err(error) => return Err(RpcError::Other(Box::new(error))), + Poll::Pending => return Poll::Pending, + Poll::Ready(Some(Err(error))) => return Poll::Ready(Some(Err(RpcError::Other(Box::new(error))))), } } } From 93b2887a0e7d78d40b7c28d409daf5296fb57b08 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 28 Dec 2020 14:57:47 +0100 Subject: [PATCH 52/73] Remove futures01 from core-client --- core-client/Cargo.toml | 10 ++++------ core-client/src/lib.rs | 3 --- core-client/transports/Cargo.toml | 1 - 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/core-client/Cargo.toml b/core-client/Cargo.toml index 7bf00a1aa..a0e7953fe 100644 --- a/core-client/Cargo.toml +++ b/core-client/Cargo.toml @@ -19,16 +19,14 @@ categories = [ ] [features] -tls = ["jsonrpc-client-transports/tls", "futures01"] -http = ["jsonrpc-client-transports/http", "futures01"] -ws = ["jsonrpc-client-transports/ws", "futures01"] -ipc = ["jsonrpc-client-transports/ipc", "futures01"] +tls = ["jsonrpc-client-transports/tls"] +http = ["jsonrpc-client-transports/http"] +ws = ["jsonrpc-client-transports/ws"] +ipc = ["jsonrpc-client-transports/ipc"] arbitrary_precision = ["jsonrpc-client-transports/arbitrary_precision"] [dependencies] jsonrpc-client-transports = { version = "16.0", path = "./transports", default-features = false } -# Only for client transports, should be removed when we fully transition to futures=0.3 -futures01 = { version = "0.1", package = "futures", optional = true } futures = { version = "0.3", features = [ "compat" ] } [badges] diff --git a/core-client/src/lib.rs b/core-client/src/lib.rs index c7294c871..81b8b462e 100644 --- a/core-client/src/lib.rs +++ b/core-client/src/lib.rs @@ -9,6 +9,3 @@ pub use futures; pub use jsonrpc_client_transports::*; - -#[cfg(feature = "futures01")] -pub use futures01; diff --git a/core-client/transports/Cargo.toml b/core-client/transports/Cargo.toml index 21a423820..2dffa6183 100644 --- a/core-client/transports/Cargo.toml +++ b/core-client/transports/Cargo.toml @@ -43,7 +43,6 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" url = "1.7" -futures01 = { version = "0.1.26", package = "futures", optional = true } hyper = { version = "0.13", optional = true } hyper-tls = { version = "0.4", optional = true } jsonrpc-server-utils = { version = "16.0", path = "../../server-utils", optional = true } From 931f7ffa8d8f085f290c674bc9b7cee45e41762f Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 28 Dec 2020 15:12:12 +0100 Subject: [PATCH 53/73] Prefer futures to futures03 in server-utils --- server-utils/Cargo.toml | 2 +- server-utils/src/reactor.rs | 4 ++-- server-utils/src/suspendable_stream.rs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server-utils/Cargo.toml b/server-utils/Cargo.toml index f21b5ed88..205f8d8dd 100644 --- a/server-utils/Cargo.toml +++ b/server-utils/Cargo.toml @@ -12,7 +12,7 @@ version = "16.0.0" [dependencies] bytes = { version = "0.5", package = "bytes" } -futures03 = { version = "0.3", package = "futures" } +futures = "0.3" globset = "0.4" jsonrpc-core = { version = "16.0", path = "../core" } lazy_static = "1.1.0" diff --git a/server-utils/src/reactor.rs b/server-utils/src/reactor.rs index d6f61a633..a38e69b75 100644 --- a/server-utils/src/reactor.rs +++ b/server-utils/src/reactor.rs @@ -76,7 +76,7 @@ impl Executor { #[derive(Debug)] pub struct RpcEventLoop { executor: TaskExecutor, - close: Option>, + close: Option>, runtime: Option, } @@ -94,7 +94,7 @@ impl RpcEventLoop { /// Spawns a new named thread with the `EventLoop`. pub fn with_name(name: Option) -> io::Result { - let (stop, stopped) = futures03::channel::oneshot::channel(); + let (stop, stopped) = futures::channel::oneshot::channel(); let mut tb = runtime::Builder::new(); tb.core_threads(1); diff --git a/server-utils/src/suspendable_stream.rs b/server-utils/src/suspendable_stream.rs index fd0448985..ffda0356e 100644 --- a/server-utils/src/suspendable_stream.rs +++ b/server-utils/src/suspendable_stream.rs @@ -36,9 +36,9 @@ impl SuspendableStream { } } -impl futures03::Stream for SuspendableStream +impl futures::Stream for SuspendableStream where - S: futures03::Stream> + Unpin, + S: futures::Stream> + Unpin, { type Item = I; From 2bbac0cf9e6703843650256aaebf8616f0f97bed Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 28 Dec 2020 19:14:36 +0100 Subject: [PATCH 54/73] Migrate ipc server to std::future --- ipc/Cargo.toml | 3 +- ipc/src/select_with_weak.rs | 48 +++++----- ipc/src/server.rs | 181 +++++++++++++++++------------------- 3 files changed, 114 insertions(+), 118 deletions(-) diff --git a/ipc/Cargo.toml b/ipc/Cargo.toml index 13bd4c381..6598a639b 100644 --- a/ipc/Cargo.toml +++ b/ipc/Cargo.toml @@ -10,8 +10,7 @@ repository = "https://github.com/paritytech/jsonrpc" version = "16.0.0" [dependencies] -futures01 = { version = "0.1", package = "futures" } -futures03 = { version = "0.3", package = "futures", features = [ "compat" ] } +futures03 = { version = "0.3", package = "futures" } log = "0.4" tower-service = "0.3" jsonrpc-core = { version = "16.0", path = "../core" } diff --git a/ipc/src/select_with_weak.rs b/ipc/src/select_with_weak.rs index 409aa46cd..32c5fef50 100644 --- a/ipc/src/select_with_weak.rs +++ b/ipc/src/select_with_weak.rs @@ -1,10 +1,13 @@ -use futures01::stream::{Fuse, Stream}; -use futures01::{Async, Poll}; +use std::task::Poll; +use std::task::Context; +use std::pin::Pin; + +use futures03::stream::{Fuse, Stream}; pub trait SelectWithWeakExt: Stream { fn select_with_weak(self, other: S) -> SelectWithWeak where - S: Stream, + S: Stream, Self: Sized; } @@ -14,7 +17,7 @@ where { fn select_with_weak(self, other: S) -> SelectWithWeak where - S: Stream, + S: Stream, Self: Sized, { new(self, other) @@ -39,8 +42,9 @@ pub struct SelectWithWeak { fn new(stream1: S1, stream2: S2) -> SelectWithWeak where S1: Stream, - S2: Stream, + S2: Stream, { + use futures03::StreamExt; SelectWithWeak { strong: stream1.fuse(), weak: stream2.fuse(), @@ -50,36 +54,36 @@ where impl Stream for SelectWithWeak where - S1: Stream, - S2: Stream, + S1: Stream + std::marker::Unpin, + S2: Stream + std::marker::Unpin, { type Item = S1::Item; - type Error = S1::Error; - fn poll(&mut self) -> Poll, S1::Error> { + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let mut this = Pin::into_inner(self); let mut checked_strong = false; loop { - if self.use_strong { - match self.strong.poll()? { - Async::Ready(Some(item)) => { - self.use_strong = false; - return Ok(Some(item).into()); + if this.use_strong { + match Pin::new(&mut this.strong).poll_next(cx) { + Poll::Ready(Some(item)) => { + this.use_strong = false; + return Poll::Ready(Some(item).into()); } - Async::Ready(None) => return Ok(None.into()), - Async::NotReady => { + Poll::Ready(None) => return Poll::Ready(None), + Poll::Pending => { if !checked_strong { - self.use_strong = false; + this.use_strong = false; } else { - return Ok(Async::NotReady); + return Poll::Pending; } } } checked_strong = true; } else { - self.use_strong = true; - match self.weak.poll()? { - Async::Ready(Some(item)) => return Ok(Some(item).into()), - Async::Ready(None) | Async::NotReady => (), + this.use_strong = true; + match Pin::new(&mut this.strong).poll_next(cx) { + Poll::Ready(Some(item)) => return Poll::Ready(Some(item).into()), + Poll::Ready(None) | Poll::Pending => (), } } } diff --git a/ipc/src/server.rs b/ipc/src/server.rs index 435aea4e4..6dcba4250 100644 --- a/ipc/src/server.rs +++ b/ipc/src/server.rs @@ -7,7 +7,8 @@ use crate::jsonrpc::futures::channel::mpsc; use crate::jsonrpc::{middleware, MetaIoHandler, Metadata, Middleware}; use crate::meta::{MetaExtractor, NoopExtractor, RequestContext}; use crate::select_with_weak::SelectWithWeakExt; -use futures01::{future, sync::oneshot, Future as _, Stream}; +use futures03::channel::oneshot; +use futures03::StreamExt; use parity_tokio_ipc::Endpoint; use parking_lot::Mutex; use tower_service::Service as _; @@ -153,12 +154,13 @@ where let incoming_separator = self.incoming_separator; let outgoing_separator = self.outgoing_separator; let (stop_signal, stop_receiver) = oneshot::channel(); - let (start_signal, start_receiver) = oneshot::channel(); - let (wait_signal, wait_receiver) = oneshot::channel(); + // NOTE: These channels are only waited upon in synchronous fashion + let (start_signal, start_receiver) = std::sync::mpsc::channel(); + let (wait_signal, wait_receiver) = std::sync::mpsc::channel(); let security_attributes = self.security_attributes; let client_buffer_size = self.client_buffer_size; - let fut = future::lazy(move || { + let fut = async move { let mut endpoint = Endpoint::new(endpoint_addr); endpoint.set_security_attributes(security_attributes); @@ -176,14 +178,14 @@ where start_signal .send(Err(e)) .expect("Cannot fail since receiver never dropped before receiving"); - return future::Either::A(future::ok(())); + return; } }; let mut id = 0u64; use futures03::TryStreamExt; - let server = connections.compat().map(move |io_stream| { + let server = connections.map_ok(move |io_stream| { id = id.wrapping_add(1); let session_id = id; let session_stats = session_stats.clone(); @@ -204,50 +206,45 @@ where let (writer, reader) = futures03::StreamExt::split(framed); let responses = reader - .compat() - .map(move |req| { - use futures03::TryFutureExt; + .map_ok(move |req| { service.call(req) - .map_err(|()| std::io::ErrorKind::Other.into()) - .compat() + // Ignore service errors + .map(|x| Ok(x.ok().flatten())) }) - .buffer_unordered(client_buffer_size) - .filter_map(|x| x) + .try_buffer_unordered(client_buffer_size) + // Filter out previously ignored service errors as `None`s + .try_filter_map(|x| futures03::future::ok(x)) // we use `select_with_weak` here, instead of `select`, to close the stream // as soon as the ipc pipe is closed - .select_with_weak(futures03::TryStreamExt::compat(futures03::StreamExt::map(receiver, std::io::Result::Ok))); + .select_with_weak(receiver.map(Ok)); - responses.forward(futures03::SinkExt::compat(writer)) + responses.forward(writer) .then(move |_| { trace!(target: "ipc", "Peer: service finished"); if let Some(stats) = session_stats.as_ref() { stats.close_session(session_id) } - Ok(()) + + async { Ok(()) } }) }); start_signal .send(Ok(())) .expect("Cannot fail since receiver never dropped before receiving"); + let stop = stop_receiver.map_err(|_| std::io::ErrorKind::Interrupted); + let stop = Box::pin(stop); + + let server = server.try_buffer_unordered(1024).for_each(|_| async { }); + + let result = futures03::future::select(Box::pin(server), stop).await; + // We drop the server first to prevent a situation where main thread terminates + // before the server is properly dropped (see #504 for more details) + drop(result); + let _ = wait_signal.send(()); + }; - let stop = stop_receiver.map_err(|_| std::io::ErrorKind::Interrupted.into()); - future::Either::B( - server - .buffer_unordered(1024) - .for_each(|_| Ok(())) - .select(stop) - .map(|(_, server)| { - // We drop the server first to prevent a situation where main thread terminates - // before the server is properly dropped (see #504 for more details) - drop(server); - let _ = wait_signal.send(()); - }) - .map_err(|_| ()), - ) - }); - use futures03::compat::Future01CompatExt; use futures03::FutureExt; - let fut = Box::pin(fut.compat().map(drop)); + let fut = Box::pin(fut.map(drop)); executor.executor().spawn(fut); let handle = InnerHandles { @@ -256,7 +253,8 @@ where path: path.to_owned(), }; - match start_receiver.wait().expect("Message should always be sent") { + use futures03::TryFutureExt; + match start_receiver.recv().expect("Message should always be sent") { Ok(()) => Ok(Server { handles: Arc::new(Mutex::new(handle)), wait_handle: Some(wait_receiver), @@ -270,7 +268,7 @@ where #[derive(Debug)] pub struct Server { handles: Arc>, - wait_handle: Option>, + wait_handle: Option>, } impl Server { @@ -288,7 +286,9 @@ impl Server { /// Wait for the server to finish pub fn wait(mut self) { - self.wait_handle.take().map(|wait_receiver| wait_receiver.wait()); + if let Some(wait_receiver) = self.wait_handle.take() { + let _ = wait_receiver.recv(); + } } } @@ -332,7 +332,6 @@ impl CloseHandle { mod tests { use super::*; - use futures01::{Future, Stream}; use jsonrpc_core::Value; use std::thread; use std::time::{self, Duration}; @@ -351,7 +350,7 @@ mod tests { } fn dummy_request_str(path: &str, data: &str) -> String { - use futures03::{StreamExt, SinkExt}; + use futures03::SinkExt; let reply = async move { use tokio02::net::UnixStream; @@ -396,7 +395,7 @@ mod tests { crate::logger::init_log(); let path = "/tmp/test-ipc-40000"; let server = run(path); - let (stop_signal, stop_receiver) = oneshot::channel(); + let (stop_signal, stop_receiver) = std::sync::mpsc::channel(); let t = thread::spawn(move || { let result = dummy_request_str( @@ -407,15 +406,13 @@ mod tests { }); t.join().unwrap(); - let _ = stop_receiver - .map(move |result: String| { - assert_eq!( - result, "{\"jsonrpc\":\"2.0\",\"result\":\"hello\",\"id\":1}", - "Response does not exactly match the expected response", - ); - server.close(); - }) - .wait(); + let result = stop_receiver.recv().unwrap(); + + assert_eq!( + result, "{\"jsonrpc\":\"2.0\",\"result\":\"hello\",\"id\":1}", + "Response does not exactly match the expected response", + ); + server.close(); } #[test] @@ -423,7 +420,7 @@ mod tests { crate::logger::init_log(); let path = "/tmp/test-ipc-45000"; let server = run(path); - let (stop_signal, stop_receiver) = futures01::sync::mpsc::channel(400); + let (stop_signal, stop_receiver) = futures03::channel::mpsc::channel(400); let mut handles = Vec::new(); for _ in 0..4 { @@ -444,16 +441,18 @@ mod tests { handle.join().unwrap(); } - let _ = stop_receiver - .map(|result| { - assert_eq!( - result, "{\"jsonrpc\":\"2.0\",\"result\":\"hello\",\"id\":1}", - "Response does not exactly match the expected response", - ); - }) - .take(400) - .collect() - .wait(); + thread::spawn(move || { + let fut = stop_receiver + .map(|result| { + assert_eq!( + result, "{\"jsonrpc\":\"2.0\",\"result\":\"hello\",\"id\":1}", + "Response does not exactly match the expected response", + ); + }) + .take(400) + .for_each(|_| async {}); + futures03::executor::block_on(fut); + }).join().unwrap(); server.close(); } @@ -514,16 +513,17 @@ mod tests { }); t.join().unwrap(); - let _ = stop_receiver - .map(move |result: String| { + thread::spawn(move || { + futures03::executor::block_on(async move { + let result = stop_receiver.await.unwrap(); assert_eq!( result, huge_response_test_json(), "Response does not exactly match the expected response", ); server.close(); - }) - .wait(); + }); + }).join().unwrap(); } #[test] @@ -540,7 +540,7 @@ mod tests { } struct SessionEndExtractor { - drop_receivers: Arc>>>, + drop_receivers: Arc>>>, } impl MetaExtractor> for SessionEndExtractor { @@ -556,7 +556,7 @@ mod tests { crate::logger::init_log(); let path = "/tmp/test-ipc-30009"; - let (signal, receiver) = futures01::sync::mpsc::channel(16); + let (signal, receiver) = futures03::channel::mpsc::channel(16); let session_metadata_extractor = SessionEndExtractor { drop_receivers: Arc::new(Mutex::new(signal)), }; @@ -568,12 +568,12 @@ mod tests { let _ = UnixStream::connect(path).expect("Socket should connect"); } - receiver - .into_future() - .map_err(|_| ()) - .and_then(|drop_receiver| drop_receiver.0.unwrap().map_err(|_| ())) - .wait() - .unwrap(); + thread::spawn(move || { + futures03::executor::block_on(async move { + let (drop_receiver, ..) = receiver.into_future().await; + drop_receiver.unwrap().await.unwrap(); + }); + }).join().unwrap(); server.close(); } @@ -607,30 +607,23 @@ mod tests { tx.send(true).expect("failed to report that the server has stopped"); }); - use futures03::FutureExt; - use futures03::TryFutureExt; - let delay = futures01::future::lazy(|| - // Lazily bound to Tokio timer instance - tokio02::time::delay_for(Duration::from_millis(500)) - .map(|_| Ok(false)) - .compat() - ); - - let result_fut = rx.map_err(|_| ()).select(delay).then(move |result| match result { - Ok((result, _)) => { - assert_eq!(result, true, "Wait timeout exceeded"); - assert!( - UnixStream::connect(path).is_err(), - "Connection to the closed socket should fail" - ); - Ok(()) - } - Err(_) => Err(()), - }); - - use futures03::compat::Future01CompatExt; let mut rt = tokio02::runtime::Runtime::new().unwrap(); - rt.block_on(result_fut.compat()).unwrap(); + rt.block_on(async move { + let timeout = tokio02::time::delay_for(Duration::from_millis(500)); + + match futures03::future::select(rx, timeout).await { + futures03::future::Either::Left((result, _)) => { + assert!(result.is_ok(), "Rx failed"); + assert_eq!(result, Ok(true), "Wait timeout exceeded"); + assert!( + UnixStream::connect(path).is_err(), + "Connection to the closed socket should fail" + ); + Ok(()) + }, + futures03::future::Either::Right(_) => Err("timed out"), + } + }).unwrap(); } #[test] From 0b8326c83161b60171a526502838c90a3fa2146d Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 28 Dec 2020 19:17:08 +0100 Subject: [PATCH 55/73] Prefer futures to futures03 in ipc server --- ipc/Cargo.toml | 2 +- ipc/src/select_with_weak.rs | 4 ++-- ipc/src/server.rs | 38 ++++++++++++++++++------------------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/ipc/Cargo.toml b/ipc/Cargo.toml index 6598a639b..b8d152e6a 100644 --- a/ipc/Cargo.toml +++ b/ipc/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/paritytech/jsonrpc" version = "16.0.0" [dependencies] -futures03 = { version = "0.3", package = "futures" } +futures = "0.3" log = "0.4" tower-service = "0.3" jsonrpc-core = { version = "16.0", path = "../core" } diff --git a/ipc/src/select_with_weak.rs b/ipc/src/select_with_weak.rs index 32c5fef50..9422a90b1 100644 --- a/ipc/src/select_with_weak.rs +++ b/ipc/src/select_with_weak.rs @@ -2,7 +2,7 @@ use std::task::Poll; use std::task::Context; use std::pin::Pin; -use futures03::stream::{Fuse, Stream}; +use futures::stream::{Fuse, Stream}; pub trait SelectWithWeakExt: Stream { fn select_with_weak(self, other: S) -> SelectWithWeak @@ -44,7 +44,7 @@ where S1: Stream, S2: Stream, { - use futures03::StreamExt; + use futures::StreamExt; SelectWithWeak { strong: stream1.fuse(), weak: stream2.fuse(), diff --git a/ipc/src/server.rs b/ipc/src/server.rs index 6dcba4250..a6881ff10 100644 --- a/ipc/src/server.rs +++ b/ipc/src/server.rs @@ -7,8 +7,8 @@ use crate::jsonrpc::futures::channel::mpsc; use crate::jsonrpc::{middleware, MetaIoHandler, Metadata, Middleware}; use crate::meta::{MetaExtractor, NoopExtractor, RequestContext}; use crate::select_with_weak::SelectWithWeakExt; -use futures03::channel::oneshot; -use futures03::StreamExt; +use futures::channel::oneshot; +use futures::StreamExt; use parity_tokio_ipc::Endpoint; use parking_lot::Mutex; use tower_service::Service as _; @@ -49,7 +49,7 @@ where } fn call(&mut self, req: String) -> Self::Future { - use futures03::FutureExt; + use futures::FutureExt; trace!(target: "ipc", "Received request: {}", req); Box::pin(self.handler.handle_request(&req, self.meta.clone()).map(Ok)) } @@ -184,7 +184,7 @@ where let mut id = 0u64; - use futures03::TryStreamExt; + use futures::TryStreamExt; let server = connections.map_ok(move |io_stream| { id = id.wrapping_add(1); let session_id = id; @@ -203,7 +203,7 @@ where let mut service = Service::new(rpc_handler.clone(), meta); let codec = codecs::StreamCodec::new(incoming_separator.clone(), outgoing_separator.clone()); let framed = tokio_util::codec::Decoder::framed(codec, io_stream); - let (writer, reader) = futures03::StreamExt::split(framed); + let (writer, reader) = futures::StreamExt::split(framed); let responses = reader .map_ok(move |req| { @@ -213,7 +213,7 @@ where }) .try_buffer_unordered(client_buffer_size) // Filter out previously ignored service errors as `None`s - .try_filter_map(|x| futures03::future::ok(x)) + .try_filter_map(|x| futures::future::ok(x)) // we use `select_with_weak` here, instead of `select`, to close the stream // as soon as the ipc pipe is closed .select_with_weak(receiver.map(Ok)); @@ -236,14 +236,14 @@ where let server = server.try_buffer_unordered(1024).for_each(|_| async { }); - let result = futures03::future::select(Box::pin(server), stop).await; + let result = futures::future::select(Box::pin(server), stop).await; // We drop the server first to prevent a situation where main thread terminates // before the server is properly dropped (see #504 for more details) drop(result); let _ = wait_signal.send(()); }; - use futures03::FutureExt; + use futures::FutureExt; let fut = Box::pin(fut.map(drop)); executor.executor().spawn(fut); @@ -253,7 +253,7 @@ where path: path.to_owned(), }; - use futures03::TryFutureExt; + use futures::TryFutureExt; match start_receiver.recv().expect("Message should always be sent") { Ok(()) => Ok(Server { handles: Arc::new(Mutex::new(handle)), @@ -350,7 +350,7 @@ mod tests { } fn dummy_request_str(path: &str, data: &str) -> String { - use futures03::SinkExt; + use futures::SinkExt; let reply = async move { use tokio02::net::UnixStream; @@ -420,7 +420,7 @@ mod tests { crate::logger::init_log(); let path = "/tmp/test-ipc-45000"; let server = run(path); - let (stop_signal, stop_receiver) = futures03::channel::mpsc::channel(400); + let (stop_signal, stop_receiver) = futures::channel::mpsc::channel(400); let mut handles = Vec::new(); for _ in 0..4 { @@ -451,7 +451,7 @@ mod tests { }) .take(400) .for_each(|_| async {}); - futures03::executor::block_on(fut); + futures::executor::block_on(fut); }).join().unwrap(); server.close(); } @@ -514,7 +514,7 @@ mod tests { t.join().unwrap(); thread::spawn(move || { - futures03::executor::block_on(async move { + futures::executor::block_on(async move { let result = stop_receiver.await.unwrap(); assert_eq!( result, @@ -540,7 +540,7 @@ mod tests { } struct SessionEndExtractor { - drop_receivers: Arc>>>, + drop_receivers: Arc>>>, } impl MetaExtractor> for SessionEndExtractor { @@ -556,7 +556,7 @@ mod tests { crate::logger::init_log(); let path = "/tmp/test-ipc-30009"; - let (signal, receiver) = futures03::channel::mpsc::channel(16); + let (signal, receiver) = futures::channel::mpsc::channel(16); let session_metadata_extractor = SessionEndExtractor { drop_receivers: Arc::new(Mutex::new(signal)), }; @@ -569,7 +569,7 @@ mod tests { } thread::spawn(move || { - futures03::executor::block_on(async move { + futures::executor::block_on(async move { let (drop_receiver, ..) = receiver.into_future().await; drop_receiver.unwrap().await.unwrap(); }); @@ -611,8 +611,8 @@ mod tests { rt.block_on(async move { let timeout = tokio02::time::delay_for(Duration::from_millis(500)); - match futures03::future::select(rx, timeout).await { - futures03::future::Either::Left((result, _)) => { + match futures::future::select(rx, timeout).await { + futures::future::Either::Left((result, _)) => { assert!(result.is_ok(), "Rx failed"); assert_eq!(result, Ok(true), "Wait timeout exceeded"); assert!( @@ -621,7 +621,7 @@ mod tests { ); Ok(()) }, - futures03::future::Either::Right(_) => Err("timed out"), + futures::future::Either::Right(_) => Err("timed out"), } }).unwrap(); } From 52d9fa0bd45539f4dc5fe48744fb72b0f599417d Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 28 Dec 2020 19:20:18 +0100 Subject: [PATCH 56/73] Prefer futures to futures03 in tcp server --- tcp/src/dispatch.rs | 4 +--- tcp/src/lib.rs | 2 +- tcp/src/server.rs | 12 ++++++------ tcp/src/service.rs | 4 ++-- tcp/src/tests.rs | 4 ++-- 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/tcp/src/dispatch.rs b/tcp/src/dispatch.rs index 69fe9632d..7b3a3f75d 100644 --- a/tcp/src/dispatch.rs +++ b/tcp/src/dispatch.rs @@ -5,9 +5,7 @@ use std::sync::Arc; use std::pin::Pin; use std::task::Poll; -use crate::futures03::{self, channel::mpsc}; - -use futures03::Stream; +use crate::futures::{channel::mpsc, Stream}; use parking_lot::Mutex; diff --git a/tcp/src/lib.rs b/tcp/src/lib.rs index d4c161e2e..b78e4be54 100644 --- a/tcp/src/lib.rs +++ b/tcp/src/lib.rs @@ -42,7 +42,7 @@ mod tests; use jsonrpc_core as jsonrpc; -pub(crate) use crate::jsonrpc::futures as futures03; +pub(crate) use crate::jsonrpc::futures; pub use self::server_utils::{codecs::Separator, tokio}; pub use crate::dispatch::{Dispatcher, PushMessageError}; diff --git a/tcp/src/server.rs b/tcp/src/server.rs index 020e2486f..51a9d554c 100644 --- a/tcp/src/server.rs +++ b/tcp/src/server.rs @@ -8,7 +8,7 @@ use tower_service::Service as _; use crate::jsonrpc::{middleware, MetaIoHandler, Metadata, Middleware}; use crate::server_utils::{codecs, reactor, tokio, tokio_util::codec::Framed, SuspendableStream}; -use crate::futures03::{self, future}; +use crate::futures::{self, future}; use crate::dispatch::{Dispatcher, PeerMessageQueue, SenderChannels}; use crate::meta::{MetaExtractor, NoopExtractor, RequestContext}; @@ -87,11 +87,11 @@ where let outgoing_separator = self.outgoing_separator; let address = addr.to_owned(); let (tx, rx) = std::sync::mpsc::channel(); - let (stop_tx, stop_rx) = futures03::channel::oneshot::channel(); + let (stop_tx, stop_rx) = futures::channel::oneshot::channel(); let executor = self.executor.initialize()?; - use futures03::{FutureExt, SinkExt, StreamExt, TryStreamExt}; + use futures::{FutureExt, SinkExt, StreamExt, TryStreamExt}; executor.executor().spawn(async move { let start = async { let listener = tokio::net::TcpListener::bind(&address).await?; @@ -108,7 +108,7 @@ where } }; trace!(target: "tcp", "Accepted incoming connection from {}", &peer_addr); - let (sender, receiver) = futures03::channel::mpsc::unbounded(); + let (sender, receiver) = futures::channel::mpsc::unbounded(); let context = RequestContext { peer_addr, @@ -124,7 +124,7 @@ where .split(); // Work around https://github.com/rust-lang/rust/issues/64552 by boxing the stream type - let responses: Pin> + Send>> = + let responses: Pin> + Send>> = Box::pin(reader.and_then(move |req| { service.call(req).then(|response| match response { Err(e) => { @@ -197,7 +197,7 @@ where /// TCP Server handle pub struct Server { executor: Option, - stop: Option>, + stop: Option>, } impl Server { diff --git a/tcp/src/service.rs b/tcp/src/service.rs index 472839903..01c20d55d 100644 --- a/tcp/src/service.rs +++ b/tcp/src/service.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use std::task::{Context, Poll}; use crate::jsonrpc::{middleware, MetaIoHandler, Metadata, Middleware}; -use crate::futures03; +use crate::futures; pub struct Service = middleware::Noop> { @@ -43,7 +43,7 @@ where // Produce a future for computing a response from a request. fn call(&mut self, req: String) -> Self::Future { - use futures03::FutureExt; + use futures::FutureExt; trace!(target: "tcp", "Accepted request from peer {}: {}", &self.peer_addr, req); Box::pin(self.handler.handle_request(&req, self.meta.clone()).map(Ok)) } diff --git a/tcp/src/tests.rs b/tcp/src/tests.rs index a3d3ee2c8..eaf211ce9 100644 --- a/tcp/src/tests.rs +++ b/tcp/src/tests.rs @@ -8,7 +8,7 @@ use tokio::io::AsyncWriteExt; use tokio::io::AsyncReadExt; use crate::server_utils::tokio::{self, net::TcpStream}; -use crate::futures03; +use crate::futures; use parking_lot::Mutex; @@ -246,7 +246,7 @@ fn message() { let client = async move { let stream = TcpStream::connect(&addr); let delay = tokio::time::delay_for(Duration::from_millis(500)); - let (stream, _) = futures03::join!(stream, delay); + let (stream, _) = futures::join!(stream, delay); let mut stream = stream?; let peer_addr = peer_list.lock()[0].clone(); From ffee8904fce1539e38bb5c921c7223a2c14ac818 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 28 Dec 2020 19:40:59 +0100 Subject: [PATCH 57/73] Migrate ws server to std::future --- ws/Cargo.toml | 3 +-- ws/src/metadata.rs | 38 +++++++++++++++++--------------------- ws/src/session.rs | 32 +++++++++++++++----------------- 3 files changed, 33 insertions(+), 40 deletions(-) diff --git a/ws/Cargo.toml b/ws/Cargo.toml index 2247d1098..514c71ffb 100644 --- a/ws/Cargo.toml +++ b/ws/Cargo.toml @@ -10,8 +10,7 @@ repository = "https://github.com/paritytech/jsonrpc" version = "16.0.0" [dependencies] -futures01 = { version = "0.1", package = "futures" } -futures03 = { version = "0.3", package = "futures", features = [ "compat" ] } +futures03 = { version = "0.3", package = "futures" } jsonrpc-core = { version = "16.0", path = "../core" } jsonrpc-server-utils = { version = "16.0", path = "../server-utils" } log = "0.4" diff --git a/ws/src/metadata.rs b/ws/src/metadata.rs index 4b642f45a..bcfb3a55c 100644 --- a/ws/src/metadata.rs +++ b/ws/src/metadata.rs @@ -1,5 +1,8 @@ +use std::future::Future; use std::fmt; +use std::pin::Pin; use std::sync::{atomic, Arc}; +use std::task::{Poll, Context}; use crate::core; use crate::core::futures::channel::mpsc; @@ -79,12 +82,9 @@ impl RequestContext { /// Get this session as a `Sink` spawning a new future /// in the underlying event loop. pub fn sender(&self) -> mpsc::UnboundedSender { - use futures03::compat::Future01CompatExt; - use futures03::{StreamExt, TryStreamExt}; let out = self.out.clone(); let (sender, receiver) = mpsc::unbounded(); - let receiver = receiver.map(Ok).compat(); - self.executor.spawn(SenderFuture(out, Box::new(receiver)).compat()); + self.executor.spawn(SenderFuture(out, Box::new(receiver))); sender } } @@ -124,27 +124,23 @@ impl MetaExtractor for NoopExtractor { } } -struct SenderFuture(Sender, Box + Send>); -impl futures01::Future for SenderFuture { - type Item = (); - type Error = (); +struct SenderFuture(Sender, Box + Send + Unpin>); - fn poll(&mut self) -> futures01::Poll { - use futures01::Stream; +impl Future for SenderFuture { + type Output = (); + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + use futures03::Stream; + + let this = Pin::into_inner(self); loop { - let item = self.1.poll()?; - match item { - futures01::Async::NotReady => { - return Ok(futures01::Async::NotReady); - } - futures01::Async::Ready(None) => { - return Ok(futures01::Async::Ready(())); - } - futures01::Async::Ready(Some(val)) => { - if let Err(e) = self.0.send(val) { + match Pin::new(&mut this.1).poll_next(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(None) => return Poll::Ready(()), + Poll::Ready(Some(val)) => { + if let Err(e) = this.0.send(val) { warn!("Error sending a subscription update: {:?}", e); - return Ok(futures01::Async::Ready(())); + return Poll::Ready(()); } } } diff --git a/ws/src/session.rs b/ws/src/session.rs index 3855fddfd..85b1c59c5 100644 --- a/ws/src/session.rs +++ b/ws/src/session.rs @@ -1,9 +1,12 @@ -use std; +use std::future::Future; +use std::pin::Pin; use std::sync::{atomic, Arc}; +use std::task::{Poll, Context}; use crate::core; -use futures01::sync::oneshot; -use futures01::{Async, Future, Poll}; +use futures03::future; +use futures03::channel::oneshot; +use futures03::FutureExt; use parking_lot::Mutex; use slab::Slab; @@ -123,16 +126,16 @@ impl LivenessPoll { } impl Future for LivenessPoll { - type Item = (); - type Error = (); + type Output = (); - fn poll(&mut self) -> Poll { + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = Pin::into_inner(self); // if the future resolves ok then we've been signalled to return. // it should never be cancelled, but if it was the session definitely // isn't live. - match self.rx.poll() { - Ok(Async::Ready(_)) | Err(_) => Ok(Async::Ready(())), - Ok(Async::NotReady) => Ok(Async::NotReady), + match Pin::new(&mut this.rx).poll(cx) { + Poll::Ready(_) => Poll::Ready(()), + Poll::Pending => Poll::Pending, } } } @@ -270,8 +273,6 @@ where let active_lock = self.active.clone(); let response = self.handler.handle_request(req, metadata); - use futures03::{FutureExt, TryFutureExt}; - let response = response.map(Ok).compat(); let future = response .map(move |response| { if !active_lock.load(atomic::Ordering::SeqCst) { @@ -289,13 +290,10 @@ where _ => {} } } - }) - .select(poll_liveness) - .map(|_| ()) - .map_err(|_| ()); + }); - use futures03::compat::Future01CompatExt; - self.executor.spawn(future.compat()); + let future = future::select(future, poll_liveness); + self.executor.spawn(future); Ok(()) } From 274e653777b821d9ff657127164fb17e1b12bee6 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 28 Dec 2020 19:42:12 +0100 Subject: [PATCH 58/73] Prefer futures to futures03 in ws server --- ws/Cargo.toml | 2 +- ws/src/metadata.rs | 4 ++-- ws/src/session.rs | 6 +++--- ws/src/tests.rs | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ws/Cargo.toml b/ws/Cargo.toml index 514c71ffb..37324fbe3 100644 --- a/ws/Cargo.toml +++ b/ws/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/paritytech/jsonrpc" version = "16.0.0" [dependencies] -futures03 = { version = "0.3", package = "futures" } +futures = "0.3" jsonrpc-core = { version = "16.0", path = "../core" } jsonrpc-server-utils = { version = "16.0", path = "../server-utils" } log = "0.4" diff --git a/ws/src/metadata.rs b/ws/src/metadata.rs index bcfb3a55c..5d2621787 100644 --- a/ws/src/metadata.rs +++ b/ws/src/metadata.rs @@ -124,13 +124,13 @@ impl MetaExtractor for NoopExtractor { } } -struct SenderFuture(Sender, Box + Send + Unpin>); +struct SenderFuture(Sender, Box + Send + Unpin>); impl Future for SenderFuture { type Output = (); fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - use futures03::Stream; + use futures::Stream; let this = Pin::into_inner(self); loop { diff --git a/ws/src/session.rs b/ws/src/session.rs index 85b1c59c5..d89401078 100644 --- a/ws/src/session.rs +++ b/ws/src/session.rs @@ -4,9 +4,9 @@ use std::sync::{atomic, Arc}; use std::task::{Poll, Context}; use crate::core; -use futures03::future; -use futures03::channel::oneshot; -use futures03::FutureExt; +use futures::future; +use futures::channel::oneshot; +use futures::FutureExt; use parking_lot::Mutex; use slab::Slab; diff --git a/ws/src/tests.rs b/ws/src/tests.rs index b416c9384..0d2928460 100644 --- a/ws/src/tests.rs +++ b/ws/src/tests.rs @@ -60,7 +60,7 @@ fn request(server: Server, request: &str) -> Response { } fn serve(port: u16) -> (Server, Arc) { - use futures03::{channel::oneshot, future, FutureExt}; + use futures::{channel::oneshot, future, FutureExt}; let pending = Arc::new(AtomicUsize::new(0)); let counter = pending.clone(); From 57205067f7c53fb4f37c320b33333e6e4b40e631 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Tue, 29 Dec 2020 11:55:03 +0100 Subject: [PATCH 59/73] Prefer Unpin over std::marker::Unpin --- http/src/handler.rs | 4 ++-- http/src/lib.rs | 4 ++-- ipc/src/select_with_weak.rs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/http/src/handler.rs b/http/src/handler.rs index cbe0efb42..9fa6e12f2 100644 --- a/http/src/handler.rs +++ b/http/src/handler.rs @@ -62,7 +62,7 @@ impl> ServerHandler { impl> Service> for ServerHandler where S::Future: Unpin, - S::CallFuture: Unpin, M: std::marker::Unpin + S::CallFuture: Unpin, M: Unpin { type Response = hyper::Response; type Error = hyper::Error; @@ -223,7 +223,7 @@ pub struct RpcHandler> { impl> Future for RpcHandler where S::Future: Unpin, - S::CallFuture: Unpin, M: std::marker::Unpin + S::CallFuture: Unpin, M: Unpin { type Output = hyper::Result>; diff --git a/http/src/lib.rs b/http/src/lib.rs index 1d7fd5c92..44fe85c88 100644 --- a/http/src/lib.rs +++ b/http/src/lib.rs @@ -271,7 +271,7 @@ where impl> ServerBuilder where S::Future: Unpin, - S::CallFuture: Unpin, M: std::marker::Unpin + S::CallFuture: Unpin, M: Unpin { /// Creates new `ServerBuilder` for given `IoHandler`. /// @@ -537,7 +537,7 @@ fn serve>( max_request_body_size: usize, ) where S::Future: Unpin, - S::CallFuture: Unpin, M: std::marker::Unpin + S::CallFuture: Unpin, M: Unpin { let (shutdown_signal, local_addr_tx, done_tx) = signals; executor.spawn(async move { diff --git a/ipc/src/select_with_weak.rs b/ipc/src/select_with_weak.rs index 9422a90b1..61f0ce5e4 100644 --- a/ipc/src/select_with_weak.rs +++ b/ipc/src/select_with_weak.rs @@ -54,8 +54,8 @@ where impl Stream for SelectWithWeak where - S1: Stream + std::marker::Unpin, - S2: Stream + std::marker::Unpin, + S1: Stream + Unpin, + S2: Stream + Unpin, { type Item = S1::Item; From 714b02371adbefb632b11f3cfe19caa48985a445 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Tue, 29 Dec 2020 12:17:22 +0100 Subject: [PATCH 60/73] Remove stray eprintlns --- http/src/lib.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/http/src/lib.rs b/http/src/lib.rs index 44fe85c88..911bb6a3f 100644 --- a/http/src/lib.rs +++ b/http/src/lib.rs @@ -670,12 +670,9 @@ impl CloseHandle { pub fn close(self) { if let Some(executors) = self.0.lock().take() { for (executor, closer) in executors { - // First send shutdown signal so we can proceed with select - eprintln!("Before sending shutdown signal"); + // First send shutdown signal so we can proceed with underlying select let _ = closer.send(()); - eprintln!("After sending shutdown signal"); executor.close(); - eprintln!("After closing executor"); } } } From d33c2f94f35d594467c2b6a7e22a79c1d94b5dc6 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Tue, 29 Dec 2020 12:34:59 +0100 Subject: [PATCH 61/73] Remove unnecessary Into casts --- ipc/src/select_with_weak.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ipc/src/select_with_weak.rs b/ipc/src/select_with_weak.rs index 61f0ce5e4..0e8da6b7a 100644 --- a/ipc/src/select_with_weak.rs +++ b/ipc/src/select_with_weak.rs @@ -67,7 +67,7 @@ where match Pin::new(&mut this.strong).poll_next(cx) { Poll::Ready(Some(item)) => { this.use_strong = false; - return Poll::Ready(Some(item).into()); + return Poll::Ready(Some(item)); } Poll::Ready(None) => return Poll::Ready(None), Poll::Pending => { @@ -82,7 +82,7 @@ where } else { this.use_strong = true; match Pin::new(&mut this.strong).poll_next(cx) { - Poll::Ready(Some(item)) => return Poll::Ready(Some(item).into()), + Poll::Ready(Some(item)) => return Poll::Ready(Some(item)), Poll::Ready(None) | Poll::Pending => (), } } From e012842fa48c862c9fe6fb4177c337a61babdaf4 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Tue, 29 Dec 2020 12:36:12 +0100 Subject: [PATCH 62/73] Format the world --- core-client/transports/src/transports/http.rs | 18 ++++--- core-client/transports/src/transports/ipc.rs | 50 ++++++++--------- core-client/transports/src/transports/ws.rs | 48 +++++++++-------- http/src/handler.rs | 53 ++++++++++--------- http/src/lib.rs | 28 +++++----- ipc/src/lib.rs | 2 +- ipc/src/select_with_weak.rs | 4 +- ipc/src/server.rs | 45 ++++++++-------- server-utils/src/reactor.rs | 13 +++-- server-utils/src/suspendable_stream.rs | 4 +- stdio/src/lib.rs | 2 +- tcp/src/dispatch.rs | 2 +- tcp/src/server.rs | 44 +++++++-------- tcp/src/service.rs | 3 +- tcp/src/tests.rs | 8 ++- ws/src/metadata.rs | 6 +-- ws/src/session.rs | 37 +++++++------ 17 files changed, 186 insertions(+), 181 deletions(-) diff --git a/core-client/transports/src/transports/http.rs b/core-client/transports/src/transports/http.rs index 8e2401305..ccbf3578d 100644 --- a/core-client/transports/src/transports/http.rs +++ b/core-client/transports/src/transports/http.rs @@ -46,7 +46,7 @@ where let api = receiver.await.expect("Server closed prematurely"); Ok(TClient::from(api)) - }) + }); } async fn do_connect(url: Uri) -> (RpcChannel, impl Future) { @@ -91,9 +91,9 @@ async fn do_connect(url: Uri) -> (RpcChannel, impl Future) { .body(request.into()) .expect("Uri and request headers are valid; qed"); - client.request(request).then(|response| async move { - (response, sender) - }) + client + .request(request) + .then(|response| async move { (response, sender) }) }) .buffer_unordered(max_parallel) .for_each(|(response, sender)| async { @@ -104,11 +104,13 @@ async fn do_connect(url: Uri) -> (RpcChannel, impl Future) { "Unexpected response status code: {}", res.status() ))) - }, + } Err(err) => Err(RpcError::Other(Box::new(err))), - Ok(res) => hyper::body::to_bytes(res.into_body()) - .map_err(|e| RpcError::ParseError(e.to_string(), Box::new(e))) - .await, + Ok(res) => { + hyper::body::to_bytes(res.into_body()) + .map_err(|e| RpcError::ParseError(e.to_string(), Box::new(e))) + .await + } }; if let Some(sender) = sender { diff --git a/core-client/transports/src/transports/ipc.rs b/core-client/transports/src/transports/ipc.rs index 1b475f7f0..d50022597 100644 --- a/core-client/transports/src/transports/ipc.rs +++ b/core-client/transports/src/transports/ipc.rs @@ -5,32 +5,32 @@ use crate::transports::duplex::duplex; use crate::{RpcChannel, RpcError}; use futures::{SinkExt, StreamExt, TryStreamExt}; use jsonrpc_server_utils::codecs::StreamCodec; -use jsonrpc_server_utils::tokio_util::codec::Decoder as _; use jsonrpc_server_utils::tokio; +use jsonrpc_server_utils::tokio_util::codec::Decoder as _; use parity_tokio_ipc::Endpoint; use std::path::Path; /// Connect to a JSON-RPC IPC server. -pub async fn connect, Client: From>( - path: P, -) -> Result { - let connection = Endpoint::connect(path).await.map_err(|e| RpcError::Other(Box::new(e)))?; - let (sink, stream) = StreamCodec::stream_incoming().framed(connection).split(); - let sink = sink.sink_map_err(|e| RpcError::Other(Box::new(e))); - let stream = stream.map_err(|e| log::error!("IPC stream error: {}", e)); - - let (client, sender) = duplex( - Box::pin(sink), - Box::pin( - stream - .take_while(|x| futures::future::ready(x.is_ok())) - .map(|x| x.expect("Stream is closed upon first error.")), - ), - ); - - tokio::spawn(client); - - Ok(sender.into()) +pub async fn connect, Client: From>(path: P) -> Result { + let connection = Endpoint::connect(path) + .await + .map_err(|e| RpcError::Other(Box::new(e)))?; + let (sink, stream) = StreamCodec::stream_incoming().framed(connection).split(); + let sink = sink.sink_map_err(|e| RpcError::Other(Box::new(e))); + let stream = stream.map_err(|e| log::error!("IPC stream error: {}", e)); + + let (client, sender) = duplex( + Box::pin(sink), + Box::pin( + stream + .take_while(|x| futures::future::ready(x.is_ok())) + .map(|x| x.expect("Stream is closed upon first error.")), + ), + ); + + tokio::spawn(client); + + Ok(sender.into()) } #[cfg(test)] @@ -78,10 +78,10 @@ mod tests { #[test] fn should_fail_without_server() { let test_fut = async move { - match connect::<_, RawClient>(dummy_endpoint()).await { - Err(..) => {} - Ok(..) => panic!("Should not be able to connect to an IPC socket that's not open"), - } + match connect::<_, RawClient>(dummy_endpoint()).await { + Err(..) => {} + Ok(..) => panic!("Should not be able to connect to an IPC socket that's not open"), + } }; tokio::runtime::Runtime::new().unwrap().block_on(test_fut); diff --git a/core-client/transports/src/transports/ws.rs b/core-client/transports/src/transports/ws.rs index 29112e157..8f0ea77e3 100644 --- a/core-client/transports/src/transports/ws.rs +++ b/core-client/transports/src/transports/ws.rs @@ -39,28 +39,30 @@ where use futures::{SinkExt, StreamExt, TryFutureExt, TryStreamExt}; use websocket::futures::Stream; - client_builder.async_connect(None).compat() - .map_err(|error| RpcError::Other(Box::new(error))) - .map_ok(|(client, _)| { - let (sink, stream) = client.split(); - - let sink = sink.sink_compat().sink_map_err(|e| RpcError::Other(Box::new(e))); - let stream = stream.compat().map_err(|e| RpcError::Other(Box::new(e))); - let (sink, stream) = WebsocketClient::new(sink, stream).split(); - let (sink, stream) = ( - Box::pin(sink), - Box::pin( - stream - .take_while(|x| futures::future::ready(x.is_ok())) - .map(|x| x.expect("Stream is closed upon first error.")), - ), - ); - let (rpc_client, sender) = super::duplex(sink, stream); - let rpc_client = rpc_client.map_err(|error| log::error!("{:?}", error)); - tokio::spawn(rpc_client); - - sender.into() - }) + client_builder + .async_connect(None) + .compat() + .map_err(|error| RpcError::Other(Box::new(error))) + .map_ok(|(client, _)| { + let (sink, stream) = client.split(); + + let sink = sink.sink_compat().sink_map_err(|e| RpcError::Other(Box::new(e))); + let stream = stream.compat().map_err(|e| RpcError::Other(Box::new(e))); + let (sink, stream) = WebsocketClient::new(sink, stream).split(); + let (sink, stream) = ( + Box::pin(sink), + Box::pin( + stream + .take_while(|x| futures::future::ready(x.is_ok())) + .map(|x| x.expect("Stream is closed upon first error.")), + ), + ); + let (rpc_client, sender) = super::duplex(sink, stream); + let rpc_client = rpc_client.map_err(|error| log::error!("{:?}", error)); + tokio::spawn(rpc_client); + + sender.into() + }) } struct WebsocketClient { @@ -112,7 +114,7 @@ where this.queue.push_front(request); return poll; } - } + }, None => break, } } diff --git a/http/src/handler.rs b/http/src/handler.rs index 9fa6e12f2..0466ee844 100644 --- a/http/src/handler.rs +++ b/http/src/handler.rs @@ -1,10 +1,10 @@ use crate::WeakRpc; use std::future::Future; -use std::sync::Arc; -use std::{fmt, mem, str}; use std::pin::Pin; +use std::sync::Arc; use std::task::{self, Poll}; +use std::{fmt, mem, str}; use hyper::header::{self, HeaderMap, HeaderValue}; use hyper::{self, service::Service, Body, Method}; @@ -62,7 +62,8 @@ impl> ServerHandler { impl> Service> for ServerHandler where S::Future: Unpin, - S::CallFuture: Unpin, M: Unpin + S::CallFuture: Unpin, + M: Unpin, { type Response = hyper::Response; type Error = hyper::Error; @@ -129,7 +130,8 @@ pub enum Handler> { impl> Future for Handler where S::Future: Unpin, - S::CallFuture: Unpin, M: Unpin + S::CallFuture: Unpin, + M: Unpin, { type Output = hyper::Result>; @@ -137,12 +139,10 @@ where match Pin::into_inner(self) { Handler::Rpc(ref mut handler) => Pin::new(handler).poll(cx), Handler::Middleware(ref mut middleware) => Pin::new(middleware).poll(cx), - Handler::Err(ref mut response) => Poll::Ready(Ok( - response - .take() - .expect("Response always Some initialy. Returning `Ready` so will never be polled again; qed") - .into(), - )), + Handler::Err(ref mut response) => Poll::Ready(Ok(response + .take() + .expect("Response always Some initialy. Returning `Ready` so will never be polled again; qed") + .into())), } } } @@ -223,7 +223,8 @@ pub struct RpcHandler> { impl> Future for RpcHandler where S::Future: Unpin, - S::CallFuture: Unpin, M: Unpin + S::CallFuture: Unpin, + M: Unpin, { type Output = hyper::Result>; @@ -409,21 +410,23 @@ where None => return Ok(RpcPollState::Ready(RpcHandlerState::Writing(Response::closing()))), }; - Ok(RpcPollState::Ready(RpcHandlerState::WaitingForResponse(Box::pin(async { - match response.await { - Some(core::Response::Single(Output::Success(Success { result, .. }))) => { - let result = serde_json::to_string(&result).expect("Serialization of result is infallible;qed"); + Ok(RpcPollState::Ready(RpcHandlerState::WaitingForResponse(Box::pin( + async { + match response.await { + Some(core::Response::Single(Output::Success(Success { result, .. }))) => { + let result = serde_json::to_string(&result).expect("Serialization of result is infallible;qed"); - Response::ok(result) - } - Some(core::Response::Single(Output::Failure(Failure { error, .. }))) => { - let result = serde_json::to_string(&error).expect("Serialization of error is infallible;qed"); + Response::ok(result) + } + Some(core::Response::Single(Output::Failure(Failure { error, .. }))) => { + let result = serde_json::to_string(&error).expect("Serialization of error is infallible;qed"); - Response::service_unavailable(result) + Response::service_unavailable(result) + } + e => Response::internal_error(format!("Invalid response for health request: {:?}", e)), } - e => Response::internal_error(format!("Invalid response for health request: {:?}", e)), - } - })))) + }, + )))) } fn process_rest(&self, uri: hyper::Uri, metadata: M) -> Result, hyper::Error> { @@ -456,7 +459,9 @@ where }; Ok(RpcPollState::Ready(RpcHandlerState::Waiting(Box::pin(async { - response.await.map(|x| serde_json::to_string(&x).expect("Serialization of response is infallible;qed")) + response + .await + .map(|x| serde_json::to_string(&x).expect("Serialization of response is infallible;qed")) })))) } diff --git a/http/src/lib.rs b/http/src/lib.rs index 911bb6a3f..a229ae509 100644 --- a/http/src/lib.rs +++ b/http/src/lib.rs @@ -35,9 +35,9 @@ mod response; mod tests; mod utils; -use std::io; use std::convert::Infallible; use std::future::Future; +use std::io; use std::net::SocketAddr; use std::pin::Pin; use std::sync::{mpsc, Arc, Weak}; @@ -47,7 +47,7 @@ use parking_lot::Mutex; use crate::jsonrpc::MetaIoHandler; use crate::server_utils::reactor::{Executor, UninitializedExecutor}; -use futures::{future, channel::oneshot}; +use futures::{channel::oneshot, future}; use hyper::Body; use jsonrpc_core as jsonrpc; @@ -55,8 +55,8 @@ pub use crate::handler::ServerHandler; pub use crate::response::Response; pub use crate::server_utils::cors::{self, AccessControlAllowOrigin, AllowCors, Origin}; pub use crate::server_utils::hosts::{DomainsValidation, Host}; +pub use crate::server_utils::reactor::TaskExecutor; pub use crate::server_utils::{tokio, SuspendableStream}; -pub use crate::server_utils::{reactor::TaskExecutor}; pub use crate::utils::{cors_allow_headers, cors_allow_origin, is_host_allowed}; /// Action undertaken by a middleware. @@ -253,7 +253,8 @@ pub struct ServerBuilder = impl> ServerBuilder where S::Future: Unpin, - S::CallFuture: Unpin, M: Unpin + S::CallFuture: Unpin, + M: Unpin, { /// Creates new `ServerBuilder` for given `IoHandler`. /// @@ -271,7 +272,8 @@ where impl> ServerBuilder where S::Future: Unpin, - S::CallFuture: Unpin, M: Unpin + S::CallFuture: Unpin, + M: Unpin, { /// Creates new `ServerBuilder` for given `IoHandler`. /// @@ -537,7 +539,8 @@ fn serve>( max_request_body_size: usize, ) where S::Future: Unpin, - S::CallFuture: Unpin, M: Unpin + S::CallFuture: Unpin, + M: Unpin, { let (shutdown_signal, local_addr_tx, done_tx) = signals; executor.spawn(async move { @@ -562,8 +565,8 @@ fn serve>( #[cfg(not(windows))] let raw_socket = (); - let server_builder = hyper::Server::from_tcp(listener) - .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + let server_builder = + hyper::Server::from_tcp(listener).map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; // Add current host to allowed headers. // NOTE: we need to use `l.local_addr()` instead of `addr` // it might be different! @@ -634,10 +637,7 @@ fn serve>( // which closes the underlying OS socket. // Remove this once we migrate to Tokio 1.0. #[cfg(windows)] - let _: std::net::TcpListener = unsafe { - std::os::windows::io::FromRawSocket::from_raw_socket(_raw_socket) - }; - + let _: std::net::TcpListener = unsafe { std::os::windows::io::FromRawSocket::from_raw_socket(_raw_socket) }; done_tx.send(()) }); @@ -711,9 +711,7 @@ impl Server { if let Some(receivers) = self.done.take() { // NOTE: Gracefully handle the case where we may wait on a *nested* // local task pool (for now, wait on a dedicated, spawned thread) - let _ = std::thread::spawn(move || { - futures::executor::block_on(future::try_join_all(receivers)) - }).join(); + let _ = std::thread::spawn(move || futures::executor::block_on(future::try_join_all(receivers))).join(); } } } diff --git a/ipc/src/lib.rs b/ipc/src/lib.rs index 5cfe374ad..0629de809 100644 --- a/ipc/src/lib.rs +++ b/ipc/src/lib.rs @@ -25,5 +25,5 @@ use jsonrpc_core as jsonrpc; pub use crate::meta::{MetaExtractor, NoopExtractor, RequestContext}; pub use crate::server::{CloseHandle, SecurityAttributes, Server, ServerBuilder}; +pub use self::server_utils::codecs::Separator; pub use self::server_utils::session::{SessionId, SessionStats}; -pub use self::server_utils::{codecs::Separator}; diff --git a/ipc/src/select_with_weak.rs b/ipc/src/select_with_weak.rs index 0e8da6b7a..5a68aa258 100644 --- a/ipc/src/select_with_weak.rs +++ b/ipc/src/select_with_weak.rs @@ -1,6 +1,6 @@ -use std::task::Poll; -use std::task::Context; use std::pin::Pin; +use std::task::Context; +use std::task::Poll; use futures::stream::{Fuse, Stream}; diff --git a/ipc/src/server.rs b/ipc/src/server.rs index a6881ff10..8d02421fe 100644 --- a/ipc/src/server.rs +++ b/ipc/src/server.rs @@ -13,11 +13,7 @@ use parity_tokio_ipc::Endpoint; use parking_lot::Mutex; use tower_service::Service as _; -use crate::server_utils::{ - codecs, reactor, session, - reactor::TaskExecutor, - tokio_util, -}; +use crate::server_utils::{codecs, reactor, reactor::TaskExecutor, session, tokio_util}; pub use parity_tokio_ipc::SecurityAttributes; @@ -207,7 +203,8 @@ where let responses = reader .map_ok(move |req| { - service.call(req) + service + .call(req) // Ignore service errors .map(|x| Ok(x.ok().flatten())) }) @@ -218,15 +215,14 @@ where // as soon as the ipc pipe is closed .select_with_weak(receiver.map(Ok)); - responses.forward(writer) - .then(move |_| { - trace!(target: "ipc", "Peer: service finished"); - if let Some(stats) = session_stats.as_ref() { - stats.close_session(session_id) - } + responses.forward(writer).then(move |_| { + trace!(target: "ipc", "Peer: service finished"); + if let Some(stats) = session_stats.as_ref() { + stats.close_session(session_id) + } - async { Ok(()) } - }) + async { Ok(()) } + }) }); start_signal .send(Ok(())) @@ -234,7 +230,7 @@ where let stop = stop_receiver.map_err(|_| std::io::ErrorKind::Interrupted); let stop = Box::pin(stop); - let server = server.try_buffer_unordered(1024).for_each(|_| async { }); + let server = server.try_buffer_unordered(1024).for_each(|_| async {}); let result = futures::future::select(Box::pin(server), stop).await; // We drop the server first to prevent a situation where main thread terminates @@ -333,9 +329,9 @@ mod tests { use super::*; use jsonrpc_core::Value; + use std::os::unix::net::UnixStream; use std::thread; use std::time::{self, Duration}; - use std::os::unix::net::UnixStream; fn server_builder() -> ServerBuilder { let mut io = MetaIoHandler::<()>::default(); @@ -452,7 +448,9 @@ mod tests { .take(400) .for_each(|_| async {}); futures::executor::block_on(fut); - }).join().unwrap(); + }) + .join() + .unwrap(); server.close(); } @@ -523,7 +521,9 @@ mod tests { ); server.close(); }); - }).join().unwrap(); + }) + .join() + .unwrap(); } #[test] @@ -573,7 +573,9 @@ mod tests { let (drop_receiver, ..) = receiver.into_future().await; drop_receiver.unwrap().await.unwrap(); }); - }).join().unwrap(); + }) + .join() + .unwrap(); server.close(); } @@ -620,10 +622,11 @@ mod tests { "Connection to the closed socket should fail" ); Ok(()) - }, + } futures::future::Either::Right(_) => Err("timed out"), } - }).unwrap(); + }) + .unwrap(); } #[test] diff --git a/server-utils/src/reactor.rs b/server-utils/src/reactor.rs index a38e69b75..df8afd408 100644 --- a/server-utils/src/reactor.rs +++ b/server-utils/src/reactor.rs @@ -108,7 +108,9 @@ impl RpcEventLoop { let runtime = tb.build()?; let executor = runtime.handle().to_owned(); - runtime.spawn(async { let _ = stopped.await; }); + runtime.spawn(async { + let _ = stopped.await; + }); Ok(RpcEventLoop { executor, @@ -119,10 +121,11 @@ impl RpcEventLoop { /// Get executor for this event loop. pub fn executor(&self) -> runtime::Handle { - self.runtime.as_ref() - .expect("Runtime is only None if we're being dropped; qed") - .handle() - .clone() + self.runtime + .as_ref() + .expect("Runtime is only None if we're being dropped; qed") + .handle() + .clone() } /// Blocks current thread and waits until the event loop is finished. diff --git a/server-utils/src/suspendable_stream.rs b/server-utils/src/suspendable_stream.rs index ffda0356e..7d6d57b52 100644 --- a/server-utils/src/suspendable_stream.rs +++ b/server-utils/src/suspendable_stream.rs @@ -1,5 +1,5 @@ -use std::io; use std::future::Future; +use std::io; use std::pin::Pin; use std::task::Poll; use std::time::Duration; @@ -57,7 +57,7 @@ where self.next_delay = self.initial_delay; } return Poll::Pending; - }, + } Poll::Ready(None) => { if self.next_delay > self.initial_delay { self.next_delay = self.initial_delay; diff --git a/stdio/src/lib.rs b/stdio/src/lib.rs index a31dbf6b2..8d3608a63 100644 --- a/stdio/src/lib.rs +++ b/stdio/src/lib.rs @@ -84,7 +84,7 @@ where /// Process a request asynchronously fn process(io: &Arc>, input: String) -> impl std::future::Future + Send { - use jsonrpc_core::futures::{FutureExt}; + use jsonrpc_core::futures::FutureExt; let f = io.handle_request(&input, Default::default()); f.map(move |result| match result { Some(res) => res, diff --git a/tcp/src/dispatch.rs b/tcp/src/dispatch.rs index 7b3a3f75d..e8563856e 100644 --- a/tcp/src/dispatch.rs +++ b/tcp/src/dispatch.rs @@ -1,8 +1,8 @@ use std; use std::collections::HashMap; use std::net::SocketAddr; -use std::sync::Arc; use std::pin::Pin; +use std::sync::Arc; use std::task::Poll; use crate::futures::{channel::mpsc, Stream}; diff --git a/tcp/src/server.rs b/tcp/src/server.rs index 51a9d554c..41b3a8fce 100644 --- a/tcp/src/server.rs +++ b/tcp/src/server.rs @@ -1,14 +1,14 @@ use std; use std::io; use std::net::SocketAddr; -use std::sync::Arc; use std::pin::Pin; +use std::sync::Arc; use tower_service::Service as _; +use crate::futures::{self, future}; use crate::jsonrpc::{middleware, MetaIoHandler, Metadata, Middleware}; use crate::server_utils::{codecs, reactor, tokio, tokio_util::codec::Framed, SuspendableStream}; -use crate::futures::{self, future}; use crate::dispatch::{Dispatcher, PeerMessageQueue, SenderChannels}; use crate::meta::{MetaExtractor, NoopExtractor, RequestContext}; @@ -102,9 +102,7 @@ where Ok(addr) => addr, Err(e) => { warn!(target: "tcp", "Unable to determine socket peer address, ignoring connection {}", e); - return future::Either::Left(async { - io::Result::Ok(()) - }); + return future::Either::Left(async { io::Result::Ok(()) }); } }; trace!(target: "tcp", "Accepted incoming connection from {}", &peer_addr); @@ -125,22 +123,22 @@ where // Work around https://github.com/rust-lang/rust/issues/64552 by boxing the stream type let responses: Pin> + Send>> = - Box::pin(reader.and_then(move |req| { - service.call(req).then(|response| match response { - Err(e) => { - warn!(target: "tcp", "Error while processing request: {:?}", e); - future::ok(String::new()) - } - Ok(None) => { - trace!(target: "tcp", "JSON RPC request produced no response"); - future::ok(String::new()) - } - Ok(Some(response_data)) => { - trace!(target: "tcp", "Sent response: {}", &response_data); - future::ok(response_data) - } - }) - })); + Box::pin(reader.and_then(move |req| { + service.call(req).then(|response| match response { + Err(e) => { + warn!(target: "tcp", "Error while processing request: {:?}", e); + future::ok(String::new()) + } + Ok(None) => { + trace!(target: "tcp", "JSON RPC request produced no response"); + future::ok(String::new()) + } + Ok(Some(response_data)) => { + trace!(target: "tcp", "Sent response: {}", &response_data); + future::ok(response_data) + } + }) + })); let mut peer_message_queue = { let mut channels = channels.lock(); @@ -167,9 +165,7 @@ where match start.await { Ok(server) => { tx.send(Ok(())).expect("Rx is blocking parent thread."); - let server = server - .buffer_unordered(1024) - .for_each(|_| async { () }); + let server = server.buffer_unordered(1024).for_each(|_| async { () }); future::select(Box::pin(server), stop_rx).await; } diff --git a/tcp/src/service.rs b/tcp/src/service.rs index 01c20d55d..558b2feba 100644 --- a/tcp/src/service.rs +++ b/tcp/src/service.rs @@ -4,9 +4,8 @@ use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; -use crate::jsonrpc::{middleware, MetaIoHandler, Metadata, Middleware}; use crate::futures; - +use crate::jsonrpc::{middleware, MetaIoHandler, Metadata, Middleware}; pub struct Service = middleware::Noop> { handler: Arc>, diff --git a/tcp/src/tests.rs b/tcp/src/tests.rs index eaf211ce9..b95e1b288 100644 --- a/tcp/src/tests.rs +++ b/tcp/src/tests.rs @@ -4,11 +4,11 @@ use std::sync::Arc; use std::time::Duration; use jsonrpc_core::{MetaIoHandler, Metadata, Value}; -use tokio::io::AsyncWriteExt; use tokio::io::AsyncReadExt; +use tokio::io::AsyncWriteExt; -use crate::server_utils::tokio::{self, net::TcpStream}; use crate::futures; +use crate::server_utils::tokio::{self, net::TcpStream}; use parking_lot::Mutex; @@ -48,9 +48,7 @@ fn doc_test_connect() { let server = casual_server(); let _server = server.start(&addr).expect("Server must run with no issues"); - run_future(async move { - TcpStream::connect(&addr).await - }).expect("Server connection error"); + run_future(async move { TcpStream::connect(&addr).await }).expect("Server connection error"); } #[test] diff --git a/ws/src/metadata.rs b/ws/src/metadata.rs index 5d2621787..25b1bed82 100644 --- a/ws/src/metadata.rs +++ b/ws/src/metadata.rs @@ -1,12 +1,12 @@ -use std::future::Future; use std::fmt; +use std::future::Future; use std::pin::Pin; use std::sync::{atomic, Arc}; -use std::task::{Poll, Context}; +use std::task::{Context, Poll}; use crate::core; use crate::core::futures::channel::mpsc; -use crate::server_utils::{session, reactor::TaskExecutor}; +use crate::server_utils::{reactor::TaskExecutor, session}; use crate::ws; use crate::error; diff --git a/ws/src/session.rs b/ws/src/session.rs index d89401078..eef7b0e9e 100644 --- a/ws/src/session.rs +++ b/ws/src/session.rs @@ -1,11 +1,11 @@ use std::future::Future; use std::pin::Pin; use std::sync::{atomic, Arc}; -use std::task::{Poll, Context}; +use std::task::{Context, Poll}; use crate::core; -use futures::future; use futures::channel::oneshot; +use futures::future; use futures::FutureExt; use parking_lot::Mutex; @@ -13,8 +13,8 @@ use slab::Slab; use crate::server_utils::cors::Origin; use crate::server_utils::hosts::Host; -use crate::server_utils::session::{SessionId, SessionStats}; use crate::server_utils::reactor::TaskExecutor; +use crate::server_utils::session::{SessionId, SessionStats}; use crate::server_utils::Pattern; use crate::ws; @@ -273,24 +273,23 @@ where let active_lock = self.active.clone(); let response = self.handler.handle_request(req, metadata); - let future = response - .map(move |response| { - if !active_lock.load(atomic::Ordering::SeqCst) { - return; - } - if let Some(result) = response { - let res = out.send(result); - match res { - Err(error::Error::ConnectionClosed) => { - active_lock.store(false, atomic::Ordering::SeqCst); - } - Err(e) => { - warn!("Error while sending response: {:?}", e); - } - _ => {} + let future = response.map(move |response| { + if !active_lock.load(atomic::Ordering::SeqCst) { + return; + } + if let Some(result) = response { + let res = out.send(result); + match res { + Err(error::Error::ConnectionClosed) => { + active_lock.store(false, atomic::Ordering::SeqCst); } + Err(e) => { + warn!("Error while sending response: {:?}", e); + } + _ => {} } - }); + } + }); let future = future::select(future, poll_liveness); self.executor.spawn(future); From 7d4faa0a7002055645bd670d872ca919ccc3fbf0 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 4 Jan 2021 15:18:38 +0100 Subject: [PATCH 63/73] Don't do redundant work when suspendable stream is pending --- server-utils/src/suspendable_stream.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/server-utils/src/suspendable_stream.rs b/server-utils/src/suspendable_stream.rs index 7d6d57b52..8d3179da9 100644 --- a/server-utils/src/suspendable_stream.rs +++ b/server-utils/src/suspendable_stream.rs @@ -52,12 +52,7 @@ where } match Pin::new(&mut self.stream).poll_next(cx) { - Poll::Pending => { - if self.next_delay > self.initial_delay { - self.next_delay = self.initial_delay; - } - return Poll::Pending; - } + Poll::Pending => return Poll::Pending, Poll::Ready(None) => { if self.next_delay > self.initial_delay { self.next_delay = self.initial_delay; From a883edcdde94764b0efb19d579982f60ed8a029a Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 4 Jan 2021 16:16:41 +0100 Subject: [PATCH 64/73] Return future directly instead in stdio server --- stdio/Cargo.toml | 3 ++- stdio/examples/stdio.rs | 6 ++++-- stdio/src/lib.rs | 35 ++++++++++++++++++----------------- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/stdio/Cargo.toml b/stdio/Cargo.toml index e74e28867..c7a745dae 100644 --- a/stdio/Cargo.toml +++ b/stdio/Cargo.toml @@ -13,10 +13,11 @@ version = "16.0.0" futures = { version = "0.3", features = [ "compat" ] } jsonrpc-core = { version = "16.0", path = "../core" } log = "0.4" -tokio = { version = "0.2", features = ["io-std", "rt-core", "io-driver", "io-util"] } +tokio = { version = "0.2", features = ["io-std", "io-driver", "io-util"] } tokio-util = { version = "0.3", features = ["codec"] } [dev-dependencies] +tokio = { version = "0.2", features = ["rt-core", "macros"] } lazy_static = "1.0" env_logger = "0.7" diff --git a/stdio/examples/stdio.rs b/stdio/examples/stdio.rs index 974ea9df4..bd2bc2caa 100644 --- a/stdio/examples/stdio.rs +++ b/stdio/examples/stdio.rs @@ -1,9 +1,11 @@ use jsonrpc_stdio_server::jsonrpc_core::*; use jsonrpc_stdio_server::ServerBuilder; -fn main() { +#[tokio::main] +async fn main() { let mut io = IoHandler::default(); io.add_sync_method("say_hello", |_params| Ok(Value::String("hello".to_owned()))); - ServerBuilder::new(io).build(); + let server = ServerBuilder::new(io).build(); + server.await; } diff --git a/stdio/src/lib.rs b/stdio/src/lib.rs index 8d3608a63..591423358 100644 --- a/stdio/src/lib.rs +++ b/stdio/src/lib.rs @@ -5,24 +5,28 @@ //! use jsonrpc_stdio_server::ServerBuilder; //! use jsonrpc_stdio_server::jsonrpc_core::*; //! -//! fn main() { +//! #[tokio::main] +//! async fn main() { //! let mut io = IoHandler::default(); //! io.add_sync_method("say_hello", |_params| { //! Ok(Value::String("hello".to_owned())) //! }); //! -//! ServerBuilder::new(io).build(); +//! let server = ServerBuilder::new(io).build(); +//! server.await; //! } //! ``` #![deny(missing_docs)] use std::sync::Arc; +use std::future::Future; #[macro_use] extern crate log; pub use jsonrpc_core; +pub use tokio; use jsonrpc_core::{MetaIoHandler, Metadata, Middleware}; use tokio_util::codec::{FramedRead, LinesCodec}; @@ -45,18 +49,21 @@ where } } + /// Returns a server future that needs to be polled in order to make progress. + /// /// Will block until EOF is read or until an error occurs. /// The server reads from STDIN line-by-line, one request is taken /// per line and each response is written to STDOUT on a new line. - pub fn build(&self) { - let stdin = tokio::io::stdin(); - let mut stdout = tokio::io::stdout(); + pub fn build(&self) -> impl Future + 'static { + let handler = self.handler.clone(); - let mut framed_stdin = FramedRead::new(stdin, LinesCodec::new()); + async move { + let stdin = tokio::io::stdin(); + let mut stdout = tokio::io::stdout(); - let handler = self.handler.clone(); - use futures::StreamExt; - let future = async { + let mut framed_stdin = FramedRead::new(stdin, LinesCodec::new()); + + use futures::StreamExt; while let Some(request) = framed_stdin.next().await { match request { Ok(line) => { @@ -73,17 +80,11 @@ where } } } - }; - let mut rt = tokio::runtime::Builder::new() - .basic_scheduler() - .enable_io() - .build() - .unwrap(); - rt.block_on(future); + } } /// Process a request asynchronously - fn process(io: &Arc>, input: String) -> impl std::future::Future + Send { + fn process(io: &Arc>, input: String) -> impl Future + Send { use jsonrpc_core::futures::FutureExt; let f = io.handle_request(&input, Default::default()); f.map(move |result| match result { From 7039504762d1b9e9c890be7f1f7023fe690d59e1 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 4 Jan 2021 16:21:31 +0100 Subject: [PATCH 65/73] Simplify some imports in Cargo.tomls --- core-client/transports/Cargo.toml | 2 +- ipc/Cargo.toml | 2 +- ipc/src/server.rs | 8 ++++---- server-utils/Cargo.toml | 4 ++-- stdio/Cargo.toml | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/core-client/transports/Cargo.toml b/core-client/transports/Cargo.toml index 2dffa6183..fb00df194 100644 --- a/core-client/transports/Cargo.toml +++ b/core-client/transports/Cargo.toml @@ -35,7 +35,7 @@ arbitrary_precision = ["serde_json/arbitrary_precision", "jsonrpc-core/arbitrary [dependencies] derive_more = "0.99" -futures = { version = "0.3", features = [ "compat" ] } +futures = "0.3" jsonrpc-core = { version = "16.0", path = "../../core" } jsonrpc-pubsub = { version = "16.0", path = "../../pubsub" } log = "0.4" diff --git a/ipc/Cargo.toml b/ipc/Cargo.toml index b8d152e6a..d2e3bcdc9 100644 --- a/ipc/Cargo.toml +++ b/ipc/Cargo.toml @@ -23,7 +23,7 @@ env_logger = "0.7" lazy_static = "1.0" [target.'cfg(not(windows))'.dev-dependencies] -tokio02 = { package = "tokio", version = "0.2", default-features = false, features = ["uds", "time", "rt-threaded", "io-driver"] } +tokio = { version = "0.2", default-features = false, features = ["uds", "time", "rt-threaded", "io-driver"] } [badges] travis-ci = { repository = "paritytech/jsonrpc", branch = "master"} diff --git a/ipc/src/server.rs b/ipc/src/server.rs index 8d02421fe..107150304 100644 --- a/ipc/src/server.rs +++ b/ipc/src/server.rs @@ -349,7 +349,7 @@ mod tests { use futures::SinkExt; let reply = async move { - use tokio02::net::UnixStream; + use tokio::net::UnixStream; let stream: UnixStream = UnixStream::connect(path).await?; let codec = codecs::StreamCodec::stream_incoming(); @@ -360,7 +360,7 @@ mod tests { reply.expect("there should be one reply") }; - let mut rt = tokio02::runtime::Runtime::new().unwrap(); + let mut rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(reply).expect("wait for reply") } @@ -609,9 +609,9 @@ mod tests { tx.send(true).expect("failed to report that the server has stopped"); }); - let mut rt = tokio02::runtime::Runtime::new().unwrap(); + let mut rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async move { - let timeout = tokio02::time::delay_for(Duration::from_millis(500)); + let timeout = tokio::time::delay_for(Duration::from_millis(500)); match futures::future::select(rx, timeout).await { futures::future::Either::Left((result, _)) => { diff --git a/server-utils/Cargo.toml b/server-utils/Cargo.toml index 205f8d8dd..e40481835 100644 --- a/server-utils/Cargo.toml +++ b/server-utils/Cargo.toml @@ -11,14 +11,14 @@ repository = "https://github.com/paritytech/jsonrpc" version = "16.0.0" [dependencies] -bytes = { version = "0.5", package = "bytes" } +bytes = "0.5" futures = "0.3" globset = "0.4" jsonrpc-core = { version = "16.0", path = "../core" } lazy_static = "1.1.0" log = "0.4" tokio = { version = "0.2", features = ["rt-threaded", "io-driver", "io-util", "time", "tcp"] } -tokio-util = { version = "0.3", features = ["codec", "compat"] } +tokio-util = { version = "0.3", features = ["codec"] } unicase = "2.0" diff --git a/stdio/Cargo.toml b/stdio/Cargo.toml index c7a745dae..44fcf9592 100644 --- a/stdio/Cargo.toml +++ b/stdio/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/paritytech/jsonrpc" version = "16.0.0" [dependencies] -futures = { version = "0.3", features = [ "compat" ] } +futures = "0.3" jsonrpc-core = { version = "16.0", path = "../core" } log = "0.4" tokio = { version = "0.2", features = ["io-std", "io-driver", "io-util"] } From 010701da77956a5a016b981414e00fae589f4a9f Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 4 Jan 2021 16:23:17 +0100 Subject: [PATCH 66/73] Format again --- stdio/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdio/src/lib.rs b/stdio/src/lib.rs index 591423358..79918c44a 100644 --- a/stdio/src/lib.rs +++ b/stdio/src/lib.rs @@ -19,8 +19,8 @@ #![deny(missing_docs)] -use std::sync::Arc; use std::future::Future; +use std::sync::Arc; #[macro_use] extern crate log; From bb42d82e024e73ece070b9a7e45c2a4b40e94f36 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Tue, 5 Jan 2021 00:53:30 +0100 Subject: [PATCH 67/73] Add missing feature for WS client transport --- core-client/transports/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/core-client/transports/Cargo.toml b/core-client/transports/Cargo.toml index fb00df194..cc576f1d3 100644 --- a/core-client/transports/Cargo.toml +++ b/core-client/transports/Cargo.toml @@ -25,6 +25,7 @@ http = ["hyper", "tokio/full"] ws = [ "websocket", "tokio", + "futures/compat" ] ipc = [ "parity-tokio-ipc", From dc1a81483c390625ffd712616f3219cd8c9f3d1b Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Tue, 5 Jan 2021 00:59:12 +0100 Subject: [PATCH 68/73] Don't spawn a thread in HTTP client connect --- core-client/transports/src/transports/http.rs | 58 +++++-------------- 1 file changed, 14 insertions(+), 44 deletions(-) diff --git a/core-client/transports/src/transports/http.rs b/core-client/transports/src/transports/http.rs index ccbf3578d..df518340a 100644 --- a/core-client/transports/src/transports/http.rs +++ b/core-client/transports/src/transports/http.rs @@ -8,45 +8,16 @@ use futures::{future, Future, FutureExt, StreamExt, TryFutureExt}; use hyper::{http, Client, Request, Uri}; /// Create a HTTP Client -pub fn connect(url: &str) -> impl Future> +pub async fn connect(url: &str) -> RpcResult where TClient: From, { - use futures::future::Either; + let url: Uri = url.parse().map_err(|e| RpcError::Other(Box::new(e)))?; - let (sender, receiver) = futures::channel::oneshot::channel(); - let url = url.to_owned(); + let (client_api, client_worker) = do_connect(url).await; + tokio::spawn(client_worker); - let url: Uri = match url.parse() { - Ok(url) => url, - Err(e) => return Either::Left(async { Err(RpcError::Other(Box::new(e))) }), - }; - - std::thread::spawn(move || { - let mut rt = tokio::runtime::Builder::new() - .basic_scheduler() - .enable_all() - .build() - .unwrap(); - - rt.block_on(async { - let (client_api, client_worker) = do_connect(url).await; - - if sender.send(client_api).is_err() { - panic!("The caller did not wait for the server."); - } - - // NOTE: We need to explicitly wait on a returned worker task; - // idleness tracking in runtime was removed from Tokio 0.1 - client_worker.await; - }); - }); - - return Either::Right(async move { - let api = receiver.await.expect("Server closed prematurely"); - - Ok(TClient::from(api)) - }); + Ok(TClient::from(client_api)) } async fn do_connect(url: Uri) -> (RpcChannel, impl Future) { @@ -220,7 +191,7 @@ mod tests { Ok(()) as RpcResult<_> }; - futures::executor::block_on(run).unwrap(); + tokio::runtime::Runtime::new().unwrap().block_on(run).unwrap(); } #[test] @@ -229,18 +200,16 @@ mod tests { // given let server = TestServer::serve(id); - let (tx, rx) = std::sync::mpsc::channel(); // when - let run = async move { + let run = async { let client: TestClient = connect(&server.uri).await.unwrap(); client.notify(12).unwrap(); - tx.send(()).unwrap(); }; - let pool = futures::executor::ThreadPool::builder().pool_size(1).create().unwrap(); - pool.spawn_ok(run); - rx.recv().unwrap(); + tokio::runtime::Runtime::new().unwrap().block_on(run); + // Ensure that server has not been moved into runtime + drop(server); } #[test] @@ -251,7 +220,8 @@ mod tests { let invalid_uri = "invalid uri"; // when - let res: RpcResult = futures::executor::block_on(connect(invalid_uri)); + let fut = connect(invalid_uri); + let res: RpcResult = tokio::runtime::Runtime::new().unwrap().block_on(fut); // then assert_matches!( @@ -273,7 +243,7 @@ mod tests { let client: TestClient = connect(&server.uri).await?; client.fail().await }; - let res = futures::executor::block_on(run); + let res = tokio::runtime::Runtime::new().unwrap().block_on(run); // then if let Err(RpcError::JsonRpcError(err)) = res { @@ -314,6 +284,6 @@ mod tests { Ok(()) as RpcResult<_> }; - futures::executor::block_on(run).unwrap(); + tokio::runtime::Runtime::new().unwrap().block_on(run).unwrap(); } } From 94a4b232feeb708d7e10c64320efd7609081524c Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Tue, 5 Jan 2021 03:09:41 +0100 Subject: [PATCH 69/73] Fix ws client sink impl with an unbounded queue --- core-client/transports/src/transports/ws.rs | 96 ++++++++++++++------- 1 file changed, 63 insertions(+), 33 deletions(-) diff --git a/core-client/transports/src/transports/ws.rs b/core-client/transports/src/transports/ws.rs index 8f0ea77e3..84fe451b2 100644 --- a/core-client/transports/src/transports/ws.rs +++ b/core-client/transports/src/transports/ws.rs @@ -73,8 +73,8 @@ struct WebsocketClient { impl WebsocketClient where - TSink: futures::Sink, - TStream: futures::Stream>, + TSink: futures::Sink + Unpin, + TStream: futures::Stream> + Unpin, TError: std::error::Error + Send + 'static, { pub fn new(sink: TSink, stream: TStream) -> Self { @@ -84,8 +84,39 @@ where queue: VecDeque::new(), } } + + // Drains the internal buffer and attempts to forward as much of the items + // as possible to the underlying sink + fn try_empty_buffer( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + let this = Pin::into_inner(self); + + match Pin::new(&mut this.sink).poll_ready(cx) { + Poll::Ready(value) => value?, + Poll::Pending => return Poll::Pending, + } + + while let Some(item) = this.queue.pop_front() { + Pin::new(&mut this.sink).start_send(item)?; + + if !this.queue.is_empty() { + match Pin::new(&mut this.sink).poll_ready(cx) { + Poll::Ready(value) => value?, + Poll::Pending => return Poll::Pending, + } + } + } + + Poll::Ready(Ok(())) + } } +// This mostly forwards to the underlying sink but also adds an unbounded queue +// for when the underlying sink is incapable of receiving more items. +// See https://docs.rs/futures-util/0.3.8/futures_util/sink/struct.Buffer.html +// for the variant with a fixed-size buffer. impl futures::Sink for WebsocketClient where TSink: futures::Sink + Unpin, @@ -94,52 +125,51 @@ where type Error = RpcError; fn start_send(mut self: Pin<&mut Self>, request: String) -> Result<(), Self::Error> { - self.queue.push_back(OwnedMessage::Text(request)); - Ok(()) + let request = OwnedMessage::Text(request); + + if self.queue.is_empty() { + let this = Pin::into_inner(self); + Pin::new(&mut this.sink).start_send(request) + } else { + self.queue.push_back(request); + Ok(()) + } } fn poll_ready(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = Pin::into_inner(self); - loop { - match this.queue.pop_front() { - Some(request) => match Pin::new(&mut this.sink).poll_ready(cx) { - Poll::Ready(Ok(())) => { - if let Err(err) = Pin::new(&mut this.sink).start_send(request) { - return Poll::Ready(Err(RpcError::Other(Box::new(err)))); - } - continue; - } - poll => { - this.queue.push_front(request); - return poll; - } - }, - None => break, - } + if this.queue.is_empty() { + return Pin::new(&mut this.sink).poll_ready(cx); } - Pin::new(&mut this.sink) - .poll_ready(cx) - .map_err(|error| RpcError::Other(Box::new(error))) + let _ = Pin::new(this).try_empty_buffer(cx)?; + + Poll::Ready(Ok(())) } fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - if !self.queue.is_empty() { - self.poll_ready(cx) - } else { - let this = Pin::into_inner(self); - Pin::new(&mut this.sink).poll_flush(cx) + let this = Pin::into_inner(self); + + match Pin::new(&mut *this).try_empty_buffer(cx) { + Poll::Ready(value) => value?, + Poll::Pending => return Poll::Pending, } + debug_assert!(this.queue.is_empty()); + + Pin::new(&mut this.sink).poll_flush(cx) } fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - if !self.queue.is_empty() { - self.poll_ready(cx) - } else { - let this = Pin::into_inner(self); - Pin::new(&mut this.sink).poll_close(cx) + let this = Pin::into_inner(self); + + match Pin::new(&mut *this).try_empty_buffer(cx) { + Poll::Ready(value) => value?, + Poll::Pending => return Poll::Pending, } + debug_assert!(this.queue.is_empty()); + + Pin::new(&mut this.sink).poll_close(cx) } } From 1bd2fc5a82b77e2a7598a4e0dd7de556cbc6b534 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Tue, 5 Jan 2021 03:14:25 +0100 Subject: [PATCH 70/73] Format again --- core-client/transports/src/transports/ws.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/core-client/transports/src/transports/ws.rs b/core-client/transports/src/transports/ws.rs index 84fe451b2..976d48748 100644 --- a/core-client/transports/src/transports/ws.rs +++ b/core-client/transports/src/transports/ws.rs @@ -87,10 +87,7 @@ where // Drains the internal buffer and attempts to forward as much of the items // as possible to the underlying sink - fn try_empty_buffer( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll> { + fn try_empty_buffer(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = Pin::into_inner(self); match Pin::new(&mut this.sink).poll_ready(cx) { @@ -98,19 +95,19 @@ where Poll::Pending => return Poll::Pending, } - while let Some(item) = this.queue.pop_front() { + while let Some(item) = this.queue.pop_front() { Pin::new(&mut this.sink).start_send(item)?; - if !this.queue.is_empty() { + if !this.queue.is_empty() { match Pin::new(&mut this.sink).poll_ready(cx) { Poll::Ready(value) => value?, Poll::Pending => return Poll::Pending, } - } + } } - Poll::Ready(Ok(())) - } + Poll::Ready(Ok(())) + } } // This mostly forwards to the underlying sink but also adds an unbounded queue From 82c5be144d35bd209031fcf70c2fcfc766602644 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Tue, 5 Jan 2021 10:11:59 +0100 Subject: [PATCH 71/73] Use released parity-tokio-ipc v0.8 crate --- Cargo.toml | 3 --- core-client/transports/Cargo.toml | 2 +- ipc/Cargo.toml | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4c27e6340..b509827d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,3 @@ members = [ "test", "ws", ] - -[patch.crates-io] -parity-tokio-ipc = { git = "https://github.com/Xanewok/parity-tokio-ipc", branch = "endpoint-incoming-static" } diff --git a/core-client/transports/Cargo.toml b/core-client/transports/Cargo.toml index cc576f1d3..2cad8f8ad 100644 --- a/core-client/transports/Cargo.toml +++ b/core-client/transports/Cargo.toml @@ -47,7 +47,7 @@ url = "1.7" hyper = { version = "0.13", optional = true } hyper-tls = { version = "0.4", optional = true } jsonrpc-server-utils = { version = "16.0", path = "../../server-utils", optional = true } -parity-tokio-ipc = { version = "0.7", optional = true } +parity-tokio-ipc = { version = "0.8", optional = true } tokio = { version = "0.2", optional = true } websocket = { version = "0.24", optional = true } diff --git a/ipc/Cargo.toml b/ipc/Cargo.toml index d2e3bcdc9..63678c33c 100644 --- a/ipc/Cargo.toml +++ b/ipc/Cargo.toml @@ -15,7 +15,7 @@ log = "0.4" tower-service = "0.3" jsonrpc-core = { version = "16.0", path = "../core" } jsonrpc-server-utils = { version = "16.0", path = "../server-utils", default-features = false } -parity-tokio-ipc = "0.7" +parity-tokio-ipc = "0.8" parking_lot = "0.11.0" [dev-dependencies] From 35cabd123cc8707de959eda7c7b190e644884465 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 11 Jan 2021 15:15:30 +0100 Subject: [PATCH 72/73] Remove extraneous `use std;` imports --- tcp/src/dispatch.rs | 1 - tcp/src/server.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/tcp/src/dispatch.rs b/tcp/src/dispatch.rs index e8563856e..664e97a33 100644 --- a/tcp/src/dispatch.rs +++ b/tcp/src/dispatch.rs @@ -1,4 +1,3 @@ -use std; use std::collections::HashMap; use std::net::SocketAddr; use std::pin::Pin; diff --git a/tcp/src/server.rs b/tcp/src/server.rs index 41b3a8fce..55a2b8d4a 100644 --- a/tcp/src/server.rs +++ b/tcp/src/server.rs @@ -1,4 +1,3 @@ -use std; use std::io; use std::net::SocketAddr; use std::pin::Pin; From a05cd4ed8ae48b3838831eecb0dd608fac13de50 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 11 Jan 2021 15:28:57 +0100 Subject: [PATCH 73/73] Re-export tokio from ipc server --- ipc/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipc/src/lib.rs b/ipc/src/lib.rs index 0629de809..ab9e73130 100644 --- a/ipc/src/lib.rs +++ b/ipc/src/lib.rs @@ -25,5 +25,5 @@ use jsonrpc_core as jsonrpc; pub use crate::meta::{MetaExtractor, NoopExtractor, RequestContext}; pub use crate::server::{CloseHandle, SecurityAttributes, Server, ServerBuilder}; -pub use self::server_utils::codecs::Separator; pub use self::server_utils::session::{SessionId, SessionStats}; +pub use self::server_utils::{codecs::Separator, tokio};