From c4159b9e0bc490a4785dec1753b3c9796b763a2b Mon Sep 17 00:00:00 2001 From: "249639015@qq.conm" <249639015@qq.conm> Date: Sat, 12 Oct 2024 01:58:59 +0800 Subject: [PATCH 1/5] handle : create macro format code style. --- crates/core/src/runtime/group.rs | 1 + crates/core/src/runtime/nodes/function_nodes/function/mod.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/crates/core/src/runtime/group.rs b/crates/core/src/runtime/group.rs index b8f9a660..cb20a9d0 100644 --- a/crates/core/src/runtime/group.rs +++ b/crates/core/src/runtime/group.rs @@ -1,5 +1,6 @@ use std::sync::Arc; use std::sync::Weak; +use crate::utils::constants::ENV_STR; use super::env::*; use super::flow::*; diff --git a/crates/core/src/runtime/nodes/function_nodes/function/mod.rs b/crates/core/src/runtime/nodes/function_nodes/function/mod.rs index 78ed167a..664b57ea 100644 --- a/crates/core/src/runtime/nodes/function_nodes/function/mod.rs +++ b/crates/core/src/runtime/nodes/function_nodes/function/mod.rs @@ -16,6 +16,7 @@ use crate::runtime::flow::Flow; use crate::runtime::model::*; use crate::runtime::nodes::*; use edgelink_macro::*; +use crate::utils::constants::ENV_STR; mod context_class; mod edgelink_class; From cd4b128bdb6afdb4f5879ddf8867d5db999c4e7d Mon Sep 17 00:00:00 2001 From: "249639015@qq.conm" <249639015@qq.conm> Date: Sat, 12 Oct 2024 18:07:27 +0800 Subject: [PATCH 2/5] =?UTF-8?q?constants=20:=20static=20=E5=88=87=E6=8D=A2?= =?UTF-8?q?=E6=88=90const=20.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/core/src/utils/constants.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/core/src/utils/constants.rs b/crates/core/src/utils/constants.rs index a086a2a2..e1df1afa 100644 --- a/crates/core/src/utils/constants.rs +++ b/crates/core/src/utils/constants.rs @@ -1,8 +1,8 @@ -pub static SUB_FLOW_TYPE:&'static str="subflow"; -pub static SUB_FLOW_TYPE_HEAD:&'static str="subflow:"; -pub static TAB_STR:&'static str="tab"; -pub static FLOW_STR:&'static str="flow"; -pub static TYPE_STR:&'static str="type"; -pub static NAME_STR:&'static str="name"; -pub static ID_STR:&'static str="id"; -pub static ENV_STR:&'static str="env"; \ No newline at end of file +pub const SUB_FLOW_TYPE:&'static str="subflow"; +pub const SUB_FLOW_TYPE_HEAD:&'static str="subflow:"; +pub const TAB_STR:&'static str="tab"; +pub const FLOW_STR:&'static str="flow"; +pub const TYPE_STR:&'static str="type"; +pub const NAME_STR:&'static str="name"; +pub const ID_STR:&'static str="id"; +pub const ENV_STR:&'static str="env"; \ No newline at end of file From 68eaf95ebedf0dd5e7184788e29d3ee91ac61d01 Mon Sep 17 00:00:00 2001 From: "249639015@qq.conm" <249639015@qq.conm> Date: Sat, 12 Oct 2024 18:15:28 +0800 Subject: [PATCH 3/5] constants : static ident change to const . --- crates/core/src/utils/handle.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/core/src/utils/handle.rs b/crates/core/src/utils/handle.rs index e2752b05..4fb22d40 100644 --- a/crates/core/src/utils/handle.rs +++ b/crates/core/src/utils/handle.rs @@ -1,7 +1,7 @@ -/// [handle_error] 通用的msg处理方式实现error throw 的代码优雅程度。 +/// [handle_option] 通用的msg处理方式实现error throw 的代码优雅程度。 /// /// # Example /// From 47521bcd16453f1773fbfbb62e77178a72403207 Mon Sep 17 00:00:00 2001 From: "249639015@qq.conm" <249639015@qq.conm> Date: Sat, 12 Oct 2024 22:30:53 +0800 Subject: [PATCH 4/5] deser : logic match node properties json --- Cargo.lock | 132 +++++++++++++++++ crates/core/Cargo.toml | 2 + crates/core/src/runtime/model/json/mod.rs | 1 + crates/core/src/runtime/model/json/npdeser.rs | 140 ++++++++++++++++++ crates/core/src/runtime/nodes/mod.rs | 2 + .../src/runtime/nodes/storage_nodes/file.rs | 113 ++++++++++++++ .../runtime/nodes/storage_nodes/file_in.rs | 0 .../src/runtime/nodes/storage_nodes/mod.rs | 3 + .../src/runtime/nodes/storage_nodes/watch.rs | 0 9 files changed, 393 insertions(+) create mode 100644 crates/core/src/runtime/model/json/npdeser.rs create mode 100644 crates/core/src/runtime/nodes/storage_nodes/file.rs create mode 100644 crates/core/src/runtime/nodes/storage_nodes/file_in.rs create mode 100644 crates/core/src/runtime/nodes/storage_nodes/mod.rs create mode 100644 crates/core/src/runtime/nodes/storage_nodes/watch.rs diff --git a/Cargo.lock b/Cargo.lock index bdf8fd05..a41e9030 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -382,6 +382,41 @@ dependencies = [ "syn 2.0.79", ] +[[package]] +name = "darling" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.79", +] + +[[package]] +name = "darling_macro" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.79", +] + [[package]] name = "dashmap" version = "6.1.0" @@ -488,6 +523,7 @@ dependencies = [ "tokio", "tokio-cron-scheduler", "tokio-util", + "validator", ] [[package]] @@ -576,6 +612,15 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + [[package]] name = "futures" version = "0.3.30" @@ -756,6 +801,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "indexmap" version = "2.5.0" @@ -1044,6 +1099,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + [[package]] name = "pin-project-lite" version = "0.2.14" @@ -1565,6 +1626,21 @@ dependencies = [ "winapi", ] +[[package]] +name = "tinyvec" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.40.0" @@ -1706,12 +1782,27 @@ dependencies = [ "unsafe-any-ors", ] +[[package]] +name = "unicode-bidi" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" + [[package]] name = "unicode-ident" version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +[[package]] +name = "unicode-normalization" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -1733,6 +1824,17 @@ dependencies = [ "destructure_traitobject", ] +[[package]] +name = "url" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + [[package]] name = "utf8parse" version = "0.2.2" @@ -1748,6 +1850,36 @@ dependencies = [ "getrandom", ] +[[package]] +name = "validator" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db79c75af171630a3148bd3e6d7c4f42b6a9a014c2945bc5ed0020cbb8d9478e" +dependencies = [ + "idna", + "once_cell", + "regex", + "serde", + "serde_derive", + "serde_json", + "url", + "validator_derive", +] + +[[package]] +name = "validator_derive" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0bcf92720c40105ac4b2dda2a4ea3aa717d4d6a862cc217da653a4bd5c6b10" +dependencies = [ + "darling", + "once_cell", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "version_check" version = "0.9.5" diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 2a5dfd0c..33410356 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -50,6 +50,8 @@ smallvec.workspace = true smallstr.workspace = true inventory.workspace = true arrayvec = { workspace = true, features = ["std", "serde"] } +validator = { version = "0.18.1", features = ["derive"] } + [dev-dependencies] # Enable test-utilities in dev mode only. This is mostly for tests. diff --git a/crates/core/src/runtime/model/json/mod.rs b/crates/core/src/runtime/model/json/mod.rs index 747f8dd3..211a0f41 100644 --- a/crates/core/src/runtime/model/json/mod.rs +++ b/crates/core/src/runtime/model/json/mod.rs @@ -5,6 +5,7 @@ use serde_json::Value as JsonValue; pub mod deser; pub mod helpers; +mod npdeser; #[derive(serde::Deserialize, Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] pub struct RedPortConfig { diff --git a/crates/core/src/runtime/model/json/npdeser.rs b/crates/core/src/runtime/model/json/npdeser.rs new file mode 100644 index 00000000..5c5b5fae --- /dev/null +++ b/crates/core/src/runtime/model/json/npdeser.rs @@ -0,0 +1,140 @@ +//! [npdeser] this mod use to de deserializer node logic properties transport to real logic. +//! +//! # example +//! > this appendNewline config is belong to node red core node [file], Used to determine whether +//! > to wrap a file ,it's could It should be a boolean type, but the code logic allows it to be +//! > any non undefined, true false 0 and 1, and any character ,and any str. so need this mod handle +//! > this scene +//! ```js +//! this.appendNewline = n.appendNewline; +//! +//! if ((node.appendNewline) && (!Buffer.isBuffer(data)) && aflg) { data += os.EOL; } +//! ``` +//! +use std::borrow::Cow; +use std::str; +use serde::{de, Deserializer}; +use serde::de::{Error, Unexpected}; + +pub fn deser_bool_in_if_condition<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, +{ + struct BoolVisitor; + + impl<'de> de::Visitor<'de> for BoolVisitor { + type Value = bool; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a bool, convert failed") + } + fn visit_bytes(self, v: &[u8]) -> Result where E: Error { + match str::from_utf8(v) { + Ok(s) => { + if s =="" { + return Ok(false); + }else if s=="0" { + return Ok(false); + }else if s.contains("false")||s.contains("False")||s.contains("FALSE"){ + return Ok(false); + } + Ok(true) + }, + Err(_) => Err(Error::invalid_value(Unexpected::Bool(false), &self)), + } + } + fn visit_u64(self, v: u64) -> Result where E: Error { + println!("xxxxxxxxx"); + if v==0 { + return Ok(false); + } + Ok(true) + } + + + fn visit_f64(self, v: f64) -> Result where E: Error { + if v==0.0 { + return Ok(false); + } + Ok(true) + } + + fn visit_f32(self, v: f32) -> Result where E: Error { + if v==0.0 { + return Ok(false); + } + Ok(true) + } + + fn visit_str(self, v: &str) -> Result where E: Error { + if v =="" { + return Ok(false); + }else if v=="0" { + return Ok(false); + }else if v.contains("false")||v.contains("False")||v.contains("FALSE"){ + return Ok(false); + } + Ok(true) + } + } + + deserializer.deserialize_any(BoolVisitor) +} + + +#[cfg(test)] +mod tests { + use std::net::IpAddr; + use serde::Deserialize; + use serde_json::json; + use super::*; + + #[derive(Deserialize, Debug)] + struct TestNodeConfig { + #[serde(deserialize_with = "crate::runtime::model::json::npdeser::deser_bool_in_if_condition")] + test: bool + } + + #[test] + fn test_deser_bool_in_if_condition() { + let value_str = json!({"test":"xxx"}); + let result = TestNodeConfig::deserialize(value_str).unwrap(); + assert!(result.test); + + let value_str = json!({"test":"true"}); + let result = TestNodeConfig::deserialize(value_str).unwrap(); + assert!(result.test); + + + let value_str = json!({"test":"false"}); + let result = TestNodeConfig::deserialize(value_str).unwrap(); + assert!(!result.test); + + + let value_str = json!({"test":"False"}); + let result = TestNodeConfig::deserialize(value_str).unwrap(); + assert!(!result.test); + + let value_str = json!({"test":"0"}); + let result = TestNodeConfig::deserialize(value_str).unwrap(); + assert!(!result.test); + + let value_str = json!({"test":1.0}); + let result = TestNodeConfig::deserialize(value_str).unwrap(); + assert!(result.test); + + let value_str = json!({"test":0.0}); + let result = TestNodeConfig::deserialize(value_str).unwrap(); + assert!(!result.test); + + + let value_str = json!({"test":0}); + let result = TestNodeConfig::deserialize(value_str).unwrap(); + assert!(!result.test); + + + let value_str = json!({"test":1}); + let result = TestNodeConfig::deserialize(value_str).unwrap(); + assert!(result.test); + + } +} \ No newline at end of file diff --git a/crates/core/src/runtime/nodes/mod.rs b/crates/core/src/runtime/nodes/mod.rs index 3bd6a33b..f0651ee6 100644 --- a/crates/core/src/runtime/nodes/mod.rs +++ b/crates/core/src/runtime/nodes/mod.rs @@ -20,6 +20,8 @@ use crate::utils::constants::FLOW_STR; pub(crate) mod common_nodes; mod function_nodes; +mod storage_nodes; + #[cfg(feature = "net")] mod network_nodes; diff --git a/crates/core/src/runtime/nodes/storage_nodes/file.rs b/crates/core/src/runtime/nodes/storage_nodes/file.rs new file mode 100644 index 00000000..a0324279 --- /dev/null +++ b/crates/core/src/runtime/nodes/storage_nodes/file.rs @@ -0,0 +1,113 @@ +use std::net::{IpAddr, SocketAddr}; +use std::sync::Arc; +use tokio::net::UdpSocket; + +use base64::prelude::*; +use serde::Deserialize; + +use crate::runtime::flow::Flow; +use crate::runtime::nodes::*; +use edgelink_macro::*; +use validator::{Validate, ValidationError}; + + + + + +#[derive(Debug)] +#[flow_node("file")] +struct FileNode { + base: FlowNode, + config: FileNodeConfig, +} + +impl FileNode { + fn build(_flow: &Flow, state: FlowNode, config: &RedFlowNodeConfig) -> crate::Result> { + let udp_config = FileNodeConfig::deserialize(&config.rest)?; + + + + let node = FileNode { base: state, config: udp_config }; + Ok(Box::new(node)) + } +} + +#[derive(Deserialize,Validate, Debug)] +struct FileNodeConfig { + #[validate(length(min = 1))] + filename: String, + #[serde(rename = "filenameType")] + filename_type: Option, + #[serde( rename= "appendNewline")] + append_new_line: String, + #[serde( rename= "overwriteFile")] + overwrite_file: String, + create_dir: bool, + encoding: String, + +} + +impl FileNode { + async fn uow(&self, msg: MsgHandle, socket: &UdpSocket) -> crate::Result<()> { + let msg_guard = msg.read().await; + if let Some(payload) = msg_guard.get("payload") { + let remote_addr = std::net::SocketAddr::new( + self.config.addr.unwrap(), // TODO FIXME + self.config.port.unwrap(), + ); + + if let Some(bytes) = payload.as_bytes() { + if self.config.base64 { + let b64_str = BASE64_STANDARD.encode(bytes); + let bytes = b64_str.as_bytes(); + socket.send_to(bytes, remote_addr).await?; + } else { + socket.send_to(bytes, remote_addr).await?; + } + } + if let Some(bytes) = payload.to_bytes() { + socket.send_to(&bytes, remote_addr).await?; + } else { + log::warn!("Failed to convert payload into bytes"); + } + } + + Ok(()) + } +} + +#[async_trait] +impl FlowNodeBehavior for FileNode { + fn get_node(&self) -> &FlowNode { + &self.base + } + + async fn run(self: Arc, stop_token: CancellationToken) { + let local_addr: SocketAddr = match self.config.outport { + Some(port) => SocketAddr::new(self.config.iface.unwrap(), port), + _ => match self.config.ipv { + UdpIpV::V4 => "0.0.0.0:0".parse().unwrap(), + UdpIpV::V6 => "[::]:0".parse().unwrap(), + }, + }; + + match tokio::net::UdpSocket::bind(local_addr).await { + Ok(socket) => { + let socket = Arc::new(socket); + while !stop_token.is_cancelled() { + let cloned_socket = socket.clone(); + + let node = self.clone(); + with_uow(node.as_ref(), stop_token.clone(), |node, msg| async move { + node.uow(msg, &cloned_socket).await + }) + .await; + } + } + + Err(e) => { + log::error!("Can not bind local address: {:?}", e); + } + } + } +} diff --git a/crates/core/src/runtime/nodes/storage_nodes/file_in.rs b/crates/core/src/runtime/nodes/storage_nodes/file_in.rs new file mode 100644 index 00000000..e69de29b diff --git a/crates/core/src/runtime/nodes/storage_nodes/mod.rs b/crates/core/src/runtime/nodes/storage_nodes/mod.rs new file mode 100644 index 00000000..59e6f308 --- /dev/null +++ b/crates/core/src/runtime/nodes/storage_nodes/mod.rs @@ -0,0 +1,3 @@ +// mod file; +mod watch; +mod file_in; \ No newline at end of file diff --git a/crates/core/src/runtime/nodes/storage_nodes/watch.rs b/crates/core/src/runtime/nodes/storage_nodes/watch.rs new file mode 100644 index 00000000..e69de29b From 8f0529c77c6e0f0e0419efdfd9dba97347d6b54b Mon Sep 17 00:00:00 2001 From: ZJ van de Weg Date: Sat, 12 Oct 2024 19:56:43 +0200 Subject: [PATCH 5/5] readme: Config by flags on the executable The docs suggested there's a configuration file, which I was unable to locate. The way I've been able to configure the behaviour is through flags. This change records it in the README. --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 20dc030d..76163205 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,8 @@ After completing the flow design in Node-RED, please ensure that you click the b ### 1. Build +Using Rust 1.80 or later, run: + ```bash cargo build -r ``` @@ -99,7 +101,8 @@ py.test ## Configuration -Adjust various settings in the configuration file, such as port number, `flows.json` path, etc. Refer to [CONFIG.md](docs/CONFIG.md) for more information. +Adjust various settings and configuration, please execute `edgelinkd` with flags. +The flags available can be found when executing `edgelinkd --help`. ## Project Status