HTTP remote lists and Spam filter improvements

Cargo.toml

@@ -2,14 +2,14 @@
 resolver = "2"
 members = [
     "crates/main",
-    "crates/jmap",
-    "crates/jmap-proto",
-    "crates/imap",
-    "crates/imap-proto",
-    "crates/smtp",
-    "crates/managesieve",
-    "crates/pop3",
-    "crates/spam-filter",
+#    "crates/jmap",
+#    "crates/jmap-proto",
+#    "crates/imap",
+#    "crates/imap-proto",
+#    "crates/smtp",
+#    "crates/managesieve",
+#    "crates/pop3",
+#    "crates/spam-filter",
     "crates/nlp",
     "crates/store",
     "crates/directory",

crates/common/src/config/inner.rs

@@ -104,7 +104,6 @@ impl Data {
                 shard_amount,
             ),
             smtp_connectors: TlsConnectors::default(),
-            remote_lists: Default::default(),
             asn_geo_data: Default::default(),
         }
     }
@@ -121,7 +120,6 @@ impl Default for Data {
             blocked_ips_version: 0.into(),
             permissions: Default::default(),
             permissions_version: 0.into(),
-            remote_lists: Default::default(),
             jmap_id_gen: Default::default(),
             queue_id_gen: Default::default(),
             span_id_gen: Default::default(),

crates/common/src/config/scripts.rs

@@ -4,11 +4,7 @@
  * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
  */
 
-use std::{
-    collections::HashSet,
-    sync::Arc,
-    time::{Duration, Instant},
-};
+use std::{sync::Arc, time::Duration};
 
 use ahash::AHashMap;
 use sieve::{compiler::grammar::Capability, Compiler, Runtime, Sieve};
@@ -34,12 +30,6 @@ pub struct Scripting {
     pub untrusted_scripts: AHashMap<String, Arc<Sieve>>,
 }
 
-#[derive(Clone)]
-pub struct RemoteList {
-    pub entries: HashSet<String>,
-    pub expires: Instant,
-}
-
 impl Scripting {
     pub async fn parse(config: &mut Config, stores: &Stores) -> Self {
         // Parse untrusted compiler

crates/common/src/config/spamfilter.rs

@@ -11,7 +11,7 @@ use nlp::bayes::BayesClassifier;
 use tokio::net::lookup_host;
 use utils::{
     config::{utils::ParseValue, Config},
-    glob::{GlobMap, GlobSet},
+    glob::GlobMap,
 };
 
 use super::{if_block::IfBlock, tokenizer::TokenMap};
@@ -61,16 +61,8 @@ pub struct DnsBlConfig {
 
 #[derive(Debug, Clone, Default)]
 pub struct SpamFilterLists {
-    pub dmarc_allow: GlobSet,
-    pub spf_dkim_allow: GlobSet,
-    pub freemail_providers: GlobSet,
-    pub disposable_providers: GlobSet,
-    pub trusted_domains: GlobSet,
-    pub url_redirectors: GlobSet,
     pub file_extensions: GlobMap<FileExtension>,
     pub scores: GlobMap<SpamFilterAction<f64>>,
-    pub spamtraps: GlobSet,
-    pub remote: Vec<RemoteListConfig>,
 }
 
 #[derive(Debug, Clone)]
@@ -162,21 +154,6 @@ pub enum Location {
     Tcp,
 }
 
-#[derive(Debug, Clone)]
-pub struct RemoteListConfig {
-    pub id: String,
-    pub url: String,
-    pub retry: Duration,
-    pub refresh: Duration,
-    pub timeout: Duration,
-    pub max_size: usize,
-    pub max_entries: usize,
-    pub max_entry_size: usize,
-    pub format: RemoteListFormat,
-    pub scope: Element,
-    pub tag: String,
-}
-
 #[derive(Debug, Clone)]
 pub struct DnsBlServer {
     pub id: String,
@@ -185,16 +162,6 @@ pub struct DnsBlServer {
     pub tags: IfBlock,
 }
 
-#[derive(Debug, Clone)]
-pub enum RemoteListFormat {
-    List,
-    Csv {
-        column: u32,
-        separator: char,
-        skip_first: bool,
-    },
-}
-
 impl SpamFilterConfig {
     pub async fn parse(config: &mut Config) -> Self {
         SpamFilterConfig {
@@ -373,16 +340,8 @@ impl SpamFilterHeaderConfig {
 impl SpamFilterLists {
     pub fn parse(config: &mut Config) -> Self {
         let mut lists = SpamFilterLists {
-            dmarc_allow: GlobSet::default(),
-            spf_dkim_allow: GlobSet::default(),
-            freemail_providers: GlobSet::default(),
-            disposable_providers: GlobSet::default(),
-            trusted_domains: GlobSet::default(),
-            url_redirectors: GlobSet::default(),
-            spamtraps: GlobSet::default(),
             file_extensions: GlobMap::default(),
             scores: GlobMap::default(),
-            remote: Default::default(),
         };
 
         // Parse local lists
@@ -393,27 +352,6 @@ impl SpamFilterLists {
                 .filter(|(id, key)| !id.is_empty() && !key.is_empty())
             {
                 match id {
-                    "dmarc-allow" => {
-                        lists.dmarc_allow.insert(key);
-                    }
-                    "spf-dkim-allow" => {
-                        lists.spf_dkim_allow.insert(key);
-                    }
-                    "freemail-providers" => {
-                        lists.freemail_providers.insert(key);
-                    }
-                    "disposable-providers" => {
-                        lists.disposable_providers.insert(key);
-                    }
-                    "trusted-domains" => {
-                        lists.trusted_domains.insert(key);
-                    }
-                    "url-redirectors" => {
-                        lists.url_redirectors.insert(key);
-                    }
-                    "spam-traps" => {
-                        lists.spamtraps.insert(key);
-                    }
                     "scores" => {
                         let action = match value.to_lowercase().as_str() {
                             "reject" => SpamFilterAction::Reject,
@@ -470,104 +408,6 @@ impl SpamFilterLists {
             config.new_parse_error(key, error);
         }
 
-        // Parse remote lists
-        for id in config
-            .sub_keys("spam-filter.remote-list", ".url")
-            .map(|k| k.to_string())
-            .collect::<Vec<_>>()
-        {
-            let id_ = id.as_str();
-            if !config
-                .property_or_default(("spam-filter.remote-list", id_, "enable"), "true")
-                .unwrap_or(true)
-            {
-                continue;
-            }
-
-            let format = match config
-                .value_require(("spam-filter.remote-list", id_, "format"))
-                .unwrap_or_default()
-            {
-                "list" => RemoteListFormat::List,
-                "csv" => RemoteListFormat::Csv {
-                    column: config
-                        .property_require(("spam-filter.remote-list", id_, "column"))
-                        .unwrap_or(0),
-                    separator: config
-                        .property_or_default::<String>(
-                            ("spam-filter.remote-list", id_, "separator"),
-                            ",",
-                        )
-                        .unwrap_or_default()
-                        .chars()
-                        .next()
-                        .unwrap_or(','),
-                    skip_first: config
-                        .property_or_default::<bool>(
-                            ("spam-filter.remote-list", id_, "skip-first"),
-                            "false",
-                        )
-                        .unwrap_or(false),
-                },
-                other => {
-                    let message = format!("Invalid format: {other:?}");
-                    config.new_build_error(("spam-filter.remote-list", id_, "format"), message);
-                    continue;
-                }
-            };
-
-            lists.remote.push(RemoteListConfig {
-                url: config
-                    .value_require(("spam-filter.remote-list", id_, "url"))
-                    .unwrap_or_default()
-                    .to_string(),
-                retry: config
-                    .property_or_default::<Duration>(
-                        ("spam-filter.remote-list", id_, "retry"),
-                        "1h",
-                    )
-                    .unwrap_or(Duration::from_secs(3600)),
-                refresh: config
-                    .property_or_default::<Duration>(
-                        ("spam-filter.remote-list", id_, "refresh"),
-                        "12h",
-                    )
-                    .unwrap_or(Duration::from_secs(43200)),
-                timeout: config
-                    .property_or_default::<Duration>(
-                        ("spam-filter.remote-list", id_, "timeout"),
-                        "30s",
-                    )
-                    .unwrap_or(Duration::from_secs(30)),
-                max_size: config
-                    .property_or_default::<usize>(
-                        ("spam-filter.remote-list", id_, "limits.size"),
-                        "104857600",
-                    )
-                    .unwrap_or(104857600),
-                max_entries: config
-                    .property_or_default::<usize>(
-                        ("spam-filter.remote-list", id_, "limits.entries"),
-                        "100000",
-                    )
-                    .unwrap_or(100000),
-                max_entry_size: config
-                    .property_or_default::<usize>(
-                        ("spam-filter.remote-list", id_, "limits.entry-size"),
-                        "512",
-                    )
-                    .unwrap_or(512),
-                format,
-                scope: config
-                    .property_require(("spam-filter.remote-list", id_, "scope"))
-                    .unwrap_or_default(),
-                tag: config
-                    .property_require(("spam-filter.remote-list", id_, "tag"))
-                    .unwrap_or_else(|| format!("REMOTE_LIST_{}", id_.to_uppercase())),
-                id,
-            });
-        }
-
         lists
     }
 }

crates/common/src/core.rs

@@ -66,7 +66,11 @@ impl Server {
         })
     }
 
-    pub fn get_in_memory_store(&self, name: &str, session_id: u64) -> &InMemoryStore {
+    pub fn get_in_memory_store(&self, name: &str) -> Option<&InMemoryStore> {
+        self.core.storage.lookups.get(name)
+    }
+
+    pub fn get_in_memory_store_or_default(&self, name: &str, session_id: u64) -> &InMemoryStore {
         self.core.storage.lookups.get(name).unwrap_or_else(|| {
             if !name.is_empty() {
                 trc::event!(

crates/common/src/enterprise/mod.rs

@@ -26,9 +26,9 @@ use llm::AiApiConfig;
 use mail_parser::DateTime;
 use store::Store;
 use trc::{AddContext, EventType, MetricType};
-use utils::config::cron::SimpleCron;
+use utils::{config::cron::SimpleCron, HttpLimitResponse};
 
-use crate::{expr::Expression, manager::webadmin::Resource, Core, HttpLimitResponse, Server};
+use crate::{expr::Expression, manager::webadmin::Resource, Core, Server};
 
 #[derive(Clone)]
 pub struct Enterprise {

crates/common/src/expr/functions/asynch.rs

@@ -43,8 +43,8 @@ impl Server {
                 let store = params.next_as_string();
                 let key = params.next_as_string();
 
-                self.get_in_memory_store(store.as_ref(), session_id)
-                    .key_get::<VariableWrapper>(key.into_owned().into_bytes())
+                self.get_in_memory_store_or_default(store.as_ref(), session_id)
+                    .key_get::<VariableWrapper>(key)
                     .await
                     .map(|value| value.map(|v| v.into_inner()).unwrap_or_default())
                     .caused_by(trc::location!())
@@ -53,8 +53,8 @@ impl Server {
                 let store = params.next_as_string();
                 let key = params.next_as_string();
 
-                self.get_in_memory_store(store.as_ref(), session_id)
-                    .key_exists(key.into_owned().into_bytes())
+                self.get_in_memory_store_or_default(store.as_ref(), session_id)
+                    .key_exists(key)
                     .await
                     .caused_by(trc::location!())
                     .map(|v| v.into())
@@ -64,7 +64,7 @@ impl Server {
                 let key = params.next_as_string();
                 let value = params.next_as_string();
 
-                self.get_in_memory_store(store.as_ref(), session_id)
+                self.get_in_memory_store_or_default(store.as_ref(), session_id)
                     .key_set(KeyValue::new(
                         key.into_owned().into_bytes(),
                         value.into_owned().into_bytes(),
@@ -79,7 +79,7 @@ impl Server {
                 let key = params.next_as_string();
                 let value = params.next_as_integer();
 
-                self.get_in_memory_store(store.as_ref(), session_id)
+                self.get_in_memory_store_or_default(store.as_ref(), session_id)
                     .counter_incr(KeyValue::new(key.into_owned(), value))
                     .await
                     .map(Variable::Integer)
@@ -89,7 +89,7 @@ impl Server {
                 let store = params.next_as_string();
                 let key = params.next_as_string();
 
-                self.get_in_memory_store(store.as_ref(), session_id)
+                self.get_in_memory_store_or_default(store.as_ref(), session_id)
                     .counter_get(key.into_owned().into_bytes())
                     .await
                     .map(Variable::Integer)

crates/common/src/expr/functions/misc.rs

@@ -48,3 +48,16 @@ pub(crate) fn fn_ip_reverse_name(v: Vec<Variable>) -> Variable {
         .unwrap_or_default()
         .into()
 }
+
+pub(crate) fn fn_if_then(v: Vec<Variable>) -> Variable {
+    let mut v = v.into_iter();
+    let condition = v.next().unwrap();
+    let iff = v.next().unwrap();
+    let then = v.next().unwrap();
+
+    if condition.to_bool() {
+        iff
+    } else {
+        then
+    }
+}

crates/common/src/expr/functions/mod.rs

@@ -80,6 +80,7 @@ pub(crate) const FUNCTIONS: &[(&str, fn(Vec<Variable>) -> Variable, u32)] = &[
     ("split_n", text::fn_split_n, 3),
     ("split_words", text::fn_split_words, 1),
     ("hash", text::fn_hash, 2),
+    ("if_then", misc::fn_if_then, 3),
 ];
 
 pub const F_IS_LOCAL_DOMAIN: u32 = 0;