From 91245a4dd9ac2be20acb9ce53d2d59905e3513bb Mon Sep 17 00:00:00 2001 From: simplepad <35456194+simplepad@users.noreply.github.com> Date: Fri, 9 May 2025 17:05:49 +0300 Subject: [PATCH 001/305] feat(config): add `wildcard_matching` config option (#23011) * add relaxed_wildcart_matching config option * allow empty results for glob inputs if relaxed_wildcard_matching is enabled * optimize condition * fix fmt * fix clippy * remove option * fix fmt * Revert "remove option" This reverts commit 901cf08d46c349c7d16d2df710fc1078a99c9e89. * generate docs * add changelog fragment * fix typo * add tests * add config example to changelog * use enum instead of boolean * regenerate docs * update changelog --- ...1_add_relaxed_wildcard_matching.feature.md | 24 ++++++ lib/vector-core/src/config/global_options.rs | 29 +++++++ lib/vector-core/src/config/mod.rs | 2 +- lib/vector-lib/src/lib.rs | 2 +- src/config/compiler.rs | 8 +- src/config/graph.rs | 82 +++++++++++++++++-- src/config/mod.rs | 2 +- src/config/unit_test/mod.rs | 6 ++ website/cue/reference/base/configuration.cue | 14 ++++ 9 files changed, 156 insertions(+), 13 deletions(-) create mode 100644 changelog.d/23011_add_relaxed_wildcard_matching.feature.md diff --git a/changelog.d/23011_add_relaxed_wildcard_matching.feature.md b/changelog.d/23011_add_relaxed_wildcard_matching.feature.md new file mode 100644 index 0000000000000..adaf44fd03177 --- /dev/null +++ b/changelog.d/23011_add_relaxed_wildcard_matching.feature.md @@ -0,0 +1,24 @@ +Added `wildcard_matching` global config option to set wildcard matching mode for inputs. Relaxed mode allows configurations with wildcards that do not match any inputs to be accepted without causing an error. + +Example config: + +```yaml +wildcard_matching: relaxed + +sources: + stdin: + type: stdin + +# note - no transforms + +sinks: + stdout: + type: console + encoding: + codec: json + inputs: + - "runtime-added-transform-*" + +``` + +authors: simplepad diff --git a/lib/vector-core/src/config/global_options.rs b/lib/vector-core/src/config/global_options.rs index 80e7316f023bc..121a5bcd2c0a5 100644 --- a/lib/vector-core/src/config/global_options.rs +++ b/lib/vector-core/src/config/global_options.rs @@ -31,6 +31,19 @@ pub(crate) enum DataDirError { }, } +/// Specifies the wildcard matching mode, relaxed allows configurations where wildcard doesn not match any existing inputs +#[configurable_component] +#[derive(Clone, Debug, Copy, PartialEq, Eq, Default)] +#[serde(rename_all = "lowercase")] +pub enum WildcardMatching { + /// Strict matching (must match at least one existing input) + #[default] + Strict, + + /// Relaxed matching (must match 0 or more inputs) + Relaxed, +} + /// Global configuration options. // // If this is modified, make sure those changes are reflected in the `ConfigBuilder::append` @@ -48,6 +61,14 @@ pub struct GlobalOptions { #[configurable(metadata(docs::common = false))] pub data_dir: Option, + /// Set wildcard matching mode for inputs + /// + /// Setting this to "relaxed" allows configurations with wildcards that do not match any inputs + /// to be accepted without causing an error. + #[serde(skip_serializing_if = "crate::serde::is_default")] + #[configurable(metadata(docs::common = false, docs::required = false))] + pub wildcard_matching: Option, + /// Default log schema for all events. /// /// This is used if a component does not have its own specific log schema. All events use a log @@ -182,6 +203,13 @@ impl GlobalOptions { pub fn merge(&self, with: Self) -> Result> { let mut errors = Vec::new(); + if conflicts( + self.wildcard_matching.as_ref(), + with.wildcard_matching.as_ref(), + ) { + errors.push("conflicting values for 'wildcard_matching' found".to_owned()); + } + if conflicts(self.proxy.http.as_ref(), with.proxy.http.as_ref()) { errors.push("conflicting values for 'proxy.http' found".to_owned()); } @@ -250,6 +278,7 @@ impl GlobalOptions { if errors.is_empty() { Ok(Self { data_dir, + wildcard_matching: self.wildcard_matching.or(with.wildcard_matching), log_schema, telemetry, acknowledgements: self.acknowledgements.merge_default(&with.acknowledgements), diff --git a/lib/vector-core/src/config/mod.rs b/lib/vector-core/src/config/mod.rs index 074b8c5fa3cab..7b47e3dbbe94d 100644 --- a/lib/vector-core/src/config/mod.rs +++ b/lib/vector-core/src/config/mod.rs @@ -13,7 +13,7 @@ pub mod proxy; mod telemetry; use crate::event::LogEvent; -pub use global_options::GlobalOptions; +pub use global_options::{GlobalOptions, WildcardMatching}; pub use log_schema::{init_log_schema, log_schema, LogSchema}; use lookup::{lookup_v2::ValuePath, path, PathPrefix}; pub use output_id::OutputId; diff --git a/lib/vector-lib/src/lib.rs b/lib/vector-lib/src/lib.rs index f7bda363cc6b7..3d2628c90e80d 100644 --- a/lib/vector-lib/src/lib.rs +++ b/lib/vector-lib/src/lib.rs @@ -34,7 +34,7 @@ pub mod config { clone_input_definitions, init_log_schema, init_telemetry, log_schema, proxy, telemetry, AcknowledgementsConfig, DataType, GlobalOptions, Input, LegacyKey, LogNamespace, LogSchema, OutputId, SourceAcknowledgementsConfig, SourceOutput, Tags, Telemetry, TransformOutput, - MEMORY_BUFFER_DEFAULT_MAX_EVENTS, + WildcardMatching, MEMORY_BUFFER_DEFAULT_MAX_EVENTS, }; } diff --git a/src/config/compiler.rs b/src/config/compiler.rs index 686572e6e56e3..bffb51bcaa93a 100644 --- a/src/config/compiler.rs +++ b/src/config/compiler.rs @@ -71,7 +71,13 @@ pub fn compile(mut builder: ConfigBuilder) -> Result<(Config, Vec), Vec< ) .collect::>(); - let graph = match Graph::new(&sources_and_table_sources, &transforms, &all_sinks, schema) { + let graph = match Graph::new( + &sources_and_table_sources, + &transforms, + &all_sinks, + schema, + global.wildcard_matching.unwrap_or_default(), + ) { Ok(graph) => graph, Err(graph_errors) => { errors.extend(graph_errors); diff --git a/src/config/graph.rs b/src/config/graph.rs index 9ed1af6ff17fe..3943aefead147 100644 --- a/src/config/graph.rs +++ b/src/config/graph.rs @@ -1,6 +1,6 @@ use super::{ schema, ComponentKey, DataType, OutputId, SinkOuter, SourceOuter, SourceOutput, TransformOuter, - TransformOutput, + TransformOutput, WildcardMatching, }; use indexmap::{set::IndexSet, IndexMap}; use std::collections::{HashMap, HashSet, VecDeque}; @@ -65,8 +65,9 @@ impl Graph { transforms: &IndexMap>, sinks: &IndexMap>, schema: schema::Options, + wildcard_matching: WildcardMatching, ) -> Result> { - Self::new_inner(sources, transforms, sinks, false, schema) + Self::new_inner(sources, transforms, sinks, false, schema, wildcard_matching) } pub fn new_unchecked( @@ -74,8 +75,10 @@ impl Graph { transforms: &IndexMap>, sinks: &IndexMap>, schema: schema::Options, + wildcard_matching: WildcardMatching, ) -> Self { - Self::new_inner(sources, transforms, sinks, true, schema).expect("errors ignored") + Self::new_inner(sources, transforms, sinks, true, schema, wildcard_matching) + .expect("errors ignored") } fn new_inner( @@ -84,6 +87,7 @@ impl Graph { sinks: &IndexMap>, ignore_errors: bool, schema: schema::Options, + wildcard_matching: WildcardMatching, ) -> Result> { let mut graph = Graph::default(); let mut errors = Vec::new(); @@ -127,7 +131,7 @@ impl Graph { for (id, config) in transforms.iter() { for input in config.inputs.iter() { - if let Err(e) = graph.add_input(input, id, &available_inputs) { + if let Err(e) = graph.add_input(input, id, &available_inputs, wildcard_matching) { errors.push(e); } } @@ -135,7 +139,7 @@ impl Graph { for (id, config) in sinks { for input in config.inputs.iter() { - if let Err(e) = graph.add_input(input, id, &available_inputs) { + if let Err(e) = graph.add_input(input, id, &available_inputs, wildcard_matching) { errors.push(e); } } @@ -153,6 +157,7 @@ impl Graph { from: &str, to: &ComponentKey, available_inputs: &HashMap, + wildcard_matching: WildcardMatching, ) -> Result<(), String> { if let Some(output_id) = available_inputs.get(from) { self.edges.push(Edge { @@ -166,6 +171,18 @@ impl Graph { Some(Node::Sink { .. }) => "sink", _ => panic!("only transforms and sinks have inputs"), }; + // allow empty result if relaxed wildcard matching is enabled + match wildcard_matching { + WildcardMatching::Relaxed => { + // using value != glob::Pattern::escape(value) to check if value is a glob + // TODO: replace with proper check when https://github.com/rust-lang/glob/issues/72 is resolved + if from != glob::Pattern::escape(from) { + info!("Input \"{from}\" for {output_type} \"{to}\" didn’t match any components, but this was ignored because `relaxed_wildcard_matching` is enabled."); + return Ok(()); + } + } + WildcardMatching::Strict => {} + } info!( "Available components:\n{}", self.nodes @@ -472,9 +489,14 @@ mod test { } } - fn test_add_input(&mut self, node: &str, input: &str) -> Result<(), String> { + fn test_add_input( + &mut self, + node: &str, + input: &str, + wildcard_matching: WildcardMatching, + ) -> Result<(), String> { let available_inputs = self.input_map().unwrap(); - self.add_input(input, &node.into(), &available_inputs) + self.add_input(input, &node.into(), &available_inputs, wildcard_matching) } } @@ -655,14 +677,22 @@ mod test { // make sure we're good with dotted paths assert_eq!( Ok(()), - graph.test_add_input("errored_log_sink", "log_to_log.errors") + graph.test_add_input( + "errored_log_sink", + "log_to_log.errors", + WildcardMatching::Strict + ) ); // make sure that we're not cool with an unknown dotted path let expected = "Input \"log_to_log.not_errors\" for sink \"bad_log_sink\" doesn't match any components.".to_string(); assert_eq!( Err(expected), - graph.test_add_input("bad_log_sink", "log_to_log.not_errors") + graph.test_add_input( + "bad_log_sink", + "log_to_log.not_errors", + WildcardMatching::Strict + ) ); } @@ -745,6 +775,40 @@ mod test { ); } + #[test] + fn wildcard_matching() { + let mut graph = Graph::default(); + graph.add_source("log_source", DataType::Log); + + // don't add inputs to these yet since they're not validated via these helpers + graph.add_sink("sink", DataType::Log, vec![]); + + // make sure we're not good with non existing inputs with relaxed wildcard matching disabled + let wildcard_matching = WildcardMatching::Strict; + let expected = + "Input \"bad_source-*\" for sink \"sink\" doesn't match any components.".to_string(); + assert_eq!( + Err(expected), + graph.test_add_input("sink", "bad_source-*", wildcard_matching) + ); + + // make sure we're good with non existing inputs with relaxed wildcard matching enabled + let wildcard_matching = WildcardMatching::Relaxed; + assert_eq!( + Ok(()), + graph.test_add_input("sink", "bad_source-*", wildcard_matching) + ); + + // make sure we're not good with non existing inputs that are not wildcards even when relaxed wildcard matching is enabled + let wildcard_matching = WildcardMatching::Relaxed; + let expected = + "Input \"bad_source-1\" for sink \"sink\" doesn't match any components.".to_string(); + assert_eq!( + Err(expected), + graph.test_add_input("sink", "bad_source-1", wildcard_matching) + ); + } + #[test] fn paths_to_sink_simple() { let mut graph = Graph::default(); diff --git a/src/config/mod.rs b/src/config/mod.rs index 6478203229475..d42d87c9753af 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -21,7 +21,7 @@ use serde::Serialize; use vector_config::configurable_component; pub use vector_lib::config::{ AcknowledgementsConfig, DataType, GlobalOptions, Input, LogNamespace, - SourceAcknowledgementsConfig, SourceOutput, TransformOutput, + SourceAcknowledgementsConfig, SourceOutput, TransformOutput, WildcardMatching, }; pub use vector_lib::configurable::component::{ GenerateConfig, SinkDescription, TransformDescription, diff --git a/src/config/unit_test/mod.rs b/src/config/unit_test/mod.rs index deafeff02cfcb..4e5b12b658c78 100644 --- a/src/config/unit_test/mod.rs +++ b/src/config/unit_test/mod.rs @@ -385,6 +385,10 @@ async fn build_unit_test( &transform_only_config.transforms, &transform_only_config.sinks, transform_only_config.schema, + transform_only_config + .global + .wildcard_matching + .unwrap_or_default(), ); let test = test.resolve_outputs(&transform_only_graph)?; @@ -401,6 +405,7 @@ async fn build_unit_test( &config_builder.transforms, &config_builder.sinks, config_builder.schema, + config_builder.global.wildcard_matching.unwrap_or_default(), ); let mut valid_components = get_relevant_test_components( @@ -432,6 +437,7 @@ async fn build_unit_test( &config_builder.transforms, &config_builder.sinks, config_builder.schema, + config_builder.global.wildcard_matching.unwrap_or_default(), ); let valid_inputs = graph.input_map()?; for (_, transform) in config_builder.transforms.iter_mut() { diff --git a/website/cue/reference/base/configuration.cue b/website/cue/reference/base/configuration.cue index 4bd38192b11b7..41650b84adf79 100644 --- a/website/cue/reference/base/configuration.cue +++ b/website/cue/reference/base/configuration.cue @@ -901,4 +901,18 @@ base: configuration: configuration: { required: false type: string: examples: ["local", "America/New_York", "EST5EDT"] } + wildcard_matching: { + common: false + description: """ + Set wildcard matching mode for inputs + + Setting this to "relaxed" allows configurations with wildcards that do not match any inputs + to be accepted without causing an error. + """ + required: false + type: string: enum: { + relaxed: "Relaxed matching (must match 0 or more inputs)" + strict: "Strict matching (must match at least one existing input)" + } + } } From 9129d95c56ad530989c238dd0d380de6a0c95ca8 Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Tue, 13 May 2025 07:31:55 +1200 Subject: [PATCH 002/305] docs(sinks): Remove slash from max_bytes text (#23035) Signed-off-by: Jed Laundry --- src/sinks/util/batch.rs | 2 +- website/cue/reference/components/sinks/base/appsignal.cue | 2 +- .../cue/reference/components/sinks/base/aws_cloudwatch_logs.cue | 2 +- .../reference/components/sinks/base/aws_cloudwatch_metrics.cue | 2 +- .../reference/components/sinks/base/aws_kinesis_firehose.cue | 2 +- .../cue/reference/components/sinks/base/aws_kinesis_streams.cue | 2 +- website/cue/reference/components/sinks/base/aws_s3.cue | 2 +- website/cue/reference/components/sinks/base/axiom.cue | 2 +- website/cue/reference/components/sinks/base/azure_blob.cue | 2 +- .../cue/reference/components/sinks/base/azure_monitor_logs.cue | 2 +- website/cue/reference/components/sinks/base/clickhouse.cue | 2 +- website/cue/reference/components/sinks/base/databend.cue | 2 +- website/cue/reference/components/sinks/base/datadog_logs.cue | 2 +- website/cue/reference/components/sinks/base/datadog_metrics.cue | 2 +- website/cue/reference/components/sinks/base/datadog_traces.cue | 2 +- website/cue/reference/components/sinks/base/elasticsearch.cue | 2 +- .../components/sinks/base/gcp_chronicle_unstructured.cue | 2 +- .../cue/reference/components/sinks/base/gcp_cloud_storage.cue | 2 +- website/cue/reference/components/sinks/base/gcp_pubsub.cue | 2 +- .../reference/components/sinks/base/gcp_stackdriver_logs.cue | 2 +- .../reference/components/sinks/base/gcp_stackdriver_metrics.cue | 2 +- website/cue/reference/components/sinks/base/greptimedb.cue | 2 +- website/cue/reference/components/sinks/base/greptimedb_logs.cue | 2 +- .../cue/reference/components/sinks/base/greptimedb_metrics.cue | 2 +- website/cue/reference/components/sinks/base/honeycomb.cue | 2 +- website/cue/reference/components/sinks/base/http.cue | 2 +- website/cue/reference/components/sinks/base/humio_logs.cue | 2 +- website/cue/reference/components/sinks/base/humio_metrics.cue | 2 +- website/cue/reference/components/sinks/base/influxdb_logs.cue | 2 +- .../cue/reference/components/sinks/base/influxdb_metrics.cue | 2 +- website/cue/reference/components/sinks/base/kafka.cue | 2 +- website/cue/reference/components/sinks/base/keep.cue | 2 +- website/cue/reference/components/sinks/base/logdna.cue | 2 +- website/cue/reference/components/sinks/base/loki.cue | 2 +- website/cue/reference/components/sinks/base/mezmo.cue | 2 +- website/cue/reference/components/sinks/base/new_relic.cue | 2 +- website/cue/reference/components/sinks/base/opentelemetry.cue | 2 +- website/cue/reference/components/sinks/base/postgres.cue | 2 +- .../reference/components/sinks/base/prometheus_remote_write.cue | 2 +- website/cue/reference/components/sinks/base/redis.cue | 2 +- website/cue/reference/components/sinks/base/sematext_logs.cue | 2 +- .../cue/reference/components/sinks/base/sematext_metrics.cue | 2 +- website/cue/reference/components/sinks/base/splunk_hec_logs.cue | 2 +- .../cue/reference/components/sinks/base/splunk_hec_metrics.cue | 2 +- website/cue/reference/components/sinks/base/statsd.cue | 2 +- website/cue/reference/components/sinks/base/vector.cue | 2 +- website/cue/reference/components/sinks/base/webhdfs.cue | 2 +- 47 files changed, 47 insertions(+), 47 deletions(-) diff --git a/src/sinks/util/batch.rs b/src/sinks/util/batch.rs index b6fdb77463704..e192360b3874a 100644 --- a/src/sinks/util/batch.rs +++ b/src/sinks/util/batch.rs @@ -101,7 +101,7 @@ where /// The maximum size of a batch that is processed by a sink. /// /// This is based on the uncompressed size of the batched events, before they are - /// serialized/compressed. + /// serialized or compressed. #[serde(default = "default_max_bytes::")] #[configurable(metadata(docs::type_unit = "bytes"))] pub max_bytes: Option, diff --git a/website/cue/reference/components/sinks/base/appsignal.cue b/website/cue/reference/components/sinks/base/appsignal.cue index f786a1fa97acb..30ce8edf72077 100644 --- a/website/cue/reference/components/sinks/base/appsignal.cue +++ b/website/cue/reference/components/sinks/base/appsignal.cue @@ -36,7 +36,7 @@ base: components: sinks: appsignal: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue index 7b53d05fa3455..e0f57c6ce0730 100644 --- a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue +++ b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue @@ -150,7 +150,7 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue b/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue index c2ced75b95861..2f587a9ab0ae9 100644 --- a/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue +++ b/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue @@ -150,7 +150,7 @@ base: components: sinks: aws_cloudwatch_metrics: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: unit: "bytes" diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue index b3319428648ba..66fc0c718bf69 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue @@ -150,7 +150,7 @@ base: components: sinks: aws_kinesis_firehose: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue index fb53b59f1b3d1..fe0c944ef400d 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue @@ -150,7 +150,7 @@ base: components: sinks: aws_kinesis_streams: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/aws_s3.cue b/website/cue/reference/components/sinks/base/aws_s3.cue index c7ebd5f2fe1ba..625daa2730ac5 100644 --- a/website/cue/reference/components/sinks/base/aws_s3.cue +++ b/website/cue/reference/components/sinks/base/aws_s3.cue @@ -225,7 +225,7 @@ base: components: sinks: aws_s3: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/axiom.cue b/website/cue/reference/components/sinks/base/axiom.cue index 789be7e72085e..ce0f396bfac30 100644 --- a/website/cue/reference/components/sinks/base/axiom.cue +++ b/website/cue/reference/components/sinks/base/axiom.cue @@ -30,7 +30,7 @@ base: components: sinks: axiom: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/azure_blob.cue b/website/cue/reference/components/sinks/base/azure_blob.cue index 08b9b6d6aa9c5..0bd8b0f494979 100644 --- a/website/cue/reference/components/sinks/base/azure_blob.cue +++ b/website/cue/reference/components/sinks/base/azure_blob.cue @@ -36,7 +36,7 @@ base: components: sinks: azure_blob: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/azure_monitor_logs.cue b/website/cue/reference/components/sinks/base/azure_monitor_logs.cue index 18a81b5f0d122..bce585a0c5325 100644 --- a/website/cue/reference/components/sinks/base/azure_monitor_logs.cue +++ b/website/cue/reference/components/sinks/base/azure_monitor_logs.cue @@ -45,7 +45,7 @@ base: components: sinks: azure_monitor_logs: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/clickhouse.cue b/website/cue/reference/components/sinks/base/clickhouse.cue index e93b7e417bf07..c472c8736c682 100644 --- a/website/cue/reference/components/sinks/base/clickhouse.cue +++ b/website/cue/reference/components/sinks/base/clickhouse.cue @@ -205,7 +205,7 @@ base: components: sinks: clickhouse: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/databend.cue b/website/cue/reference/components/sinks/base/databend.cue index 54033a7968da8..0fa20d0dd1d62 100644 --- a/website/cue/reference/components/sinks/base/databend.cue +++ b/website/cue/reference/components/sinks/base/databend.cue @@ -200,7 +200,7 @@ base: components: sinks: databend: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/datadog_logs.cue b/website/cue/reference/components/sinks/base/datadog_logs.cue index 5169c434610b8..7f9853614b4d9 100644 --- a/website/cue/reference/components/sinks/base/datadog_logs.cue +++ b/website/cue/reference/components/sinks/base/datadog_logs.cue @@ -36,7 +36,7 @@ base: components: sinks: datadog_logs: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/datadog_metrics.cue b/website/cue/reference/components/sinks/base/datadog_metrics.cue index 1b4f0b3f68e53..8acd1569f57f5 100644 --- a/website/cue/reference/components/sinks/base/datadog_metrics.cue +++ b/website/cue/reference/components/sinks/base/datadog_metrics.cue @@ -36,7 +36,7 @@ base: components: sinks: datadog_metrics: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: unit: "bytes" diff --git a/website/cue/reference/components/sinks/base/datadog_traces.cue b/website/cue/reference/components/sinks/base/datadog_traces.cue index 05f0ab9b6d272..c60bd879e881e 100644 --- a/website/cue/reference/components/sinks/base/datadog_traces.cue +++ b/website/cue/reference/components/sinks/base/datadog_traces.cue @@ -36,7 +36,7 @@ base: components: sinks: datadog_traces: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/elasticsearch.cue b/website/cue/reference/components/sinks/base/elasticsearch.cue index c2340f11a1340..aa29493a59583 100644 --- a/website/cue/reference/components/sinks/base/elasticsearch.cue +++ b/website/cue/reference/components/sinks/base/elasticsearch.cue @@ -231,7 +231,7 @@ base: components: sinks: elasticsearch: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue b/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue index e99f99c3b23c2..1b9edc8daea52 100644 --- a/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue +++ b/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue @@ -52,7 +52,7 @@ base: components: sinks: gcp_chronicle_unstructured: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue b/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue index aee9855b7fcf5..45e175ee58aee 100644 --- a/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue +++ b/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue @@ -107,7 +107,7 @@ base: components: sinks: gcp_cloud_storage: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/gcp_pubsub.cue b/website/cue/reference/components/sinks/base/gcp_pubsub.cue index 2537ce1867417..17e23079783ae 100644 --- a/website/cue/reference/components/sinks/base/gcp_pubsub.cue +++ b/website/cue/reference/components/sinks/base/gcp_pubsub.cue @@ -52,7 +52,7 @@ base: components: sinks: gcp_pubsub: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/gcp_stackdriver_logs.cue b/website/cue/reference/components/sinks/base/gcp_stackdriver_logs.cue index a2a1aa714bc30..e313a849aa81f 100644 --- a/website/cue/reference/components/sinks/base/gcp_stackdriver_logs.cue +++ b/website/cue/reference/components/sinks/base/gcp_stackdriver_logs.cue @@ -52,7 +52,7 @@ base: components: sinks: gcp_stackdriver_logs: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/gcp_stackdriver_metrics.cue b/website/cue/reference/components/sinks/base/gcp_stackdriver_metrics.cue index 63157aa821a65..1e1703d6b0820 100644 --- a/website/cue/reference/components/sinks/base/gcp_stackdriver_metrics.cue +++ b/website/cue/reference/components/sinks/base/gcp_stackdriver_metrics.cue @@ -52,7 +52,7 @@ base: components: sinks: gcp_stackdriver_metrics: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: unit: "bytes" diff --git a/website/cue/reference/components/sinks/base/greptimedb.cue b/website/cue/reference/components/sinks/base/greptimedb.cue index 42f8739e7422b..29cd94ff45e41 100644 --- a/website/cue/reference/components/sinks/base/greptimedb.cue +++ b/website/cue/reference/components/sinks/base/greptimedb.cue @@ -36,7 +36,7 @@ base: components: sinks: greptimedb: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: unit: "bytes" diff --git a/website/cue/reference/components/sinks/base/greptimedb_logs.cue b/website/cue/reference/components/sinks/base/greptimedb_logs.cue index 5b55f78f86265..777f9890bab87 100644 --- a/website/cue/reference/components/sinks/base/greptimedb_logs.cue +++ b/website/cue/reference/components/sinks/base/greptimedb_logs.cue @@ -36,7 +36,7 @@ base: components: sinks: greptimedb_logs: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: unit: "bytes" diff --git a/website/cue/reference/components/sinks/base/greptimedb_metrics.cue b/website/cue/reference/components/sinks/base/greptimedb_metrics.cue index 6444afceb9aa9..be9d9ae854a7b 100644 --- a/website/cue/reference/components/sinks/base/greptimedb_metrics.cue +++ b/website/cue/reference/components/sinks/base/greptimedb_metrics.cue @@ -36,7 +36,7 @@ base: components: sinks: greptimedb_metrics: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: unit: "bytes" diff --git a/website/cue/reference/components/sinks/base/honeycomb.cue b/website/cue/reference/components/sinks/base/honeycomb.cue index d34c539436672..0834f68ac4aea 100644 --- a/website/cue/reference/components/sinks/base/honeycomb.cue +++ b/website/cue/reference/components/sinks/base/honeycomb.cue @@ -41,7 +41,7 @@ base: components: sinks: honeycomb: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/http.cue b/website/cue/reference/components/sinks/base/http.cue index 402db85cfc8da..5cf6181e34e96 100644 --- a/website/cue/reference/components/sinks/base/http.cue +++ b/website/cue/reference/components/sinks/base/http.cue @@ -205,7 +205,7 @@ base: components: sinks: http: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/humio_logs.cue b/website/cue/reference/components/sinks/base/humio_logs.cue index 3924d1c44735f..153dfa5324fdc 100644 --- a/website/cue/reference/components/sinks/base/humio_logs.cue +++ b/website/cue/reference/components/sinks/base/humio_logs.cue @@ -36,7 +36,7 @@ base: components: sinks: humio_logs: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/humio_metrics.cue b/website/cue/reference/components/sinks/base/humio_metrics.cue index 89ff9ec8d5033..fd0e65f6daa6b 100644 --- a/website/cue/reference/components/sinks/base/humio_metrics.cue +++ b/website/cue/reference/components/sinks/base/humio_metrics.cue @@ -36,7 +36,7 @@ base: components: sinks: humio_metrics: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/influxdb_logs.cue b/website/cue/reference/components/sinks/base/influxdb_logs.cue index 1667d06b84f83..f8885c040730f 100644 --- a/website/cue/reference/components/sinks/base/influxdb_logs.cue +++ b/website/cue/reference/components/sinks/base/influxdb_logs.cue @@ -36,7 +36,7 @@ base: components: sinks: influxdb_logs: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/influxdb_metrics.cue b/website/cue/reference/components/sinks/base/influxdb_metrics.cue index 4f09582867523..093b79ba83e41 100644 --- a/website/cue/reference/components/sinks/base/influxdb_metrics.cue +++ b/website/cue/reference/components/sinks/base/influxdb_metrics.cue @@ -36,7 +36,7 @@ base: components: sinks: influxdb_metrics: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: unit: "bytes" diff --git a/website/cue/reference/components/sinks/base/kafka.cue b/website/cue/reference/components/sinks/base/kafka.cue index d43a10408c8b1..1f88e96d23741 100644 --- a/website/cue/reference/components/sinks/base/kafka.cue +++ b/website/cue/reference/components/sinks/base/kafka.cue @@ -36,7 +36,7 @@ base: components: sinks: kafka: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: unit: "bytes" diff --git a/website/cue/reference/components/sinks/base/keep.cue b/website/cue/reference/components/sinks/base/keep.cue index d7dfd13b4cfde..3cae9b1b1cbfe 100644 --- a/website/cue/reference/components/sinks/base/keep.cue +++ b/website/cue/reference/components/sinks/base/keep.cue @@ -41,7 +41,7 @@ base: components: sinks: keep: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/logdna.cue b/website/cue/reference/components/sinks/base/logdna.cue index 075465a7916dc..63f3d81553417 100644 --- a/website/cue/reference/components/sinks/base/logdna.cue +++ b/website/cue/reference/components/sinks/base/logdna.cue @@ -41,7 +41,7 @@ base: components: sinks: logdna: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/loki.cue b/website/cue/reference/components/sinks/base/loki.cue index e846d91dfb300..0641330c99865 100644 --- a/website/cue/reference/components/sinks/base/loki.cue +++ b/website/cue/reference/components/sinks/base/loki.cue @@ -205,7 +205,7 @@ base: components: sinks: loki: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/mezmo.cue b/website/cue/reference/components/sinks/base/mezmo.cue index db22b664e657a..8a8eefb22732f 100644 --- a/website/cue/reference/components/sinks/base/mezmo.cue +++ b/website/cue/reference/components/sinks/base/mezmo.cue @@ -41,7 +41,7 @@ base: components: sinks: mezmo: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/new_relic.cue b/website/cue/reference/components/sinks/base/new_relic.cue index e1e51404f4c2b..7e3895b02cd27 100644 --- a/website/cue/reference/components/sinks/base/new_relic.cue +++ b/website/cue/reference/components/sinks/base/new_relic.cue @@ -50,7 +50,7 @@ base: components: sinks: new_relic: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/opentelemetry.cue b/website/cue/reference/components/sinks/base/opentelemetry.cue index de87314957157..4d7a2d4b5228d 100644 --- a/website/cue/reference/components/sinks/base/opentelemetry.cue +++ b/website/cue/reference/components/sinks/base/opentelemetry.cue @@ -208,7 +208,7 @@ base: components: sinks: opentelemetry: configuration: protocol: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/postgres.cue b/website/cue/reference/components/sinks/base/postgres.cue index bc20d87ae14f2..d3fa9af9d9e52 100644 --- a/website/cue/reference/components/sinks/base/postgres.cue +++ b/website/cue/reference/components/sinks/base/postgres.cue @@ -47,7 +47,7 @@ base: components: sinks: postgres: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/prometheus_remote_write.cue b/website/cue/reference/components/sinks/base/prometheus_remote_write.cue index 4ca726cbd445b..62b12a901aa80 100644 --- a/website/cue/reference/components/sinks/base/prometheus_remote_write.cue +++ b/website/cue/reference/components/sinks/base/prometheus_remote_write.cue @@ -216,7 +216,7 @@ base: components: sinks: prometheus_remote_write: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: unit: "bytes" diff --git a/website/cue/reference/components/sinks/base/redis.cue b/website/cue/reference/components/sinks/base/redis.cue index f958ee2f8c53d..e93567ba823fd 100644 --- a/website/cue/reference/components/sinks/base/redis.cue +++ b/website/cue/reference/components/sinks/base/redis.cue @@ -36,7 +36,7 @@ base: components: sinks: redis: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: unit: "bytes" diff --git a/website/cue/reference/components/sinks/base/sematext_logs.cue b/website/cue/reference/components/sinks/base/sematext_logs.cue index 502dbeda2a449..45e2e2962fc2c 100644 --- a/website/cue/reference/components/sinks/base/sematext_logs.cue +++ b/website/cue/reference/components/sinks/base/sematext_logs.cue @@ -36,7 +36,7 @@ base: components: sinks: sematext_logs: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/sematext_metrics.cue b/website/cue/reference/components/sinks/base/sematext_metrics.cue index 9eb884a6707e3..c5d5571f2d4d2 100644 --- a/website/cue/reference/components/sinks/base/sematext_metrics.cue +++ b/website/cue/reference/components/sinks/base/sematext_metrics.cue @@ -36,7 +36,7 @@ base: components: sinks: sematext_metrics: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: unit: "bytes" diff --git a/website/cue/reference/components/sinks/base/splunk_hec_logs.cue b/website/cue/reference/components/sinks/base/splunk_hec_logs.cue index 5fc88d51533fc..0c8187c067592 100644 --- a/website/cue/reference/components/sinks/base/splunk_hec_logs.cue +++ b/website/cue/reference/components/sinks/base/splunk_hec_logs.cue @@ -77,7 +77,7 @@ base: components: sinks: splunk_hec_logs: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/splunk_hec_metrics.cue b/website/cue/reference/components/sinks/base/splunk_hec_metrics.cue index 603e2dedb62d5..80376f7cfe0c0 100644 --- a/website/cue/reference/components/sinks/base/splunk_hec_metrics.cue +++ b/website/cue/reference/components/sinks/base/splunk_hec_metrics.cue @@ -63,7 +63,7 @@ base: components: sinks: splunk_hec_metrics: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/statsd.cue b/website/cue/reference/components/sinks/base/statsd.cue index 485984af972ca..f411bf70f4df3 100644 --- a/website/cue/reference/components/sinks/base/statsd.cue +++ b/website/cue/reference/components/sinks/base/statsd.cue @@ -48,7 +48,7 @@ base: components: sinks: statsd: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { diff --git a/website/cue/reference/components/sinks/base/vector.cue b/website/cue/reference/components/sinks/base/vector.cue index 85dd15172cd23..50c254bf90416 100644 --- a/website/cue/reference/components/sinks/base/vector.cue +++ b/website/cue/reference/components/sinks/base/vector.cue @@ -47,7 +47,7 @@ base: components: sinks: vector: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: unit: "bytes" diff --git a/website/cue/reference/components/sinks/base/webhdfs.cue b/website/cue/reference/components/sinks/base/webhdfs.cue index 4401e42da0b5c..a011bc24262e5 100644 --- a/website/cue/reference/components/sinks/base/webhdfs.cue +++ b/website/cue/reference/components/sinks/base/webhdfs.cue @@ -36,7 +36,7 @@ base: components: sinks: webhdfs: configuration: { The maximum size of a batch that is processed by a sink. This is based on the uncompressed size of the batched events, before they are - serialized/compressed. + serialized or compressed. """ required: false type: uint: { From a39d60acbfb60a3b740c3da45a3fadfc3e1052a4 Mon Sep 17 00:00:00 2001 From: StormStake <59213963+StormStake@users.noreply.github.com> Date: Wed, 14 May 2025 12:42:31 -0500 Subject: [PATCH 003/305] feat(new source): Initial MQTT Source, #19931 (#22752) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * wip * add mqtt source cue reference * add changelog * Update src/sources/mqtt/config.rs Co-authored-by: Stephen Wakely * rustfmt * honor shutdown signa * git rm .vscode/settings.json * add deny warnings back * use new_maybe_logs * fix type, changelog, rename bad creds * add fowllow up github issue * common config WIP * Consolidated MQTT error types * More deduplication in MQTT error types * Deduplicated MQTT connector type * Formatting fixes for tests * Fixed docs tests * Fixed metrics and integration tests for MQTT * Added to changelog authorship * Fixed cue docs formating * Formatting fix on errors * Added topic metadata --------- Co-authored-by: David Mládek Co-authored-by: Stephen Wakely Co-authored-by: Pavlos Rontidis --- Cargo.toml | 4 +- changelog.d/19931_add_mqtt_source.feature.md | 3 + scripts/integration/mqtt/test.yaml | 1 + src/common/mod.rs | 4 + src/common/mqtt.rs | 123 ++++ src/sinks/mqtt/config.rs | 77 +-- src/sinks/mqtt/integration_tests.rs | 10 +- src/sinks/mqtt/sink.rs | 43 +- src/sources/mod.rs | 2 + src/sources/mqtt/config.rs | 175 ++++++ src/sources/mqtt/integration_tests.rs | 117 ++++ src/sources/mqtt/mod.rs | 5 + src/sources/mqtt/source.rs | 124 ++++ src/sources/util/mod.rs | 12 +- .../reference/configuration/sources/mqtt.md | 14 + .../reference/components/sinks/base/mqtt.cue | 2 +- .../components/sources/base/mqtt.cue | 584 ++++++++++++++++++ .../cue/reference/components/sources/mqtt.cue | 99 +++ 18 files changed, 1296 insertions(+), 103 deletions(-) create mode 100644 changelog.d/19931_add_mqtt_source.feature.md create mode 100644 src/common/mqtt.rs create mode 100644 src/sources/mqtt/config.rs create mode 100644 src/sources/mqtt/integration_tests.rs create mode 100644 src/sources/mqtt/mod.rs create mode 100644 src/sources/mqtt/source.rs create mode 100644 website/content/en/docs/reference/configuration/sources/mqtt.md create mode 100644 website/cue/reference/components/sources/base/mqtt.cue create mode 100644 website/cue/reference/components/sources/mqtt.cue diff --git a/Cargo.toml b/Cargo.toml index fdbcfb5e25eed..42c75a7de7a8d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -580,6 +580,7 @@ sources-logs = [ "sources-kafka", "sources-kubernetes_logs", "sources-logstash", + "sources-mqtt", "sources-nats", "sources-opentelemetry", "sources-pulsar", @@ -635,6 +636,7 @@ sources-kafka = ["dep:rdkafka"] sources-kubernetes_logs = ["vector-lib/file-source", "kubernetes", "transforms-reduce"] sources-logstash = ["sources-utils-net-tcp", "tokio-util/net"] sources-mongodb_metrics = ["dep:mongodb"] +sources-mqtt = ["dep:rumqttc"] sources-nats = ["dep:async-nats", "dep:nkeys"] sources-nginx_metrics = ["dep:nom"] sources-opentelemetry = ["dep:hex", "vector-lib/opentelemetry", "dep:prost", "dep:prost-types", "sources-http_server", "sources-utils-http", "sources-utils-http-headers", "sources-vector"] @@ -934,7 +936,7 @@ kafka-integration-tests = ["sinks-kafka", "sources-kafka"] logstash-integration-tests = ["docker", "sources-logstash"] loki-integration-tests = ["sinks-loki"] mongodb_metrics-integration-tests = ["sources-mongodb_metrics"] -mqtt-integration-tests = ["sinks-mqtt"] +mqtt-integration-tests = ["sinks-mqtt", "sources-mqtt"] nats-integration-tests = ["sinks-nats", "sources-nats"] nginx-integration-tests = ["sources-nginx_metrics"] opentelemetry-integration-tests = ["sources-opentelemetry", "dep:prost"] diff --git a/changelog.d/19931_add_mqtt_source.feature.md b/changelog.d/19931_add_mqtt_source.feature.md new file mode 100644 index 0000000000000..24e9dbebc6aed --- /dev/null +++ b/changelog.d/19931_add_mqtt_source.feature.md @@ -0,0 +1,3 @@ +Adds a new `mqtt` source enabling Vector to receive logs from a MQTT broker. + +authors: mladedav pront StormStake diff --git a/scripts/integration/mqtt/test.yaml b/scripts/integration/mqtt/test.yaml index 607da45eeb5e1..ec0c637e094d1 100644 --- a/scripts/integration/mqtt/test.yaml +++ b/scripts/integration/mqtt/test.yaml @@ -10,3 +10,4 @@ paths: - "src/internal_events/mqtt.rs" - "src/sinks/mqtt/**" - "src/sinks/util/**" +- "src/sources/mqtt/**" diff --git a/src/common/mod.rs b/src/common/mod.rs index cfaf80c4f9354..9caf79b8c7fbd 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -18,6 +18,10 @@ pub(crate) mod sqs; #[cfg(any(feature = "sources-aws_s3", feature = "sinks-aws_s3"))] pub(crate) mod s3; +#[cfg(any(feature = "sources-mqtt", feature = "sinks-mqtt",))] +/// Common MQTT configuration shared by MQTT components. +pub mod mqtt; + #[cfg(any(feature = "transforms-log_to_metric", feature = "sinks-loki"))] pub(crate) mod expansion; diff --git a/src/common/mqtt.rs b/src/common/mqtt.rs new file mode 100644 index 0000000000000..35f3c27812bcc --- /dev/null +++ b/src/common/mqtt.rs @@ -0,0 +1,123 @@ +use rumqttc::{AsyncClient, EventLoop, MqttOptions}; +use snafu::Snafu; +use vector_config_macros::configurable_component; +use vector_lib::tls::{TlsEnableableConfig, TlsError}; + +use crate::template::TemplateParseError; + +/// Shared MQTT configuration for sources and sinks. +#[configurable_component] +#[derive(Clone, Debug, Derivative)] +#[derivative(Default)] +#[serde(deny_unknown_fields)] +pub struct MqttCommonConfig { + /// MQTT server address (The broker’s domain name or IP address). + #[configurable(metadata(docs::examples = "mqtt.example.com", docs::examples = "127.0.0.1"))] + pub host: String, + + /// TCP port of the MQTT server to connect to. + #[configurable(derived)] + #[serde(default = "default_port")] + #[derivative(Default(value = "default_port()"))] + pub port: u16, + + /// MQTT username. + #[serde(default)] + #[configurable(derived)] + pub user: Option, + + /// MQTT password. + #[serde(default)] + #[configurable(derived)] + pub password: Option, + + /// MQTT client ID. + #[serde(default)] + #[configurable(derived)] + pub client_id: Option, + + /// Connection keep-alive interval. + #[serde(default = "default_keep_alive")] + #[derivative(Default(value = "default_keep_alive()"))] + pub keep_alive: u16, + + /// TLS configuration. + #[configurable(derived)] + pub tls: Option, +} + +const fn default_port() -> u16 { + 1883 +} + +const fn default_keep_alive() -> u16 { + 60 +} + +/// MQTT Error Types +#[derive(Debug, Snafu)] +#[snafu(visibility(pub))] +pub enum MqttError { + /// Topic template parsing failed + #[snafu(display("invalid topic template: {source}"))] + TopicTemplate { + /// Source of error + source: TemplateParseError, + }, + /// TLS error + #[snafu(display("TLS error: {source}"))] + Tls { + /// Source of error + source: TlsError, + }, + /// Configuration error + #[snafu(display("MQTT configuration error: {source}"))] + Configuration { + /// Source of error + source: ConfigurationError, + }, +} + +/// MQTT Configuration error types +#[derive(Clone, Debug, Eq, PartialEq, Snafu)] +pub enum ConfigurationError { + /// Empty client ID error + #[snafu(display("Client ID is not allowed to be empty."))] + EmptyClientId, + /// Invalid credentials provided error + #[snafu(display("Username and password must be either both provided or both missing."))] + InvalidCredentials, + /// Invalid client ID provied error + #[snafu(display( + "Client ID must be 1-23 characters long and must consist of only alphanumeric characters." + ))] + InvalidClientId, + /// Credentials provided were incomplete + #[snafu(display("Username and password must be either both or neither provided."))] + IncompleteCredentials, +} + +#[derive(Clone)] +/// Mqtt connector wrapper +pub struct MqttConnector { + /// Mqtt connection options + pub options: MqttOptions, +} + +impl MqttConnector { + /// Creates a new MqttConnector + pub const fn new(options: MqttOptions) -> Self { + Self { options } + } + + /// Connects the connector and generates a client and eventloop + pub fn connect(&self) -> (AsyncClient, EventLoop) { + let (client, eventloop) = AsyncClient::new(self.options.clone(), 1024); + (client, eventloop) + } + + /// TODO: Right now there is no way to implement the healthcheck properly: + pub async fn healthcheck(&self) -> crate::Result<()> { + Ok(()) + } +} diff --git a/src/sinks/mqtt/config.rs b/src/sinks/mqtt/config.rs index a518375a7f584..8c67c4acadf76 100644 --- a/src/sinks/mqtt/config.rs +++ b/src/sinks/mqtt/config.rs @@ -2,53 +2,32 @@ use std::time::Duration; use rand::Rng; use rumqttc::{MqttOptions, QoS, TlsConfiguration, Transport}; -use snafu::{ResultExt, Snafu}; +use snafu::ResultExt; use vector_lib::codecs::JsonSerializerConfig; use crate::template::Template; use crate::{ codecs::EncodingConfig, - config::{AcknowledgementsConfig, Input, SinkConfig, SinkContext}, - sinks::{ - mqtt::sink::{ConfigurationSnafu, MqttConnector, MqttError, MqttSink, TlsSnafu}, - prelude::*, - Healthcheck, VectorSink, + common::mqtt::{ + ConfigurationError, ConfigurationSnafu, MqttCommonConfig, MqttConnector, MqttError, + TlsSnafu, }, - tls::{MaybeTlsSettings, TlsEnableableConfig}, + config::{AcknowledgementsConfig, Input, SinkConfig, SinkContext}, + sinks::{mqtt::sink::MqttSink, prelude::*, Healthcheck, VectorSink}, + tls::MaybeTlsSettings, }; /// Configuration for the `mqtt` sink #[configurable_component(sink("mqtt"))] #[derive(Clone, Debug)] pub struct MqttSinkConfig { - /// MQTT server address (The broker’s domain name or IP address). - #[configurable(metadata(docs::examples = "mqtt.example.com", docs::examples = "127.0.0.1"))] - pub host: String, - - /// TCP port of the MQTT server to connect to. - #[serde(default = "default_port")] - pub port: u16, - - /// MQTT username. - pub user: Option, - - /// MQTT password. - pub password: Option, - - /// MQTT client ID. - pub client_id: Option, - - /// Connection keep-alive interval. - #[serde(default = "default_keep_alive")] - pub keep_alive: u16, + #[serde(flatten)] + pub common: MqttCommonConfig, /// If set to true, the MQTT session is cleaned on login. #[serde(default = "default_clean_session")] pub clean_session: bool, - #[configurable(derived)] - pub tls: Option, - /// MQTT publish topic (templates allowed) pub topic: Template, @@ -100,14 +79,6 @@ impl From for QoS { } } -const fn default_port() -> u16 { - 1883 -} - -const fn default_keep_alive() -> u16 { - 60 -} - const fn default_clean_session() -> bool { false } @@ -123,14 +94,9 @@ const fn default_retain() -> bool { impl Default for MqttSinkConfig { fn default() -> Self { Self { - host: "localhost".into(), - port: default_port(), - user: None, - password: None, - client_id: None, - keep_alive: default_keep_alive(), + common: MqttCommonConfig::default(), clean_session: default_clean_session(), - tls: None, + topic: Template::try_from("vector").expect("Cannot parse as a template"), retain: default_retain(), encoding: JsonSerializerConfig::default().into(), @@ -164,17 +130,9 @@ impl SinkConfig for MqttSinkConfig { } } -#[derive(Clone, Debug, Eq, PartialEq, Snafu)] -pub enum ConfigurationError { - #[snafu(display("Client ID is not allowed to be empty."))] - EmptyClientId, - #[snafu(display("Username and password must be either both provided or both missing."))] - InvalidCredentials, -} - impl MqttSinkConfig { fn build_connector(&self) -> Result { - let client_id = self.client_id.clone().unwrap_or_else(|| { + let client_id = self.common.client_id.clone().unwrap_or_else(|| { let hash = rand::rng() .sample_iter(&rand_distr::Alphanumeric) .take(6) @@ -186,11 +144,12 @@ impl MqttSinkConfig { if client_id.is_empty() { return Err(ConfigurationError::EmptyClientId).context(ConfigurationSnafu); } - let tls = MaybeTlsSettings::from_config(self.tls.as_ref(), false).context(TlsSnafu)?; - let mut options = MqttOptions::new(&client_id, &self.host, self.port); - options.set_keep_alive(Duration::from_secs(self.keep_alive.into())); + let tls = + MaybeTlsSettings::from_config(self.common.tls.as_ref(), false).context(TlsSnafu)?; + let mut options = MqttOptions::new(&client_id, &self.common.host, self.common.port); + options.set_keep_alive(Duration::from_secs(self.common.keep_alive.into())); options.set_clean_session(self.clean_session); - match (&self.user, &self.password) { + match (&self.common.user, &self.common.password) { (Some(user), Some(password)) => { options.set_credentials(user, password); } @@ -211,7 +170,7 @@ impl MqttSinkConfig { alpn, })); } - MqttConnector::new(options, self.topic.to_string()) + Ok(MqttConnector::new(options)) } } diff --git a/src/sinks/mqtt/integration_tests.rs b/src/sinks/mqtt/integration_tests.rs index cce25c0b81b10..6ef78bdeb9cb3 100644 --- a/src/sinks/mqtt/integration_tests.rs +++ b/src/sinks/mqtt/integration_tests.rs @@ -1,3 +1,4 @@ +use crate::common::mqtt::MqttCommonConfig; use crate::config::{SinkConfig, SinkContext}; use crate::sinks::mqtt::config::MqttQoS; use crate::sinks::mqtt::MqttSinkConfig; @@ -25,16 +26,21 @@ async fn mqtt_happy() { trace_init(); let topic = "test"; - let cnf = MqttSinkConfig { + let common = MqttCommonConfig { host: mqtt_broker_address(), port: mqtt_broker_port(), + ..Default::default() + }; + + let config = MqttSinkConfig { + common, topic: Template::try_from(topic).expect("Cannot parse the topic template"), quality_of_service: MqttQoS::AtLeastOnce, ..Default::default() }; let cx = SinkContext::default(); - let (sink, healthcheck) = cnf.build(cx).await.expect("Cannot build the sink"); + let (sink, healthcheck) = config.build(cx).await.expect("Cannot build the sink"); healthcheck.await.expect("Health check failed"); // prepare consumer diff --git a/src/sinks/mqtt/sink.rs b/src/sinks/mqtt/sink.rs index d220cf8af2d40..a0f07e3f97151 100644 --- a/src/sinks/mqtt/sink.rs +++ b/src/sinks/mqtt/sink.rs @@ -1,56 +1,22 @@ use async_trait::async_trait; use futures::{stream::BoxStream, StreamExt}; -use rumqttc::{AsyncClient, EventLoop, MqttOptions}; -use snafu::{ResultExt, Snafu}; -use vector_lib::tls::TlsError; +use crate::common::mqtt::MqttConnector; use crate::internal_events::MqttConnectionError; use crate::sinks::prelude::*; use super::{ - config::{ConfigurationError, MqttQoS}, + config::MqttQoS, request_builder::{MqttEncoder, MqttRequestBuilder}, service::MqttService, MqttSinkConfig, }; -#[derive(Debug, Snafu)] -#[snafu(visibility(pub))] -pub enum MqttError { - #[snafu(display("invalid topic template: {}", source))] - TopicTemplate { source: TemplateParseError }, - #[snafu(display("TLS error: {}", source))] - Tls { source: TlsError }, - #[snafu(display("MQTT configuration error: {}", source))] - Configuration { source: ConfigurationError }, -} - -#[derive(Clone)] -pub struct MqttConnector { - options: MqttOptions, - topic: Template, -} - -impl MqttConnector { - pub fn new(options: MqttOptions, topic: String) -> Result { - let topic = Template::try_from(topic).context(TopicTemplateSnafu)?; - Ok(Self { options, topic }) - } - - fn connect(&self) -> (AsyncClient, EventLoop) { - AsyncClient::new(self.options.clone(), 1024) - } - - pub async fn healthcheck(&self) -> crate::Result<()> { - // TODO: Right now there is no way to implement the healthcheck properly: https://github.com/bytebeamio/rumqtt/issues/562 - Ok(()) - } -} - pub struct MqttSink { transformer: Transformer, encoder: Encoder<()>, connector: MqttConnector, + topic: Template, quality_of_service: MqttQoS, retain: bool, } @@ -64,12 +30,14 @@ impl MqttSink { pub fn new(config: &MqttSinkConfig, connector: MqttConnector) -> crate::Result { let transformer = config.encoding.transformer(); let serializer = config.encoding.build()?; + let topic = config.topic.clone(); let encoder = Encoder::<()>::new(serializer); Ok(Self { transformer, encoder, connector, + topic, quality_of_service: config.quality_of_service, retain: config.retain, }) @@ -77,7 +45,6 @@ impl MqttSink { fn make_mqtt_event(&self, event: Event) -> Option { let topic = self - .connector .topic .render_string(&event) .map_err(|missing_keys| { diff --git a/src/sources/mod.rs b/src/sources/mod.rs index 8a86d9d6343d0..91e6c333d3705 100644 --- a/src/sources/mod.rs +++ b/src/sources/mod.rs @@ -58,6 +58,8 @@ pub mod kubernetes_logs; pub mod logstash; #[cfg(feature = "sources-mongodb_metrics")] pub mod mongodb_metrics; +#[cfg(feature = "sources-mqtt")] +pub mod mqtt; #[cfg(feature = "sources-nats")] pub mod nats; #[cfg(feature = "sources-nginx_metrics")] diff --git a/src/sources/mqtt/config.rs b/src/sources/mqtt/config.rs new file mode 100644 index 0000000000000..4e8c7187edd3a --- /dev/null +++ b/src/sources/mqtt/config.rs @@ -0,0 +1,175 @@ +use std::time::Duration; + +use rand::Rng; +use rumqttc::{MqttOptions, TlsConfiguration, Transport}; +use snafu::ResultExt; +use vector_lib::{ + codecs::decoding::{DeserializerConfig, FramingConfig}, + config::{LegacyKey, LogNamespace}, + configurable::configurable_component, + lookup::lookup_v2::OptionalValuePath, + lookup::owned_value_path, + tls::MaybeTlsSettings, +}; +use vrl::value::Kind; + +use crate::{ + codecs::DecodingConfig, + common::mqtt::{ + ConfigurationError, ConfigurationSnafu, MqttCommonConfig, MqttConnector, MqttError, + TlsSnafu, + }, + config::{SourceConfig, SourceContext, SourceOutput}, + serde::{default_decoding, default_framing_message_based}, +}; + +use super::source::MqttSource; + +/// Configuration for the `mqtt` source. +#[configurable_component(source("mqtt", "Collect logs from MQTT."))] +#[derive(Clone, Debug, Derivative)] +#[derivative(Default)] +#[serde(deny_unknown_fields)] +pub struct MqttSourceConfig { + #[serde(flatten)] + pub common: MqttCommonConfig, + + /// MQTT topic from which messages are to be read. + #[configurable(derived)] + #[serde(default = "default_topic")] + #[derivative(Default(value = "default_topic()"))] + pub topic: String, + + #[configurable(derived)] + #[serde(default = "default_framing_message_based")] + #[derivative(Default(value = "default_framing_message_based()"))] + pub framing: FramingConfig, + + #[configurable(derived)] + #[serde(default = "default_decoding")] + #[derivative(Default(value = "default_decoding()"))] + pub decoding: DeserializerConfig, + + /// The namespace to use for logs. This overrides the global setting. + #[configurable(metadata(docs::hidden))] + #[serde(default)] + pub log_namespace: Option, + + /// Overrides the name of the log field used to add the topic to each event. + /// + /// The value is the topic from which the MQTT message was published to. + /// + /// By default, `"topic"` is used. + #[serde(default = "default_topic_key")] + #[configurable(metadata(docs::examples = "topic"))] + pub topic_key: OptionalValuePath, +} + +fn default_topic() -> String { + "vector".to_owned() +} + +fn default_topic_key() -> OptionalValuePath { + OptionalValuePath::from(owned_value_path!("topic")) +} + +#[async_trait::async_trait] +#[typetag::serde(name = "mqtt")] +impl SourceConfig for MqttSourceConfig { + async fn build(&self, cx: SourceContext) -> crate::Result { + let log_namespace = cx.log_namespace(self.log_namespace); + + let connector = self.build_connector()?; + + let decoder = + DecodingConfig::new(self.framing.clone(), self.decoding.clone(), log_namespace) + .build()?; + + let sink = MqttSource::new(connector.clone(), decoder, log_namespace, self.clone())?; + Ok(Box::pin(sink.run(cx.out, cx.shutdown))) + } + + fn outputs(&self, global_log_namespace: LogNamespace) -> Vec { + let schema_definition = self + .decoding + .schema_definition(global_log_namespace.merge(self.log_namespace)) + .with_standard_vector_source_metadata() + .with_source_metadata( + Self::NAME, + Some(LegacyKey::Overwrite(owned_value_path!("timestamp"))), + &owned_value_path!("timestamp"), + Kind::timestamp().or_undefined(), + Some("timestamp"), + ); + + vec![SourceOutput::new_maybe_logs( + self.decoding.output_type(), + schema_definition, + )] + } + + fn can_acknowledge(&self) -> bool { + false + } +} + +impl MqttSourceConfig { + fn build_connector(&self) -> Result { + let client_id = self.common.client_id.clone().unwrap_or_else(|| { + let hash = rand::rng() + .sample_iter(&rand_distr::Alphanumeric) + .take(6) + .map(char::from) + .collect::(); + format!("vectorSource{hash}") + }); + + if client_id.is_empty() { + return Err(ConfigurationError::InvalidClientId).context(ConfigurationSnafu); + } + + let tls = + MaybeTlsSettings::from_config(self.common.tls.as_ref(), false).context(TlsSnafu)?; + let mut options = MqttOptions::new(client_id, &self.common.host, self.common.port); + options.set_keep_alive(Duration::from_secs(self.common.keep_alive.into())); + + options.set_clean_session(false); + match (&self.common.user, &self.common.password) { + (Some(user), Some(password)) => { + options.set_credentials(user, password); + } + (None, None) => { + // Credentials were not provided + } + _ => { + // We need either both username and password, or neither. MQTT also allows for providing only password, but rumqttc does not allow that so we cannot either. + return Err(ConfigurationError::IncompleteCredentials).context(ConfigurationSnafu); + } + } + + if let Some(tls) = tls.tls() { + let ca = tls.authorities_pem().flatten().collect(); + let client_auth = None; + let alpn = Some(vec!["mqtt".into()]); + options.set_transport(Transport::Tls(TlsConfiguration::Simple { + ca, + client_auth, + alpn, + })); + } + + Ok(MqttConnector::new(options)) + } +} + +impl_generate_config_from_default!(MqttSourceConfig); + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn generate_config() { + crate::test_util::test_generate_config::(); + } +} diff --git a/src/sources/mqtt/integration_tests.rs b/src/sources/mqtt/integration_tests.rs new file mode 100644 index 0000000000000..cdc2c41fde5ab --- /dev/null +++ b/src/sources/mqtt/integration_tests.rs @@ -0,0 +1,117 @@ +#![cfg(feature = "mqtt-integration-tests")] +#![cfg(test)] + +use crate::common::mqtt::MqttCommonConfig; +use crate::test_util::trace_init; +use crate::test_util::{components::SOURCE_TAGS, random_lines_with_stream, random_string}; +use rumqttc::{AsyncClient, MqttOptions, QoS}; +use std::{collections::HashSet, time::Duration}; + +use futures::StreamExt; +use tokio::time::timeout; + +use super::MqttSourceConfig; +use crate::{ + config::{log_schema, SourceConfig, SourceContext}, + event::Event, + test_util::components::assert_source_compliance, + SourceSender, +}; + +fn mqtt_broker_address() -> String { + std::env::var("MQTT_BROKER_ADDRESS").unwrap_or_else(|_| "emqx".into()) +} + +fn mqtt_broker_port() -> u16 { + std::env::var("MQTT_BROKER_PORT") + .unwrap_or_else(|_| "1883".into()) + .parse::() + .expect("Cannot parse as u16") +} + +async fn send_test_events(client: &AsyncClient, topic: &str, messages: &Vec) { + for message in messages { + client + .publish(topic, QoS::AtLeastOnce, false, message.as_bytes()) + .await + .unwrap(); + } +} + +async fn get_mqtt_client() -> AsyncClient { + let mut mqtt_options = MqttOptions::new( + "integration-test-producer", + mqtt_broker_address(), + mqtt_broker_port(), + ); + mqtt_options.set_keep_alive(Duration::from_secs(5)); + + let (client, mut eventloop) = AsyncClient::new(mqtt_options, 10); + + tokio::spawn(async move { + loop { + eventloop.poll().await.unwrap(); + } + }); + + client +} + +#[tokio::test] +async fn mqtt_happy() { + trace_init(); + let topic = "source-test"; + // We always want new client ID. If it were stable, subsequent tests could receive data sent in previous runs. + let client_id = format!("sourceTest{}", random_string(6)); + let num_events = 10; + let (input, _events) = random_lines_with_stream(100, num_events, None); + + assert_source_compliance(&SOURCE_TAGS, async { + let common = MqttCommonConfig { + host: mqtt_broker_address(), + port: mqtt_broker_port(), + client_id: Some(client_id), + ..Default::default() + }; + + let config = MqttSourceConfig { + common, + topic: topic.to_owned(), + ..MqttSourceConfig::default() + }; + + let (tx, rx) = SourceSender::new_test(); + tokio::spawn(async move { + config + .build(SourceContext::new_test(tx, None)) + .await + .unwrap() + .await + .unwrap() + }); + + tokio::time::sleep(Duration::from_millis(100)).await; + + let client = get_mqtt_client().await; + send_test_events(&client, topic, &input).await; + + let mut expected_messages: HashSet<_> = input.into_iter().collect(); + + let events: Vec = timeout(Duration::from_secs(2), rx.take(num_events).collect()) + .await + .unwrap(); + + for event in events { + let message = event + .as_log() + .get(log_schema().message_key_target_path().unwrap()) + .unwrap() + .to_string_lossy(); + if !expected_messages.remove(message.as_ref()) { + panic!("Received unexpected message: {:?}", message); + } + } + assert!(expected_messages.is_empty()); + }) + .await; +} diff --git a/src/sources/mqtt/mod.rs b/src/sources/mqtt/mod.rs new file mode 100644 index 0000000000000..a1cd149779be8 --- /dev/null +++ b/src/sources/mqtt/mod.rs @@ -0,0 +1,5 @@ +mod config; +mod integration_tests; +mod source; + +pub use config::MqttSourceConfig; diff --git a/src/sources/mqtt/source.rs b/src/sources/mqtt/source.rs new file mode 100644 index 0000000000000..62b379a75812a --- /dev/null +++ b/src/sources/mqtt/source.rs @@ -0,0 +1,124 @@ +use itertools::Itertools; +use vector_lib::config::LogNamespace; +use vector_lib::internal_event::EventsReceived; + +use crate::{ + codecs::Decoder, + common::mqtt::MqttConnector, + event::BatchNotifier, + event::Event, + internal_events::{EndpointBytesReceived, StreamClosedError}, + shutdown::ShutdownSignal, + sources::mqtt::MqttSourceConfig, + sources::util, + SourceSender, +}; +use rumqttc::{Event as MqttEvent, Incoming, Publish, QoS}; +use vector_lib::config::LegacyKey; +use vector_lib::lookup::path; + +pub struct MqttSource { + connector: MqttConnector, + decoder: Decoder, + log_namespace: LogNamespace, + config: MqttSourceConfig, +} + +impl MqttSource { + pub fn new( + connector: MqttConnector, + decoder: Decoder, + log_namespace: LogNamespace, + config: MqttSourceConfig, + ) -> crate::Result { + Ok(Self { + connector, + decoder, + log_namespace, + config, + }) + } + + pub async fn run(self, mut out: SourceSender, shutdown: ShutdownSignal) -> Result<(), ()> { + let (client, mut connection) = self.connector.connect(); + + client + .subscribe(&self.config.topic, QoS::AtLeastOnce) + .await + .map_err(|_| ())?; + + loop { + tokio::select! { + _ = shutdown.clone() => return Ok(()), + mqtt_event = connection.poll() => { + // If an error is returned here there is currently no way to tie this back + // to the event that was posted which means we can't accurately provide + // delivery guarantees. + // We need this issue resolved first: + // https://github.com/bytebeamio/rumqtt/issues/349 + match mqtt_event { + Ok(MqttEvent::Incoming(Incoming::Publish(publish))) => { + self.process_message(publish, &mut out).await; + } + Ok(MqttEvent::Incoming( + Incoming::PubAck(_) | Incoming::PubRec(_) | Incoming::PubComp(_), + )) => { + // TODO Handle acknowledgement - https://github.com/vectordotdev/vector/issues/21967 + } + _ => {} + } + } + } + } + } + + async fn process_message(&self, publish: Publish, out: &mut SourceSender) { + emit!(EndpointBytesReceived { + byte_size: publish.payload.len(), + protocol: "mqtt", + endpoint: &self.connector.options.broker_address().0, + }); + let events_received = register!(EventsReceived); + + let (batch, _batch_receiver) = BatchNotifier::maybe_new_with_receiver(false); + // Error is logged by `crate::codecs::Decoder`, no further handling + // is needed here. + let decoded = util::decode_message( + self.decoder.clone(), + "mqtt", + &publish.payload, + None, + &batch, + self.log_namespace, + &events_received, + ) + .map(|mut event| { + self.apply_metadata(&publish, &mut event); + event + }) + .collect_vec(); + + let count = decoded.len(); + + match out.send_batch(decoded).await { + Ok(()) => {} + Err(_) => emit!(StreamClosedError { count }), + } + } + + fn apply_metadata(&self, publish: &Publish, event: &mut Event) { + if let Event::Log(ref mut log) = event { + self.log_namespace.insert_source_metadata( + MqttSourceConfig::NAME, + log, + self.config + .topic_key + .path + .as_ref() + .map(LegacyKey::Overwrite), + path!("topic"), + publish.topic.clone(), + ); + } + } +} diff --git a/src/sources/util/mod.rs b/src/sources/util/mod.rs index 45e773316c4cb..f7e103d2dd848 100644 --- a/src/sources/util/mod.rs +++ b/src/sources/util/mod.rs @@ -16,7 +16,11 @@ pub mod grpc; pub mod http; #[cfg(any(feature = "sources-http_client", feature = "sources-prometheus-scrape",))] pub mod http_client; -#[cfg(any(feature = "sources-aws_sqs", feature = "sources-gcp_pubsub"))] +#[cfg(any( + feature = "sources-aws_sqs", + feature = "sources-gcp_pubsub", + feature = "sources-mqtt" +))] mod message_decoding; pub mod multiline_config; #[cfg(any(feature = "sources-utils-net-tcp", feature = "sources-utils-net-udp"))] @@ -60,7 +64,11 @@ pub use self::http::add_query_parameters; pub use self::http::decode; #[cfg(feature = "sources-utils-http-prelude")] pub use self::http::HttpSource; -#[cfg(any(feature = "sources-aws_sqs", feature = "sources-gcp_pubsub"))] +#[cfg(any( + feature = "sources-aws_sqs", + feature = "sources-gcp_pubsub", + feature = "sources-mqtt" +))] pub use self::message_decoding::decode_message; /// Extract a tag and it's value from input string delimited by a colon character. diff --git a/website/content/en/docs/reference/configuration/sources/mqtt.md b/website/content/en/docs/reference/configuration/sources/mqtt.md new file mode 100644 index 0000000000000..179b0bce2e9d5 --- /dev/null +++ b/website/content/en/docs/reference/configuration/sources/mqtt.md @@ -0,0 +1,14 @@ +--- +title: MQTT +description: Receive logs from an [MQTT](https://mqtt.org) broker +kind: source +layout: component +tags: ["mqtt", "component", "source"] +--- + +{{/* +This doc is generated using: + +1. The template in layouts/docs/component.html +2. The relevant CUE data in cue/reference/components/... +*/}} diff --git a/website/cue/reference/components/sinks/base/mqtt.cue b/website/cue/reference/components/sinks/base/mqtt.cue index 74770fef3b5ef..12bed930db3c8 100644 --- a/website/cue/reference/components/sinks/base/mqtt.cue +++ b/website/cue/reference/components/sinks/base/mqtt.cue @@ -419,7 +419,7 @@ base: components: sinks: mqtt: configuration: { type: bool: default: false } tls: { - description: "Configures the TLS options for incoming/outgoing connections." + description: "TLS configuration." required: false type: object: options: { alpn_protocols: { diff --git a/website/cue/reference/components/sources/base/mqtt.cue b/website/cue/reference/components/sources/base/mqtt.cue new file mode 100644 index 0000000000000..9ace00d90fb5f --- /dev/null +++ b/website/cue/reference/components/sources/base/mqtt.cue @@ -0,0 +1,584 @@ +package metadata + +base: components: sources: mqtt: configuration: { + client_id: { + description: "MQTT client ID." + required: false + type: string: {} + } + decoding: { + description: "Configures how events are decoded from raw bytes." + required: false + type: object: options: { + avro: { + description: "Apache Avro-specific encoder options." + relevant_when: "codec = \"avro\"" + required: true + type: object: options: { + schema: { + description: """ + The Avro schema definition. + Please note that the following [`apache_avro::types::Value`] variants are currently *not* supported: + * `Date` + * `Decimal` + * `Duration` + * `Fixed` + * `TimeMillis` + """ + required: true + type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] + } + strip_schema_id_prefix: { + description: """ + For Avro datum encoded in Kafka messages, the bytes are prefixed with the schema ID. Set this to true to strip the schema ID prefix. + According to [Confluent Kafka's document](https://docs.confluent.io/platform/current/schema-registry/fundamentals/serdes-develop/index.html#wire-format). + """ + required: true + type: bool: {} + } + } + } + codec: { + description: "The codec to use for decoding events." + required: false + type: string: { + default: "bytes" + enum: { + avro: """ + Decodes the raw bytes as as an [Apache Avro][apache_avro] message. + + [apache_avro]: https://avro.apache.org/ + """ + bytes: "Uses the raw bytes as-is." + gelf: """ + Decodes the raw bytes as a [GELF][gelf] message. + + This codec is experimental for the following reason: + + The GELF specification is more strict than the actual Graylog receiver. + Vector's decoder currently adheres more strictly to the GELF spec, with + the exception that some characters such as `@` are allowed in field names. + + Other GELF codecs such as Loki's, use a [Go SDK][implementation] that is maintained + by Graylog, and is much more relaxed than the GELF spec. + + Going forward, Vector will use that [Go SDK][implementation] as the reference implementation, which means + the codec may continue to relax the enforcement of specification. + + [gelf]: https://docs.graylog.org/docs/gelf + [implementation]: https://github.com/Graylog2/go-gelf/blob/v2/gelf/reader.go + """ + influxdb: """ + Decodes the raw bytes as an [Influxdb Line Protocol][influxdb] message. + + [influxdb]: https://docs.influxdata.com/influxdb/cloud/reference/syntax/line-protocol + """ + json: """ + Decodes the raw bytes as [JSON][json]. + + [json]: https://www.json.org/ + """ + native: """ + Decodes the raw bytes as [native Protocol Buffers format][vector_native_protobuf]. + + This codec is **[experimental][experimental]**. + + [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + native_json: """ + Decodes the raw bytes as [native JSON format][vector_native_json]. + + This codec is **[experimental][experimental]**. + + [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + protobuf: """ + Decodes the raw bytes as [protobuf][protobuf]. + + [protobuf]: https://protobuf.dev/ + """ + syslog: """ + Decodes the raw bytes as a Syslog message. + + Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the + [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + + [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt + [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + """ + vrl: """ + Decodes the raw bytes as a string and passes them as input to a [VRL][vrl] program. + + [vrl]: https://vector.dev/docs/reference/vrl + """ + } + } + } + gelf: { + description: "GELF-specific decoding options." + relevant_when: "codec = \"gelf\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of failing. + + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character + """ + required: false + type: bool: default: true + } + } + influxdb: { + description: "Influxdb-specific decoding options." + relevant_when: "codec = \"influxdb\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of failing. + + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character + """ + required: false + type: bool: default: true + } + } + json: { + description: "JSON-specific decoding options." + relevant_when: "codec = \"json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of failing. + + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character + """ + required: false + type: bool: default: true + } + } + native_json: { + description: "Vector's native JSON-specific decoding options." + relevant_when: "codec = \"native_json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of failing. + + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character + """ + required: false + type: bool: default: true + } + } + protobuf: { + description: "Protobuf-specific decoding options." + relevant_when: "codec = \"protobuf\"" + required: false + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: false + type: string: default: "" + } + message_type: { + description: "The name of the message type to use for serializing." + required: false + type: string: { + default: "" + examples: ["package.Message"] + } + } + } + } + syslog: { + description: "Syslog-specific decoding options." + relevant_when: "codec = \"syslog\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of failing. + + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character + """ + required: false + type: bool: default: true + } + } + vrl: { + description: "VRL-specific decoding options." + relevant_when: "codec = \"vrl\"" + required: true + type: object: options: { + source: { + description: """ + The [Vector Remap Language][vrl] (VRL) program to execute for each event. + Note that the final contents of the `.` target will be used as the decoding result. + Compilation error or use of 'abort' in a program will result in a decoding error. + + [vrl]: https://vector.dev/docs/reference/vrl + """ + required: true + type: string: {} + } + timezone: { + description: """ + The name of the timezone to apply to timestamp conversions that do not contain an explicit + time zone. The time zone name may be any name in the [TZ database][tz_database], or `local` + to indicate system local time. + + If not set, `local` will be used. + + [tz_database]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones + """ + required: false + type: string: examples: ["local", "America/New_York", "EST5EDT"] + } + } + } + } + } + framing: { + description: """ + Framing configuration. + + Framing handles how events are separated when encoded in a raw byte form, where each event is + a frame that must be prefixed, or delimited, in a way that marks where an event begins and + ends within the byte stream. + """ + required: false + type: object: options: { + character_delimited: { + description: "Options for the character delimited decoder." + relevant_when: "method = \"character_delimited\"" + required: true + type: object: options: { + delimiter: { + description: "The character that delimits byte sequences." + required: true + type: ascii_char: {} + } + max_length: { + description: """ + The maximum length of the byte buffer. + + This length does *not* include the trailing delimiter. + + By default, there is no maximum length enforced. If events are malformed, this can lead to + additional resource usage as events continue to be buffered in memory, and can potentially + lead to memory exhaustion in extreme cases. + + If there is a risk of processing malformed data, such as logs with user-controlled input, + consider setting the maximum length to a reasonably large value as a safety net. This + ensures that processing is not actually unbounded. + """ + required: false + type: uint: {} + } + } + } + chunked_gelf: { + description: "Options for the chunked GELF decoder." + relevant_when: "method = \"chunked_gelf\"" + required: false + type: object: options: { + decompression: { + description: "Decompression configuration for GELF messages." + required: false + type: string: { + default: "Auto" + enum: { + Auto: "Automatically detect the decompression method based on the magic bytes of the message." + Gzip: "Use Gzip decompression." + None: "Do not decompress the message." + Zlib: "Use Zlib decompression." + } + } + } + max_length: { + description: """ + The maximum length of a single GELF message, in bytes. Messages longer than this length will + be dropped. If this option is not set, the decoder does not limit the length of messages and + the per-message memory is unbounded. + + Note that a message can be composed of multiple chunks and this limit is applied to the whole + message, not to individual chunks. + + This limit takes only into account the message's payload and the GELF header bytes are excluded from the calculation. + The message's payload is the concatenation of all the chunks' payloads. + """ + required: false + type: uint: {} + } + pending_messages_limit: { + description: """ + The maximum number of pending incomplete messages. If this limit is reached, the decoder starts + dropping chunks of new messages, ensuring the memory usage of the decoder's state is bounded. + If this option is not set, the decoder does not limit the number of pending messages and the memory usage + of its messages buffer can grow unbounded. This matches Graylog Server's behavior. + """ + required: false + type: uint: {} + } + timeout_secs: { + description: """ + The timeout, in seconds, for a message to be fully received. If the timeout is reached, the + decoder drops all the received chunks of the timed out message. + """ + required: false + type: float: default: 5.0 + } + } + } + length_delimited: { + description: "Options for the length delimited decoder." + relevant_when: "method = \"length_delimited\"" + required: true + type: object: options: { + length_field_is_big_endian: { + description: "Length field byte order (little or big endian)" + required: false + type: bool: default: true + } + length_field_length: { + description: "Number of bytes representing the field length" + required: false + type: uint: default: 4 + } + length_field_offset: { + description: "Number of bytes in the header before the length field" + required: false + type: uint: default: 0 + } + max_frame_length: { + description: "Maximum frame length" + required: false + type: uint: default: 8388608 + } + } + } + method: { + description: "The framing method." + required: false + type: string: { + default: "bytes" + enum: { + bytes: "Byte frames are passed through as-is according to the underlying I/O boundaries (for example, split between messages or stream segments)." + character_delimited: "Byte frames which are delimited by a chosen character." + chunked_gelf: """ + Byte frames which are chunked GELF messages. + + [chunked_gelf]: https://go2docs.graylog.org/current/getting_in_log_data/gelf.html + """ + length_delimited: "Byte frames which are prefixed by an unsigned big-endian 32-bit integer indicating the length." + newline_delimited: "Byte frames which are delimited by a newline character." + octet_counting: """ + Byte frames according to the [octet counting][octet_counting] format. + + [octet_counting]: https://tools.ietf.org/html/rfc6587#section-3.4.1 + """ + } + } + } + newline_delimited: { + description: "Options for the newline delimited decoder." + relevant_when: "method = \"newline_delimited\"" + required: false + type: object: options: max_length: { + description: """ + The maximum length of the byte buffer. + + This length does *not* include the trailing delimiter. + + By default, there is no maximum length enforced. If events are malformed, this can lead to + additional resource usage as events continue to be buffered in memory, and can potentially + lead to memory exhaustion in extreme cases. + + If there is a risk of processing malformed data, such as logs with user-controlled input, + consider setting the maximum length to a reasonably large value as a safety net. This + ensures that processing is not actually unbounded. + """ + required: false + type: uint: {} + } + } + octet_counting: { + description: "Options for the octet counting decoder." + relevant_when: "method = \"octet_counting\"" + required: false + type: object: options: max_length: { + description: "The maximum length of the byte buffer." + required: false + type: uint: {} + } + } + } + } + host: { + description: "MQTT server address (The broker’s domain name or IP address)." + required: true + type: string: examples: ["mqtt.example.com", "127.0.0.1"] + } + keep_alive: { + description: "Connection keep-alive interval." + required: false + type: uint: default: 60 + } + password: { + description: "MQTT password." + required: false + type: string: {} + } + port: { + description: "TCP port of the MQTT server to connect to." + required: false + type: uint: default: 1883 + } + tls: { + description: "TLS configuration." + required: false + type: object: options: { + alpn_protocols: { + description: """ + Sets the list of supported ALPN protocols. + + Declare the supported ALPN protocols, which are used during negotiation with a peer. They are prioritized in the order + that they are defined. + """ + required: false + type: array: items: type: string: examples: ["h2"] + } + ca_file: { + description: """ + Absolute path to an additional CA certificate file. + + The certificate must be in the DER or PEM (X.509) format. Additionally, the certificate can be provided as an inline string in PEM format. + """ + required: false + type: string: examples: ["/path/to/certificate_authority.crt"] + } + crt_file: { + description: """ + Absolute path to a certificate file used to identify this server. + + The certificate must be in DER, PEM (X.509), or PKCS#12 format. Additionally, the certificate can be provided as + an inline string in PEM format. + + If this is set _and_ is not a PKCS#12 archive, `key_file` must also be set. + """ + required: false + type: string: examples: ["/path/to/host_certificate.crt"] + } + enabled: { + description: """ + Whether or not to require TLS for incoming or outgoing connections. + + When enabled and used for incoming connections, an identity certificate is also required. See `tls.crt_file` for + more information. + """ + required: false + type: bool: {} + } + key_file: { + description: """ + Absolute path to a private key file used to identify this server. + + The key must be in DER or PEM (PKCS#8) format. Additionally, the key can be provided as an inline string in PEM format. + """ + required: false + type: string: examples: ["/path/to/host_certificate.key"] + } + key_pass: { + description: """ + Passphrase used to unlock the encrypted key file. + + This has no effect unless `key_file` is set. + """ + required: false + type: string: examples: ["${KEY_PASS_ENV_VAR}", "PassWord1"] + } + server_name: { + description: """ + Server name to use when using Server Name Indication (SNI). + + Only relevant for outgoing connections. + """ + required: false + type: string: examples: ["www.example.com"] + } + verify_certificate: { + description: """ + Enables certificate verification. For components that create a server, this requires that the + client connections have a valid client certificate. For components that initiate requests, + this validates that the upstream has a valid certificate. + + If enabled, certificates must not be expired and must be issued by a trusted + issuer. This verification operates in a hierarchical manner, checking that the leaf certificate (the + certificate presented by the client/server) is not only valid, but that the issuer of that certificate is also valid, and + so on, until the verification process reaches a root certificate. + + Do NOT set this to `false` unless you understand the risks of not verifying the validity of certificates. + """ + required: false + type: bool: {} + } + verify_hostname: { + description: """ + Enables hostname verification. + + If enabled, the hostname used to connect to the remote host must be present in the TLS certificate presented by + the remote host, either as the Common Name or as an entry in the Subject Alternative Name extension. + + Only relevant for outgoing connections. + + Do NOT set this to `false` unless you understand the risks of not verifying the remote hostname. + """ + required: false + type: bool: {} + } + } + } + topic: { + description: "MQTT topic from which messages are to be read." + required: false + type: string: default: "vector" + } + topic_key: { + description: """ + Overrides the name of the log field used to add the topic to each event. + + The value is the topic from which the MQTT message was published to. + + By default, `"topic"` is used. + """ + required: false + type: string: { + default: "topic" + examples: [ + "topic", + ] + } + } + user: { + description: "MQTT username." + required: false + type: string: {} + } +} diff --git a/website/cue/reference/components/sources/mqtt.cue b/website/cue/reference/components/sources/mqtt.cue new file mode 100644 index 0000000000000..8cdf9b7429b55 --- /dev/null +++ b/website/cue/reference/components/sources/mqtt.cue @@ -0,0 +1,99 @@ +package metadata + +components: sources: mqtt: { + title: "MQTT" + + features: { + auto_generated: true + acknowledgements: false + collect: { + checkpoint: enabled: false + from: { + service: services.mqtt + interface: { + socket: { + api: { + title: "MQTT protocol" + url: urls.mqtt + } + direction: "incoming" + port: 1883 + protocols: ["tcp"] + ssl: "optional" + } + } + } + } + multiline: enabled: false + } + + classes: { + commonly_used: false + deployment_roles: ["aggregator"] + delivery: "best_effort" + development: "beta" + egress_method: "stream" + stateful: false + } + + support: { + targets: { + "aarch64-unknown-linux-gnu": true + "aarch64-unknown-linux-musl": true + "armv7-unknown-linux-gnueabihf": true + "armv7-unknown-linux-musleabihf": true + "x86_64-apple-darwin": true + "x86_64-pc-windows-msv": true + "x86_64-unknown-linux-gnu": true + "x86_64-unknown-linux-musl": true + } + requirements: [] + warnings: [] + notices: [] + } + + configuration: base.components.sources.mqtt.configuration + + installation: { + platform_name: null + } + + output: logs: record: { + description: "An individual MQTT message." + fields: { + message: { + description: "The raw line from the MQTT message." + required: true + type: string: { + examples: ["53.126.150.246 - - [01/Oct/2020:11:25:58 -0400] \"GET /disintermediate HTTP/2.0\" 401 20308"] + syntax: "literal" + } + } + timestamp: fields._current_timestamp & { + description: "The current time when the message has been received." + } + topic: { + description: "The MQTT topic that the message came from." + required: true + type: string: { + examples: ["topic/logs"] + syntax: "literal" + } + } + } + } + + how_it_works: components._amqp.how_it_works + + telemetry: metrics: { + open_connections: components.sources.internal_metrics.output.metrics.open_connections + connection_shutdown_total: components.sources.internal_metrics.output.metrics.connection_shutdown_total + component_errors_total: components.sources.internal_metrics.output.metrics.component_errors_total + component_received_events_total: components.sources.internal_metrics.output.metrics.component_received_events_total + component_sent_events_total: components.sources.internal_metrics.output.metrics.component_sent_events_total + component_sent_bytes_total: components.sources.internal_metrics.output.metrics.component_sent_bytes_total + component_sent_events_total: components.sources.internal_metrics.output.metrics.component_sent_events_total + component_sent_events_total: components.sources.internal_metrics.output.metrics.component_sent_events_total + component_sent_event_bytes_total: components.sources.internal_metrics.output.metrics.component_sent_event_bytes_total + } +} From 689a65d51a2f8034f220afcf5e65e1b924019687 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 14 May 2025 14:45:15 -0400 Subject: [PATCH 004/305] fix(ci): e2e dd logs failure (#23038) * wip * bump to stable debian * fix bug with docker ARG evaluation * more dbg statements * try with more recent docker.io/datadog/fakeintake * test with bigger timeout * make test runner usable locally * add retries * more tweaks to config * cleanup --- .github/workflows/ci-integration-review.yml | 6 +-- .github/workflows/e2e.yml | 4 +- .github/workflows/integration.yml | 2 +- scripts/e2e/datadog-logs/compose.yaml | 4 +- .../{ci-int-e2e-test.sh => int-e2e-test.sh} | 12 ++--- tests/data/e2e/datadog/logs/agent_only.yaml | 3 ++ tests/data/e2e/datadog/logs/agent_vector.yaml | 3 ++ tests/e2e/datadog/logs/mod.rs | 47 ++++++++++++------- 8 files changed, 50 insertions(+), 31 deletions(-) rename scripts/{ci-int-e2e-test.sh => int-e2e-test.sh} (82%) diff --git a/.github/workflows/ci-integration-review.yml b/.github/workflows/ci-integration-review.yml index 1371bac1ed337..b75bc701fba03 100644 --- a/.github/workflows/ci-integration-review.yml +++ b/.github/workflows/ci-integration-review.yml @@ -113,7 +113,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int ${{ matrix.service }} + command: bash scripts/int-e2e-test.sh int ${{ matrix.service }} e2e-tests: needs: prep-pr @@ -136,7 +136,7 @@ jobs: with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh e2e datadog-logs + command: bash scripts/int-e2e-test.sh e2e datadog-logs - name: datadog-e2e-metrics if: ${{ startsWith(github.event.review.body, '/ci-run-e2e-datadog-metrics') @@ -146,7 +146,7 @@ jobs: with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh e2e datadog-metrics + command: bash scripts/int-e2e-test.sh e2e datadog-metrics update-pr-status: name: Signal result to PR diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index cdba1faec680f..e5d322d2b8cc9 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -89,7 +89,7 @@ jobs: with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh e2e datadog-logs + command: bash scripts/int-e2e-test.sh e2e datadog-logs - if: (github.event_name == 'schedule' || needs.changes.outputs.all-e2e == 'true' || needs.changes.outputs.e2e-datadog-metrics == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') @@ -98,7 +98,7 @@ jobs: with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh e2e datadog-metrics + command: bash scripts/int-e2e-test.sh e2e datadog-metrics e2e-test-suite: diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index f8920e80f5d08..e5374cc084e14 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -148,7 +148,7 @@ jobs: "${{ needs.changes.outputs.all-int }}" == "true" || \ "$should_run" == "true" ]]; then echo "Running test for ${{ matrix.service }}" - bash scripts/ci-int-e2e-test.sh int ${{ matrix.service }} + bash scripts/int-e2e-test.sh int ${{ matrix.service }} else echo "Skipping ${{ matrix.service }} test as the value is false or conditions not met." fi diff --git a/scripts/e2e/datadog-logs/compose.yaml b/scripts/e2e/datadog-logs/compose.yaml index 56fb68350c004..fe0bafc869012 100644 --- a/scripts/e2e/datadog-logs/compose.yaml +++ b/scripts/e2e/datadog-logs/compose.yaml @@ -85,13 +85,13 @@ services: # which does the validation of consistency with the other fakeintake service. fakeintake-agent: # TODO: temporarily pegging the image as latest results in failures - image: docker.io/datadog/fakeintake:v77a06f2b + image: docker.io/datadog/fakeintake:ved764626 # Receives log data from the `datadog-agent-vector` service. Is queried by the test runner # which does the validation of consistency with the other fakeintake service. fakeintake-vector: # TODO: temporarily pegging the image as latest results in failures - image: docker.io/datadog/fakeintake:v77a06f2b + image: docker.io/datadog/fakeintake:ved764626 networks: default: diff --git a/scripts/ci-int-e2e-test.sh b/scripts/int-e2e-test.sh similarity index 82% rename from scripts/ci-int-e2e-test.sh rename to scripts/int-e2e-test.sh index 3b569db530f5f..f795c352e3d8e 100755 --- a/scripts/ci-int-e2e-test.sh +++ b/scripts/int-e2e-test.sh @@ -6,11 +6,6 @@ set -u -if [[ -z "${CI:-}" ]]; then - echo "Aborted: this script is for use in CI." >&2 - exit 1 -fi - if [ $# -ne 2 ] then echo "usage: $0 [int|e2e] TEST_NAME" @@ -27,5 +22,10 @@ sleep 30 cargo vdev -v "${TEST_TYPE}" test --retries 2 -a "${TEST_NAME}" RET=$? cargo vdev -v "${TEST_TYPE}" stop -a "${TEST_NAME}" -./scripts/upload-test-results.sh + +# Only upload test results if CI is defined +if [[ -n "${CI:-}" ]]; then + ./scripts/upload-test-results.sh +fi + exit $RET diff --git a/tests/data/e2e/datadog/logs/agent_only.yaml b/tests/data/e2e/datadog/logs/agent_only.yaml index dcdfcab9b40dd..b92e08db48f05 100644 --- a/tests/data/e2e/datadog/logs/agent_only.yaml +++ b/tests/data/e2e/datadog/logs/agent_only.yaml @@ -27,3 +27,6 @@ logs_config: logs_no_ssl: true force_use_http: true batch_wait: 1 + +# Required per https://github.com/DataDog/datadog-agent/tree/main/test/fakeintake#docker +dd_url: 'http://fakeintake-agent:80' diff --git a/tests/data/e2e/datadog/logs/agent_vector.yaml b/tests/data/e2e/datadog/logs/agent_vector.yaml index 411da03025e0c..57ca8bff2d5e7 100644 --- a/tests/data/e2e/datadog/logs/agent_vector.yaml +++ b/tests/data/e2e/datadog/logs/agent_vector.yaml @@ -32,3 +32,6 @@ vector: logs: enabled: true url: "http://vector:8181" + +# Required per https://github.com/DataDog/datadog-agent/tree/main/test/fakeintake#docker +dd_url: 'http://fakeintake-agent:80' diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index de9210388ed39..952d55d3e1d2c 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -1,11 +1,13 @@ use serde_json::Value; use tracing::info; -use vector::test_util::trace_init; - use super::*; +use std::time::Duration; +use vector::test_util::trace_init; const LOGS_ENDPOINT: &str = "/api/v2/logs"; +const MAX_RETRIES: usize = 10; +const WAIT_INTERVAL: Duration = Duration::from_secs(1); fn expected_log_events() -> usize { std::env::var("EXPECTED_LOG_EVENTS") @@ -70,25 +72,36 @@ fn reduce_to_data(payloads: Vec>) -> Vec { async fn validate() { trace_init(); - // Even with configuring docker service dependencies, we need a small buffer of time - // to ensure events flow through to fakeintake before asking for them - std::thread::sleep(std::time::Duration::from_secs(2)); - + // Retry until we have log payloads or hit max retries. + // This is to ensure events flow through to fakeintake before asking for them. info!("getting log payloads from agent-only pipeline"); - let mut agent_payloads = get_fakeintake_payloads::( - &fake_intake_agent_address(), - LOGS_ENDPOINT, - ) - .await - .payloads; - - // the logs endpoint receives an empty healthcheck payload in the beginning - if !agent_payloads.is_empty() { - agent_payloads.retain(|raw_payload| !raw_payload.data.as_array().unwrap().is_empty()) + let mut agent_payloads = Vec::new(); + for _ in 0..MAX_RETRIES { + agent_payloads = get_fakeintake_payloads::( + &fake_intake_agent_address(), + LOGS_ENDPOINT, + ) + .await + .payloads; + + if !agent_payloads.is_empty() { + break; + } + + info!("No valid payloads yet, retrying..."); + tokio::time::sleep(WAIT_INTERVAL).await; } - let mut agent_payloads = reduce_to_data(agent_payloads); + // If we still don't have valid payloads after retries, fail the test + assert!( + !agent_payloads.is_empty(), + "Failed to get valid log payloads from agent pipeline after {MAX_RETRIES} retries" + ); + + // The logs endpoint receives an empty healthcheck payload in the beginning + agent_payloads.retain(|raw_payload| !raw_payload.data.as_array().unwrap().is_empty()); + let mut agent_payloads = reduce_to_data(agent_payloads); common_assertions(&mut agent_payloads); info!("getting log payloads from agent-vector pipeline"); From 822ed0d3f6a752637a33c60b983bd158ad27cce6 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 14 May 2025 16:48:00 -0400 Subject: [PATCH 005/305] fix(azure service): revert deps bump (#23039) * fix(azure service): revert deps bump * also revert common config file changes --- .github/workflows/integration.yml | 3 +- Cargo.lock | 140 ++++++++++-------------------- Cargo.toml | 16 ++-- LICENSE-3rdparty.csv | 3 +- src/sinks/azure_common/config.rs | 12 ++- 5 files changed, 60 insertions(+), 114 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index e5374cc084e14..74eb0bebaa696 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -111,11 +111,10 @@ jobs: if: ${{ needs.setup.outputs.can_access_secrets == 'true' }} strategy: matrix: - # TODO: Add "azure" back one https://github.com/vectordotdev/vector/issues/22777 is fixed. # TODO: Add "splunk" back once https://github.com/vectordotdev/vector/issues/22379 is fixed. # If you modify this list, please also update the `int_tests` job in changes.yml. service: [ - "amqp", "appsignal", "axiom", "aws", "clickhouse", "databend", "datadog-agent", + "amqp", "appsignal", "axiom", "aws", "azure", "clickhouse", "databend", "datadog-agent", "datadog-logs", "datadog-metrics", "datadog-traces", "dnstap", "docker-logs", "elasticsearch", "eventstoredb", "fluent", "gcp", "greptimedb", "http-client", "influxdb", "kafka", "logstash", "loki", "mongodb", "nats", "nginx", "opentelemetry", "postgres", "prometheus", "pulsar", diff --git a/Cargo.lock b/Cargo.lock index e207d60a00ce8..24d055bbce81a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -407,18 +407,6 @@ dependencies = [ "futures-core", ] -[[package]] -name = "async-channel" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" -dependencies = [ - "concurrent-queue", - "event-listener-strategy", - "futures-core", - "pin-project-lite", -] - [[package]] name = "async-compression" version = "0.4.23" @@ -467,7 +455,7 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776" dependencies = [ - "async-channel 1.9.0", + "async-channel", "async-executor", "async-io 1.13.0", "async-lock 2.8.0", @@ -692,25 +680,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "async-process" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63255f1dc2381611000436537bbedfe83183faa303a5a0edaf191edef06526bb" -dependencies = [ - "async-channel 2.3.1", - "async-io 2.4.0", - "async-lock 3.4.0", - "async-signal", - "async-task", - "blocking", - "cfg-if", - "event-listener 5.3.1", - "futures-lite 2.6.0", - "rustix 0.38.40", - "tracing 0.1.41", -] - [[package]] name = "async-reactor-trait" version = "1.1.0" @@ -1537,110 +1506,94 @@ dependencies = [ [[package]] name = "azure_core" -version = "0.21.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b552ad43a45a746461ec3d3a51dfb6466b4759209414b439c165eb6a6b7729e" +checksum = "4ccd63c07d1fbfb3d4543d7ea800941bf5a30db1911b9b9e4db3b2c4210a434f" dependencies = [ "async-trait", - "base64 0.22.1", + "base64 0.21.7", "bytes 1.10.1", "dyn-clone", "futures 0.3.31", "getrandom 0.2.15", - "hmac", "http-types", - "once_cell", + "log", "paste", "pin-project", "quick-xml 0.31.0", "rand 0.8.5", - "reqwest 0.12.9", + "reqwest 0.11.26", "rustc_version 0.4.1", "serde", "serde_json", - "sha2", "time", - "tracing 0.1.41", "url", "uuid", ] [[package]] name = "azure_identity" -version = "0.21.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ddd80344317c40c04b603807b63a5cefa532f1b43522e72f480a988141f744" +checksum = "8bd7ea32ca7eb66ff4757f83baac702ff11d469e5de365b6bc6f79f9c25d3436" dependencies = [ "async-lock 3.4.0", - "async-process 2.3.0", "async-trait", "azure_core", "futures 0.3.31", + "log", "oauth2", "pin-project", "serde", + "serde_json", "time", - "tracing 0.1.41", + "tz-rs", "url", "uuid", ] [[package]] name = "azure_storage" -version = "0.21.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59f838159f4d29cb400a14d9d757578ba495ae64feb07a7516bf9e4415127126" +checksum = "83ca0a07f89fd72a006da4713e93af3d6c44a693e61a1c3c2e7985de39c182e8" dependencies = [ "RustyXML", - "async-lock 3.4.0", "async-trait", "azure_core", "bytes 1.10.1", + "futures 0.3.31", + "hmac", + "log", "serde", "serde_derive", + "serde_json", + "sha2", "time", - "tracing 0.1.41", "url", "uuid", ] [[package]] name = "azure_storage_blobs" -version = "0.21.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97e83c3636ae86d9a6a7962b2112e3b19eb3903915c50ce06ff54ff0a2e6a7e4" +checksum = "8096c04d370118323c42b2752aa1883e4880a56ef65239f317b359f263b6e194" dependencies = [ "RustyXML", "azure_core", "azure_storage", - "azure_svc_blobstorage", "bytes 1.10.1", "futures 0.3.31", + "log", "serde", "serde_derive", "serde_json", "time", - "tracing 0.1.41", "url", "uuid", ] -[[package]] -name = "azure_svc_blobstorage" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e6c6f20c5611b885ba94c7bae5e02849a267381aecb8aee577e8c35ff4064c6" -dependencies = [ - "azure_core", - "bytes 1.10.1", - "futures 0.3.31", - "log", - "once_cell", - "serde", - "serde_json", - "time", -] - [[package]] name = "backoff" version = "0.4.0" @@ -1815,7 +1768,7 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c36a4d0d48574b3dd360b4b7d95cc651d2b6557b6402848a27d4b228a473e2a" dependencies = [ - "async-channel 1.9.0", + "async-channel", "async-lock 2.8.0", "async-task", "fastrand 2.3.0", @@ -2510,6 +2463,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" +[[package]] +name = "const_fn" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f8a2ca5ac02d09563609681103aada9e1777d54fc57a5acd7a41404f9c93b6e" + [[package]] name = "convert_case" version = "0.4.0" @@ -3970,10 +3929,7 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5edaec856126859abb19ed65f39e90fea3a9574b9707f13539acf4abf7eb532" dependencies = [ - "fastrand 2.3.0", "futures-core", - "futures-io", - "parking", "pin-project-lite", ] @@ -4721,7 +4677,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad" dependencies = [ "anyhow", - "async-channel 1.9.0", + "async-channel", "base64 0.13.1", "futures-lite 1.13.0", "infer", @@ -4769,7 +4725,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.5.9", "tokio", "tower-service", "tracing 0.1.41", @@ -4957,22 +4913,6 @@ dependencies = [ "tokio-native-tls", ] -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes 1.10.1", - "http-body-util", - "hyper 1.4.1", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - [[package]] name = "hyper-util" version = "0.1.9" @@ -8591,7 +8531,7 @@ dependencies = [ "http-body 0.4.5", "hyper 0.14.28", "hyper-rustls 0.24.2", - "hyper-tls 0.5.0", + "hyper-tls", "ipnet", "js-sys", "log", @@ -8610,10 +8550,12 @@ dependencies = [ "tokio", "tokio-native-tls", "tokio-rustls 0.24.1", + "tokio-util", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "webpki-roots 0.25.2", "winreg", @@ -8634,14 +8576,12 @@ dependencies = [ "http-body-util", "hyper 1.4.1", "hyper-rustls 0.27.5", - "hyper-tls 0.6.0", "hyper-util", "ipnet", "js-sys", "log", "mime", "mime_guess", - "native-tls", "once_cell", "percent-encoding", "pin-project-lite", @@ -8654,7 +8594,6 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.1", "tokio", - "tokio-native-tls", "tokio-rustls 0.26.2", "tokio-util", "tower-service", @@ -9645,13 +9584,13 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13f2b548cd8447f8de0fdf1c592929f70f4fc7039a05e47404b0d096ec6987a1" dependencies = [ - "async-channel 1.9.0", + "async-channel", "async-executor", "async-fs", "async-io 1.13.0", "async-lock 2.8.0", "async-net", - "async-process 1.8.1", + "async-process", "blocking", "futures-lite 1.13.0", ] @@ -11244,6 +11183,15 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "tz-rs" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33851b15c848fad2cf4b105c6bb66eb9512b6f6c44a4b13f57c53c73c707e2b4" +dependencies = [ + "const_fn", +] + [[package]] name = "ua-parser" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 42c75a7de7a8d..c999e7b270d74 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -256,10 +256,10 @@ aws-smithy-runtime-api = { version = "1.7.3", default-features = false, optional aws-smithy-types = { version = "1.2.11", default-features = false, features = ["rt-tokio"], optional = true } # Azure -azure_core = { version = "0.21", default-features = false, features = ["hmac_rust", "enable_reqwest"], optional = true } -azure_identity = { version = "0.21", default-features = false, features = ["enable_reqwest"], optional = true } -azure_storage = { version = "0.21", default-features = false, optional = true } -azure_storage_blobs = { version = "0.21", default-features = false, optional = true } +azure_core = { version = "0.17", default-features = false, features = ["enable_reqwest"], optional = true } +azure_identity = { version = "0.17", default-features = false, features = ["enable_reqwest"], optional = true } +azure_storage = { version = "0.17", default-features = false, optional = true } +azure_storage_blobs = { version = "0.17", default-features = false, optional = true } # OpenDAL opendal = { version = "0.53", default-features = false, features = ["services-webhdfs"], optional = true } @@ -428,10 +428,10 @@ openssl-src = { version = "300", default-features = false, features = ["force-en approx = "0.5.1" assert_cmd = { version = "2.0.17", default-features = false } aws-smithy-runtime = { version = "1.8.3", default-features = false, features = ["tls-rustls"] } -azure_core = { version = "0.21", default-features = false, features = ["enable_reqwest", "azurite_workaround"] } -azure_identity = { version = "0.21", default-features = false, features = ["enable_reqwest"] } -azure_storage_blobs = { version = "0.21", default-features = false, features = ["azurite_workaround"] } -azure_storage = { version = "0.21", default-features = false } +azure_core = { version = "0.17", default-features = false, features = ["enable_reqwest", "azurite_workaround"] } +azure_identity = { version = "0.17", default-features = false, features = ["enable_reqwest"] } +azure_storage_blobs = { version = "0.17", default-features = false, features = ["azurite_workaround"] } +azure_storage = { version = "0.17", default-features = false } base64 = "0.22.1" criterion = { version = "0.5.1", features = ["html_reports", "async_tokio"] } itertools = { version = "0.14.0", default-features = false, features = ["use_alloc"] } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 7f7cb766ac1b3..dbe0a174c9ec3 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -84,7 +84,6 @@ azure_core,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft Corp. azure_identity,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft Corp. azure_storage,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft Corp. azure_storage_blobs,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft Corp. -azure_svc_blobstorage,https://github.com/azure/azure-sdk-for-rust,MIT,The azure_svc_blobstorage Authors backoff,https://github.com/ihrwein/backoff,MIT OR Apache-2.0,Tibor Benke backon,https://github.com/Xuanwo/backon,Apache-2.0,The backon Authors backtrace,https://github.com/rust-lang/backtrace-rs,MIT OR Apache-2.0,The Rust Project Developers @@ -146,6 +145,7 @@ community-id,https://github.com/traceflight/rs-community-id,MIT OR Apache-2.0,Ju compact_str,https://github.com/ParkMyCar/compact_str,MIT,Parker Timmerman concurrent-queue,https://github.com/smol-rs/concurrent-queue,Apache-2.0 OR MIT,"Stjepan Glavina , Taiki Endo , John Nunley " const-oid,https://github.com/RustCrypto/formats/tree/master/const-oid,Apache-2.0 OR MIT,RustCrypto Developers +const_fn,https://github.com/taiki-e/const_fn,Apache-2.0 OR MIT,The const_fn Authors convert_case,https://github.com/rutrum/convert-case,MIT,David Purdum convert_case,https://github.com/rutrum/convert-case,MIT,rutrum cookie-factory,https://github.com/rust-bakery/cookie-factory,MIT,"Geoffroy Couprie , Pierre Chifflier " @@ -677,6 +677,7 @@ twox-hash,https://github.com/shepmaster/twox-hash,MIT,Jake Goulding , Chris Morgan " typenum,https://github.com/paholg/typenum,MIT OR Apache-2.0,"Paho Lurie-Gregg , Andre Bogus " typetag,https://github.com/dtolnay/typetag,MIT OR Apache-2.0,David Tolnay +tz-rs,https://github.com/x-hgg-x/tz-rs,MIT OR Apache-2.0,x-hgg-x ua-parser,https://github.com/ua-parser/uap-rust,Apache-2.0,The ua-parser Authors ucd-trie,https://github.com/BurntSushi/ucd-generate,MIT OR Apache-2.0,Andrew Gallant unarray,https://github.com/cameron1024/unarray,MIT OR Apache-2.0,The unarray Authors diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index b0d1dbd9cf5d3..5e10ea797c305 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -1,6 +1,7 @@ use std::sync::Arc; use azure_core::{error::HttpError, RetryOptions}; +use azure_identity::{AutoRefreshingTokenCredential, DefaultAzureCredential}; use azure_storage::{prelude::*, CloudLocation, ConnectionString}; use azure_storage_blobs::{blob::operations::PutBlockBlobResponse, prelude::*}; use bytes::Bytes; @@ -141,7 +142,6 @@ pub fn build_client( // generate the storage API endpoint Some(uri) => ClientBuilder::with_location( CloudLocation::Custom { - account: "".to_string(), uri: uri.to_string(), }, connection_string.storage_credentials()?, @@ -160,17 +160,15 @@ pub fn build_client( .container_client(container_name); } (None, Some(storage_account_p)) => { - let creds = azure_identity::create_default_credential()?; - let storage_credentials = StorageCredentials::token_credential(creds); + let creds = std::sync::Arc::new(DefaultAzureCredential::default()); + let auto_creds = std::sync::Arc::new(AutoRefreshingTokenCredential::new(creds)); + let storage_credentials = StorageCredentials::token_credential(auto_creds); client = match endpoint { // If a blob_endpoint is provided in the configuration, use it with a Custom // CloudLocation, to allow overriding the blob storage API endpoint Some(endpoint) => ClientBuilder::with_location( - CloudLocation::Custom { - account: storage_account_p, - uri: endpoint, - }, + CloudLocation::Custom { uri: endpoint }, storage_credentials, ), // Use the storage_account configuration parameter and assume we are in Azure From 346b1ba993f7cdb4e1608fcfb730267c90a85496 Mon Sep 17 00:00:00 2001 From: Miro Prasil Date: Wed, 14 May 2025 19:46:54 +0000 Subject: [PATCH 006/305] fix(codecs): gelf encoder error message (#23021) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(gelf encoder): gelf encoder error message The error message in case the field name contains incorrect characters is misleading and can cause quite an confusion. The issue can be reproduced with following Vector config: ```yaml sources: test: type: demo_logs format: shuffle decoding: codec: json count: 1 lines: - '{"foo:baz": "baz", "short_message": "x"}' sinks: console: type: console encoding: codec: gelf inputs: - test ``` With current version of vector I got: ```sh ❯ vector -c /tmp/vector.yaml -q 2025-05-09T19:55:47.827681Z ERROR sink{component_kind="sink" component_id=console component_type=console}: vector::internal_events::codecs: Failed serializing frame. error=LogEvent does not contain required field: "foo:baz" error_code="encoder_serialize" error_type="encoder_failed" stage="sending" internal_log_rate_limit=true 2025-05-09T19:55:47.827769Z ERROR sink{component_kind="sink" component_id=console component_type=console}: vector_common::internal_event::component_events_dropped: Events dropped intentional=false count=1 reason="Failed serializing frame." internal_log_rate_limit=true 2025-05-09T19:55:47.827800Z ERROR sink{component_kind="sink" component_id=console component_type=console}: vector::topology: An error occurred that Vector couldn't handle: the task completed with an error. ``` The part where it says `LogEvent does not contain required field: "foo:baz"` is quite confusing, because the field is there and it's certainly not required. (though that might not be obvious for less unusual field name) It looks like the error message was there since early gelf implementation and not really intentional. (possibly a placeholder that was never replaced) The fixed version generates pretty straightforward message: ``` ❯ vector -c /tmp/vector.yaml -q 2025-05-09T20:22:13.387678Z ERROR sink{component_kind="sink" component_id=console component_type=console}: vector::internal_events::codecs: Failed serializing frame. error=LogEvent contains field with invalid character in name: "foo:baz" error_code="encoder_serialize" error_type="encoder_failed" stage="sending" internal_log_rate_limit=true 2025-05-09T20:22:13.388008Z ERROR sink{component_kind="sink" component_id=console component_type=console}: vector_common::internal_event::component_events_dropped: Events dropped intentional=false count=1 reason="Failed serializing frame." internal_log_rate_limit=true 2025-05-09T20:22:13.388544Z ERROR sink{component_kind="sink" component_id=console component_type=console}: vector::topology: An error occurred that Vector couldn't handle: the task completed with an error. ``` This should hopefully be much easier to troubleshoot. * Implement changes suggested in PR - Rename the Snafu struct to `InvalidField` - Make the Error message more specific by showing the pattern used to check field name - Rename the changelog fragment to match the expected '..md' format. * Fix rustfmt issue --- changelog.d/gelf-error.fix.md | 3 +++ lib/codecs/src/encoding/format/gelf.rs | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 changelog.d/gelf-error.fix.md diff --git a/changelog.d/gelf-error.fix.md b/changelog.d/gelf-error.fix.md new file mode 100644 index 0000000000000..9f0133e6d550c --- /dev/null +++ b/changelog.d/gelf-error.fix.md @@ -0,0 +1,3 @@ +Fixed misleading error message for invalid field name in gelf encoder. + +authors: mprasil diff --git a/lib/codecs/src/encoding/format/gelf.rs b/lib/codecs/src/encoding/format/gelf.rs index be5c861548822..ddd08e7d151f4 100644 --- a/lib/codecs/src/encoding/format/gelf.rs +++ b/lib/codecs/src/encoding/format/gelf.rs @@ -25,6 +25,12 @@ use vector_core::{ pub enum GelfSerializerError { #[snafu(display(r#"LogEvent does not contain required field: "{}""#, field))] MissingField { field: KeyString }, + #[snafu(display( + r#"LogEvent contains field with invalid name not matching pattern '{}': "{}""#, + pattern, + field, + ))] + InvalidField { field: KeyString, pattern: String }, #[snafu(display( r#"LogEvent contains a value with an invalid type. field = "{}" type = "{}" expected type = "{}""#, field, @@ -190,8 +196,9 @@ fn coerce_field_names_and_values( _ => { // additional fields must be only word chars, dashes and periods. if !VALID_FIELD_REGEX.is_match(field) { - return MissingFieldSnafu { + return InvalidFieldSnafu { field: field.clone(), + pattern: VALID_FIELD_REGEX.to_string(), } .fail() .map_err(|e| e.to_string().into()); From 1a1050dbe585bf4a79c37b2183d075ac49bbf254 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 14 May 2025 16:10:33 -0400 Subject: [PATCH 007/305] fix(ci): request ARG in every build stage (#23049) --- scripts/e2e/Dockerfile | 5 ++--- tilt/Dockerfile | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/e2e/Dockerfile b/scripts/e2e/Dockerfile index 495c69275cad4..d709f4158e29d 100644 --- a/scripts/e2e/Dockerfile +++ b/scripts/e2e/Dockerfile @@ -1,8 +1,7 @@ ARG RUST_VERSION ARG FEATURES -ARG DEBIAN_RELEASE=slim-bookworm -FROM docker.io/rust:${RUST_VERSION}-${DEBIAN_RELEASE} +FROM docker.io/rust:${RUST_VERSION}-slim-bookworm RUN apt-get update && apt-get -y --no-install-recommends install \ build-essential \ @@ -28,7 +27,7 @@ RUN git clone https://github.com/rui314/mold.git \ && cmake --build . -j $(nproc) \ && cmake --install . -RUN rustup run "${RUST_VERSION}" cargo install cargo-nextest --version 0.9.72 --locked +RUN cargo install cargo-nextest --version 0.9.72 --locked COPY scripts/environment/install-protoc.sh / COPY tests/data/ca/certs /certs diff --git a/tilt/Dockerfile b/tilt/Dockerfile index dcdcaca0abe7c..379979efd159d 100644 --- a/tilt/Dockerfile +++ b/tilt/Dockerfile @@ -31,6 +31,7 @@ RUN --mount=type=cache,target=/vector/target \ # # TARGET # +ARG DEBIAN_RELEASE FROM docker.io/debian:${DEBIAN_RELEASE}-slim RUN apt-get update && apt-get -y --no-install-recommends install zlib1g && rm -rf /var/lib/apt/lists/* COPY --from=builder /vector/vector /usr/bin/vector From 8f10b37a38daaec7bf48f91f00ebea6afa4eb183 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 15 May 2025 08:37:20 -0400 Subject: [PATCH 008/305] chore(ci): disable E2E datadog-logs test (#23055) --- .github/workflows/e2e.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e5d322d2b8cc9..17df0426cb9ed 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -82,14 +82,15 @@ jobs: echo "PR_HAS_ACCESS_TO_SECRETS=false" >> "$GITHUB_ENV" fi - - if: (github.event_name == 'schedule' || needs.changes.outputs.all-e2e == 'true' || needs.changes.outputs.e2e-datadog-logs == 'true') && - (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') - name: e2e-datadog-logs - uses: nick-fields/retry@v3 - with: - timeout_minutes: 35 - max_attempts: 3 - command: bash scripts/int-e2e-test.sh e2e datadog-logs + # Re-enable when https://github.com/vectordotdev/vector/issues/23054 is fixed. + # - if: (github.event_name == 'schedule' || needs.changes.outputs.all-e2e == 'true' || needs.changes.outputs.e2e-datadog-logs == 'true') && + # (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') + # name: e2e-datadog-logs + # uses: nick-fields/retry@v3 + # with: + # timeout_minutes: 35 + # max_attempts: 3 + # command: bash scripts/int-e2e-test.sh e2e datadog-logs - if: (github.event_name == 'schedule' || needs.changes.outputs.all-e2e == 'true' || needs.changes.outputs.e2e-datadog-metrics == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') From c1cf664ecbbdf5abe945a0b0c74001829a25fea5 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 15 May 2025 08:54:10 -0400 Subject: [PATCH 009/305] chore(ci): temporarily disable failing AMQP tests (#23057) --- .github/workflows/integration.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 74eb0bebaa696..35ba4e186dc5a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -111,10 +111,11 @@ jobs: if: ${{ needs.setup.outputs.can_access_secrets == 'true' }} strategy: matrix: + # TODO: Add "amqp" back once https://github.com/vectordotdev/vector/issues/22379 is fixed. # TODO: Add "splunk" back once https://github.com/vectordotdev/vector/issues/22379 is fixed. # If you modify this list, please also update the `int_tests` job in changes.yml. service: [ - "amqp", "appsignal", "axiom", "aws", "azure", "clickhouse", "databend", "datadog-agent", + "appsignal", "axiom", "aws", "azure", "clickhouse", "databend", "datadog-agent", "datadog-logs", "datadog-metrics", "datadog-traces", "dnstap", "docker-logs", "elasticsearch", "eventstoredb", "fluent", "gcp", "greptimedb", "http-client", "influxdb", "kafka", "logstash", "loki", "mongodb", "nats", "nginx", "opentelemetry", "postgres", "prometheus", "pulsar", From f2b4e29191e5cbdd6d94111dec1122dc3f9da5d7 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 15 May 2025 09:13:27 -0400 Subject: [PATCH 010/305] chore(ci): remove redundant rustup prefix (#23059) --- scripts/integration/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/integration/Dockerfile b/scripts/integration/Dockerfile index 07ffd19738823..587897d20bd7e 100644 --- a/scripts/integration/Dockerfile +++ b/scripts/integration/Dockerfile @@ -16,7 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ git \ && rm -rf /var/lib/apt/lists/* -RUN rustup run "${RUST_VERSION}" cargo install cargo-nextest --version 0.9.72 --locked +RUN cargo install cargo-nextest --version 0.9.72 --locked COPY scripts/environment/install-protoc.sh / COPY tests/data/ca/certs /certs From 4643acd78cdaf051c3088f7fc46be1488c109563 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 15 May 2025 09:22:23 -0400 Subject: [PATCH 011/305] chore(ci): provide default RUST_VERSION if one is not specified (#23060) --- scripts/e2e/Dockerfile | 2 +- scripts/integration/Dockerfile | 2 +- tilt/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/e2e/Dockerfile b/scripts/e2e/Dockerfile index d709f4158e29d..8de9c2834d67a 100644 --- a/scripts/e2e/Dockerfile +++ b/scripts/e2e/Dockerfile @@ -1,4 +1,4 @@ -ARG RUST_VERSION +ARG RUST_VERSION=1.85 ARG FEATURES FROM docker.io/rust:${RUST_VERSION}-slim-bookworm diff --git a/scripts/integration/Dockerfile b/scripts/integration/Dockerfile index 587897d20bd7e..a50808aba751b 100644 --- a/scripts/integration/Dockerfile +++ b/scripts/integration/Dockerfile @@ -1,4 +1,4 @@ -ARG RUST_VERSION +ARG RUST_VERSION=1.85 FROM docker.io/rust:${RUST_VERSION}-slim-bookworm RUN apt-get update && apt-get install -y --no-install-recommends \ diff --git a/tilt/Dockerfile b/tilt/Dockerfile index 379979efd159d..2831d165d4166 100644 --- a/tilt/Dockerfile +++ b/tilt/Dockerfile @@ -1,4 +1,4 @@ -ARG RUST_VERSION +ARG RUST_VERSION=1.85 ARG DEBIAN_RELEASE=bookworm # Features required for both Agent and Aggregator Helm chart configurations ARG FEATURES=api,api-client,sources-datadog_agent,sources-fluent,sources-host_metrics,sources-internal_metrics,sources-kubernetes_logs,sources-logstash,sources-splunk_hec,sources-statsd,sources-syslog,sources-vector,sinks-console,sinks-prometheus,sinks-vector From 84e5fe9bec1221c58e663602a2aef10e90c146d5 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 15 May 2025 09:57:56 -0400 Subject: [PATCH 012/305] chore(ci): bump nextest to latest version (#23058) --- scripts/e2e/Dockerfile | 2 +- scripts/environment/bootstrap-windows-2022.ps1 | 2 +- scripts/environment/prepare.sh | 4 ++-- scripts/integration/Dockerfile | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/e2e/Dockerfile b/scripts/e2e/Dockerfile index 8de9c2834d67a..c0726c94e5935 100644 --- a/scripts/e2e/Dockerfile +++ b/scripts/e2e/Dockerfile @@ -27,7 +27,7 @@ RUN git clone https://github.com/rui314/mold.git \ && cmake --build . -j $(nproc) \ && cmake --install . -RUN cargo install cargo-nextest --version 0.9.72 --locked +RUN cargo install cargo-nextest --version 0.9.95 --locked COPY scripts/environment/install-protoc.sh / COPY tests/data/ca/certs /certs diff --git a/scripts/environment/bootstrap-windows-2022.ps1 b/scripts/environment/bootstrap-windows-2022.ps1 index 27b35d9ff55b9..7af1123b2df35 100644 --- a/scripts/environment/bootstrap-windows-2022.ps1 +++ b/scripts/environment/bootstrap-windows-2022.ps1 @@ -10,7 +10,7 @@ echo "CARGO_BUILD_JOBS=$N_JOBS" | Out-File -FilePath $env:GITHUB_ENV -Encoding u if ($env:RELEASE_BUILDER -ne "true") { # Ensure we have cargo-next test installed. - rustup run stable cargo install cargo-nextest --version 0.9.72 --locked + rustup run stable cargo install cargo-nextest --version 0.9.95 --locked } # Enable retries to avoid transient network issues. diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index f7b0f2508516b..a42daaf8c427d 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -11,8 +11,8 @@ fi if [[ "$(cross --version | grep cross)" != "cross 0.2.5" ]] ; then rustup run stable cargo install cross --version 0.2.5 --force --locked fi -if [[ "$(cargo-nextest --version)" != "cargo-nextest 0.9.72" ]] ; then - rustup run stable cargo install cargo-nextest --version 0.9.72 --force --locked +if [[ "$(cargo-nextest --version)" != "cargo-nextest 0.9.95" ]] ; then + rustup run stable cargo install cargo-nextest --version 0.9.95 --force --locked fi if [[ "$(cargo-deny --version)" != "cargo-deny 0.16.1" ]] ; then rustup run stable cargo install cargo-deny --version 0.16.1 --force --locked diff --git a/scripts/integration/Dockerfile b/scripts/integration/Dockerfile index a50808aba751b..3e6b68dbfb669 100644 --- a/scripts/integration/Dockerfile +++ b/scripts/integration/Dockerfile @@ -16,7 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ git \ && rm -rf /var/lib/apt/lists/* -RUN cargo install cargo-nextest --version 0.9.72 --locked +RUN cargo install cargo-nextest --version 0.9.95 --locked COPY scripts/environment/install-protoc.sh / COPY tests/data/ca/certs /certs From 2c63454e394cdd18451bd3432b87f1b65d516ede Mon Sep 17 00:00:00 2001 From: Burkov Egor Date: Thu, 15 May 2025 16:07:01 +0300 Subject: [PATCH 013/305] fix(parsing): Handling wrong timestamps in dnstap-parser (#23048) * fix(parsing): Handling wrong timestamps in dnstap-parser * Add tests and changelog --- .../23048_fix_arith_logic_dnstap_parse.fix.md | 3 ++ lib/dnstap-parser/src/parser.rs | 54 +++++++++++++++++-- 2 files changed, 52 insertions(+), 5 deletions(-) create mode 100644 changelog.d/23048_fix_arith_logic_dnstap_parse.fix.md diff --git a/changelog.d/23048_fix_arith_logic_dnstap_parse.fix.md b/changelog.d/23048_fix_arith_logic_dnstap_parse.fix.md new file mode 100644 index 0000000000000..1a213d2bd4e4a --- /dev/null +++ b/changelog.d/23048_fix_arith_logic_dnstap_parse.fix.md @@ -0,0 +1,3 @@ +Added checks for integer operations with timestamps which used in `dnstap-parser::DnstapParser::parse`. + +authors: wooffie diff --git a/lib/dnstap-parser/src/parser.rs b/lib/dnstap-parser/src/parser.rs index 6a950c1a152b7..939f45a0b340d 100644 --- a/lib/dnstap-parser/src/parser.rs +++ b/lib/dnstap-parser/src/parser.rs @@ -236,7 +236,7 @@ impl DnstapParser { dnstap_message_type_id, dnstap_message.query_message.as_ref(), &DNSTAP_MESSAGE_REQUEST_TYPE_IDS, - ); + )?; } if let Some(response_time_sec) = dnstap_message.response_time_sec { @@ -248,7 +248,7 @@ impl DnstapParser { dnstap_message_type_id, dnstap_message.response_message.as_ref(), &DNSTAP_MESSAGE_RESPONSE_TYPE_IDS, - ); + )?; } DnstapParser::parse_dnstap_message_type( @@ -366,10 +366,29 @@ impl DnstapParser { dnstap_message_type_id: i32, message: Option<&Vec>, type_ids: &HashSet, - ) { + ) -> Result<()> { + if time_sec > i64::MAX as u64 { + return Err(Error::from("Cannot parse timestamp")); + } + let (time_in_nanosec, query_time_nsec) = match time_nsec { - Some(nsec) => (time_sec as i64 * 1_000_000_000_i64 + nsec as i64, nsec), - None => (time_sec as i64 * 1_000_000_000_i64, 0), + Some(nsec) => { + if let Some(time_in_ns) = (time_sec as i64) + .checked_mul(1_000_000_000) + .and_then(|v| v.checked_add(nsec as i64)) + { + (time_in_ns, nsec) + } else { + return Err(Error::from("Cannot parse timestamp")); + } + } + None => { + if let Some(time_in_ns) = (time_sec as i64).checked_mul(1_000_000_000) { + (time_in_ns, 0) + } else { + return Err(Error::from("Cannot parse timestamp")); + } + } }; if type_ids.contains(&dnstap_message_type_id) { @@ -392,6 +411,7 @@ impl DnstapParser { "ns", ); } + Ok(()) } fn parse_dnstap_message_socket_family<'a>( @@ -1321,6 +1341,30 @@ mod tests { assert!(e.to_string().contains("Protobuf message")); } + #[test] + fn test_parse_dnstap_data_with_invalid_timestamp() { + fn test_one_timestamp_parse(time_sec: u64, time_nsec: Option) -> Result<()> { + let mut event = LogEvent::default(); + let root = owned_value_path!(); + let type_ids = HashSet::new(); + DnstapParser::parse_dnstap_message_time( + &mut event, &root, time_sec, time_nsec, 0, None, &type_ids, + ) + } + // okay case + assert!(test_one_timestamp_parse(1337, Some(42)).is_ok()); + // overflow in cast (u64 -> i64) + assert!(test_one_timestamp_parse(u64::MAX, Some(42)).is_err()); + assert!(test_one_timestamp_parse(u64::MAX, None).is_err()); + // overflow in multiplication + assert!(test_one_timestamp_parse(i64::MAX as u64, Some(42)).is_err()); + assert!(test_one_timestamp_parse(i64::MAX as u64, None).is_err()); + // overflow in add + assert!( + test_one_timestamp_parse((i64::MAX / 1_000_000_000) as u64, Some(u32::MAX)).is_err() + ) + } + #[test] fn test_get_socket_family_name() { assert_eq!("INET", to_socket_family_name(1).unwrap()); From ace0dd21b542cfd522a18f3f3943a643a91e0e79 Mon Sep 17 00:00:00 2001 From: Burkov Egor Date: Thu, 15 May 2025 16:42:26 +0300 Subject: [PATCH 014/305] fix(parsing): Handle underflow in A6 parsing (#23047) * fix(parsing): Handle underflow in A6 parsing * Add tests and changelog * Change changelog filename for spelling --- ...47_add_a6_prefix_check_in_dns_parsing.fix.md | 3 +++ lib/dnsmsg-parser/src/dns_message_parser.rs | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 changelog.d/23047_add_a6_prefix_check_in_dns_parsing.fix.md diff --git a/changelog.d/23047_add_a6_prefix_check_in_dns_parsing.fix.md b/changelog.d/23047_add_a6_prefix_check_in_dns_parsing.fix.md new file mode 100644 index 0000000000000..7de9609bce5d7 --- /dev/null +++ b/changelog.d/23047_add_a6_prefix_check_in_dns_parsing.fix.md @@ -0,0 +1,3 @@ +Fixed a bug that allowed a record to be transmitted with an IPv6 prefix larger than allowed (128) in `dnsmsg_parser::DnsMessageParser` parse functions. + +authors: wooffie diff --git a/lib/dnsmsg-parser/src/dns_message_parser.rs b/lib/dnsmsg-parser/src/dns_message_parser.rs index f7d6ba033dc45..fadb27109ae63 100644 --- a/lib/dnsmsg-parser/src/dns_message_parser.rs +++ b/lib/dnsmsg-parser/src/dns_message_parser.rs @@ -263,6 +263,11 @@ impl DnsMessageParser { let mut decoder = BinDecoder::new(raw_rdata); let prefix = parse_u8(&mut decoder)?; let ipv6_address = { + if prefix > 128 { + return Err(DnsMessageParserError::SimpleError { + cause: String::from("IPV6 prefix can't be greater than 128."), + }); + } let address_length = (128 - prefix) / 8; let mut address_vec = parse_vec(&mut decoder, address_length)?; if address_vec.len() < 16 { @@ -1534,6 +1539,18 @@ mod tests { .is_err()); } + #[test] + fn test_parse_bad_prefix_value() { + // this testcase have prefix value of 160, + let raw_dns_message = "oAAAMgABAAAAAAABAAABAAAAACYAAC8BAAAAAaAAAAAAAA=="; + let raw_query_message = BASE64 + .decode(raw_dns_message.as_bytes()) + .expect("Invalid base64 encoded data."); + assert!(DnsMessageParser::new(raw_query_message) + .parse_as_query_message() + .is_err()); + } + #[test] fn test_parse_as_update_message() { let raw_dns_message = "xjUoAAABAAAAAQAAB2V4YW1wbGUDY29tAAAGAAECaDXADAD/AP8AAAAAAAA="; From 9cdab60b320ad14c06214b4cf7b7e0f8a441f373 Mon Sep 17 00:00:00 2001 From: nekorro Date: Thu, 15 May 2025 17:26:16 +0300 Subject: [PATCH 015/305] feat(remap transform): make `--watch-config` watch external VRL files in `remap` transforms (#23010) * feat(transforms): add files_to_watch support for transforms Invoke the `files_to_watch` function for applicable transforms to collect external file paths that should be watched for changes. * feat(remap transfrom): add files_to_watch to track VRL files Add a `files_to_watch` function to the remap transform to collect file paths from the `file` and `files` fields. This allows to watch these files for changes and trigger reloads when the VRL code is modified. * fix(config): relative path handling for external files when `--watch-config` enabled Relative paths to external files in the configuration are now correctly resolved when using --watch-config. Previously, changes to such files not trigger a reload if the path provided in the config was relative. This fix ensures that relative paths are canonicalized, allowing file changes to be properly detected. --- .../9819_watch_remap_external_vrl.enhancement.md | 3 +++ src/app.rs | 11 +++++++++++ src/config/mod.rs | 10 ++++++++-- src/config/transform.rs | 6 ++++++ src/transforms/remap.rs | 7 +++++++ 5 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 changelog.d/9819_watch_remap_external_vrl.enhancement.md diff --git a/changelog.d/9819_watch_remap_external_vrl.enhancement.md b/changelog.d/9819_watch_remap_external_vrl.enhancement.md new file mode 100644 index 0000000000000..a77b4c1809d7f --- /dev/null +++ b/changelog.d/9819_watch_remap_external_vrl.enhancement.md @@ -0,0 +1,3 @@ +Files specified in the `file` and `files` fields of `remap` transforms are now watched when `--watch-config` is enabled. Changes to these files automatically trigger a configuration reload, so there's no need to restart Vector. + +authors: nekorro diff --git a/src/app.rs b/src/app.rs index 3707071041510..04023ebba4bab 100644 --- a/src/app.rs +++ b/src/app.rs @@ -533,6 +533,13 @@ pub async fn load_configs( let mut watched_component_paths = Vec::new(); if let Some(watcher_conf) = watcher_conf { + for (name, transform) in config.transforms() { + let files = transform.inner.files_to_watch(); + let component_config = + ComponentConfig::new(files.into_iter().cloned().collect(), name.clone()); + watched_component_paths.push(component_config); + } + for (name, sink) in config.sinks() { let files = sink.inner.files_to_watch(); let component_config = @@ -544,6 +551,10 @@ pub async fn load_configs( message = "Starting watcher.", paths = ?watched_paths ); + info!( + message = "Components to watch.", + paths = ?watched_component_paths + ); // Start listening for config changes. config::watcher::spawn_thread( diff --git a/src/config/mod.rs b/src/config/mod.rs index d42d87c9753af..8da0ce75c4255 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -2,6 +2,7 @@ use std::{ collections::{HashMap, HashSet}, fmt::{self, Display, Formatter}, + fs, hash::Hash, net::SocketAddr, path::PathBuf, @@ -83,9 +84,14 @@ pub struct ComponentConfig { } impl ComponentConfig { - pub const fn new(config_paths: Vec, component_key: ComponentKey) -> Self { + pub fn new(config_paths: Vec, component_key: ComponentKey) -> Self { + let canonicalized_paths = config_paths + .into_iter() + .filter_map(|p| fs::canonicalize(p).ok()) + .collect(); + Self { - config_paths, + config_paths: canonicalized_paths, component_key, } } diff --git a/src/config/transform.rs b/src/config/transform.rs index 4ab7e21f99573..b1a32534edc8a 100644 --- a/src/config/transform.rs +++ b/src/config/transform.rs @@ -1,5 +1,6 @@ use std::cell::RefCell; use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; use async_trait::async_trait; use dyn_clone::DynClone; @@ -253,6 +254,11 @@ pub trait TransformConfig: DynClone + NamedComponent + core::fmt::Debug + Send + fn nestable(&self, _parents: &HashSet<&'static str>) -> bool { true } + + /// Gets the files to watch to trigger reload + fn files_to_watch(&self) -> Vec<&PathBuf> { + Vec::new() + } } dyn_clone::clone_trait_object!(TransformConfig); diff --git a/src/transforms/remap.rs b/src/transforms/remap.rs index 27973b3eb40f3..9f92a9a61b26b 100644 --- a/src/transforms/remap.rs +++ b/src/transforms/remap.rs @@ -383,6 +383,13 @@ impl TransformConfig for RemapConfig { fn enable_concurrency(&self) -> bool { true } + + fn files_to_watch(&self) -> Vec<&PathBuf> { + self.file + .iter() + .chain(self.files.iter().flatten()) + .collect() + } } #[derive(Debug, Clone)] From 914d1568dd1363d2d95b9e8dc4b2b1a808885d12 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 16 May 2025 12:59:37 -0400 Subject: [PATCH 016/305] chore(ci): install mold (no need to build it) (#23064) --- scripts/e2e/Dockerfile | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/scripts/e2e/Dockerfile b/scripts/e2e/Dockerfile index c0726c94e5935..3af5d83489c27 100644 --- a/scripts/e2e/Dockerfile +++ b/scripts/e2e/Dockerfile @@ -16,16 +16,8 @@ RUN apt-get update && apt-get -y --no-install-recommends install \ libxxhash-dev \ unzip \ zlib1g-dev \ - zlib1g - -RUN git clone https://github.com/rui314/mold.git \ - && mkdir mold/build \ - && cd mold/build \ - && git checkout v2.0.0 \ - && ../install-build-deps.sh \ - && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=c++ .. \ - && cmake --build . -j $(nproc) \ - && cmake --install . + zlib1g \ + mold RUN cargo install cargo-nextest --version 0.9.95 --locked @@ -40,6 +32,6 @@ ARG FEATURES RUN --mount=type=cache,target=/vector/target \ --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git \ - /usr/local/bin/mold -run cargo build --tests --lib --bin vector \ + /usr/bin/mold -run cargo build --tests --lib --bin vector \ --no-default-features --features $FEATURES && \ cp target/debug/vector /usr/bin/vector From 1d232fb5057786b704b5ac953aa0fd7a74493c26 Mon Sep 17 00:00:00 2001 From: Nick Wang Date: Fri, 16 May 2025 10:53:31 -0700 Subject: [PATCH 017/305] enhancement(enriching): add from and to date search to enrichment tables (#22926) * add from and to to evaluate_condition and add_index * add FromDate and ToDate to enum Condition * add conditions to row_equals and unit tests * update website * add changelog entry * Update comment in lib/enrichment/src/lib.rs Co-authored-by: Pavlos Rontidis * Update comment in lib/enrichment/src/lib.rs Co-authored-by: Pavlos Rontidis * Update changelog.d/22926_enrichment_function_single_bounded_date_range_search.enhancement.md Co-authored-by: Pavlos Rontidis * remove trailing space fixes failing check https://github.com/vectordotdev/vector/actions/runs/14868558637/job/41959755638 --------- Co-authored-by: Pavlos Rontidis --- ...e_bounded_date_range_search.enhancement.md | 3 + lib/enrichment/src/lib.rs | 10 ++ lib/enrichment/src/vrl_util.rs | 23 ++- src/enrichment_tables/file.rs | 142 +++++++++++++++++- website/cue/reference/remap/functions.cue | 2 +- 5 files changed, 177 insertions(+), 3 deletions(-) create mode 100644 changelog.d/22926_enrichment_function_single_bounded_date_range_search.enhancement.md diff --git a/changelog.d/22926_enrichment_function_single_bounded_date_range_search.enhancement.md b/changelog.d/22926_enrichment_function_single_bounded_date_range_search.enhancement.md new file mode 100644 index 0000000000000..e20a27c160a15 --- /dev/null +++ b/changelog.d/22926_enrichment_function_single_bounded_date_range_search.enhancement.md @@ -0,0 +1,3 @@ +The [enrichment functions](https://vector.dev/docs/reference/vrl/functions/#enrichment-functions) now support bounded date range filtering using optional `from` and `to` parameters. There are no changes to the function signatures. + +authors: nzxwang diff --git a/lib/enrichment/src/lib.rs b/lib/enrichment/src/lib.rs index 52f2a547f50f1..91195aa77338b 100644 --- a/lib/enrichment/src/lib.rs +++ b/lib/enrichment/src/lib.rs @@ -26,6 +26,16 @@ pub enum Condition<'a> { from: chrono::DateTime, to: chrono::DateTime, }, + /// The date in the field is greater than or equal to `from`. + FromDate { + field: &'a str, + from: chrono::DateTime, + }, + /// The date in the field is less than or equal to `to`. + ToDate { + field: &'a str, + to: chrono::DateTime, + }, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/lib/enrichment/src/vrl_util.rs b/lib/enrichment/src/vrl_util.rs index 61e7044c53c99..df40824cb17c6 100644 --- a/lib/enrichment/src/vrl_util.rs +++ b/lib/enrichment/src/vrl_util.rs @@ -55,6 +55,22 @@ pub(crate) fn evaluate_condition(key: &str, value: Value) -> ExpressionResult Condition::FromDate { + field: key, + from: *map + .get("from") + .expect("should contain from") + .as_timestamp() + .ok_or("from in condition must be a timestamp")?, + }, + Value::Object(map) if map.contains_key("to") => Condition::ToDate { + field: key, + to: *map + .get("to") + .expect("should contain to") + .as_timestamp() + .ok_or("to in condition must be a timestamp")?, + }, _ => Condition::Equals { field: key, value }, }) } @@ -71,7 +87,12 @@ pub(crate) fn add_index( .filter_map(|(field, value)| match value { expression::Expr::Container(expression::Container { variant: expression::Variant::Object(map), - }) if map.contains_key("from") && map.contains_key("to") => None, + }) if (map.contains_key("from") && map.contains_key("to")) + || map.contains_key("from") + || map.contains_key("to") => + { + None + } _ => Some(field.as_ref()), }) .collect::>(); diff --git a/src/enrichment_tables/file.rs b/src/enrichment_tables/file.rs index 32ec9318b88e0..20abb4fa8a4d1 100644 --- a/src/enrichment_tables/file.rs +++ b/src/enrichment_tables/file.rs @@ -317,6 +317,20 @@ impl File { _ => false, }, }, + Condition::FromDate { field, from } => match self.column_index(field) { + None => false, + Some(idx) => match row[idx] { + Value::Timestamp(date) => from <= &date, + _ => false, + }, + }, + Condition::ToDate { field, to } => match self.column_index(field) { + None => false, + Some(idx) => match row[idx] { + Value::Timestamp(date) => &date <= to, + _ => false, + }, + }, }) } @@ -1030,7 +1044,7 @@ mod tests { } #[test] - fn finds_row_with_dates() { + fn finds_row_between_dates() { let mut file = File::new( Default::default(), FileData { @@ -1096,6 +1110,132 @@ mod tests { ); } + #[test] + fn finds_row_from_date() { + let mut file = File::new( + Default::default(), + FileData { + modified: SystemTime::now(), + data: vec![ + vec![ + "zip".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2015, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp"), + ), + ], + vec![ + "zip".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2016, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp"), + ), + ], + ], + headers: vec!["field1".to_string(), "field2".to_string()], + }, + ); + + let handle = file.add_index(Case::Sensitive, &["field1"]).unwrap(); + + let conditions = [ + Condition::Equals { + field: "field1", + value: "zip".into(), + }, + Condition::FromDate { + field: "field2", + from: chrono::Utc + .with_ymd_and_hms(2016, 1, 1, 0, 0, 0) + .single() + .expect("invalid timestamp"), + }, + ]; + + assert_eq!( + Ok(ObjectMap::from([ + ("field1".into(), Value::from("zip")), + ( + "field2".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2016, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp") + ) + ) + ])), + file.find_table_row(Case::Sensitive, &conditions, None, Some(handle)) + ); + } + + #[test] + fn finds_row_to_date() { + let mut file = File::new( + Default::default(), + FileData { + modified: SystemTime::now(), + data: vec![ + vec![ + "zip".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2015, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp"), + ), + ], + vec![ + "zip".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2016, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp"), + ), + ], + ], + headers: vec!["field1".to_string(), "field2".to_string()], + }, + ); + + let handle = file.add_index(Case::Sensitive, &["field1"]).unwrap(); + + let conditions = [ + Condition::Equals { + field: "field1", + value: "zip".into(), + }, + Condition::ToDate { + field: "field2", + to: chrono::Utc + .with_ymd_and_hms(2016, 1, 1, 0, 0, 0) + .single() + .expect("invalid timestamp"), + }, + ]; + + assert_eq!( + Ok(ObjectMap::from([ + ("field1".into(), Value::from("zip")), + ( + "field2".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2015, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp") + ) + ) + ])), + file.find_table_row(Case::Sensitive, &conditions, None, Some(handle)) + ); + } + #[test] fn doesnt_find_row() { let file = File::new( diff --git a/website/cue/reference/remap/functions.cue b/website/cue/reference/remap/functions.cue index d739d474fec60..18ed9457a673c 100644 --- a/website/cue/reference/remap/functions.cue +++ b/website/cue/reference/remap/functions.cue @@ -74,7 +74,7 @@ remap: { performance perspective. 2. **Date range search**. The given field must be greater than or equal to the `from` date - and less than or equal to the `to` date. A date range search involves + and/or less than or equal to the `to` date. A date range search involves sequentially scanning through the rows that have been located using any exact match criteria. This can be an expensive operation if there are many rows returned by any exact match criteria. Therefore, use date ranges as the _only_ criteria when the enrichment From c9791e3db80659a5f763e51b4ad0824e366cbf1f Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 16 May 2025 17:15:42 -0400 Subject: [PATCH 018/305] fix(ci): path resolution for docker compose files (#23066) * fix(ci): path resolution for docker compose files * yaml spaces... --- .github/workflows/e2e.yml | 17 +++++++------- .github/workflows/integration.yml | 3 +-- scripts/e2e/datadog-logs/compose.yaml | 28 ++++++++++++------------ scripts/e2e/datadog-metrics/compose.yaml | 12 +++++----- scripts/integration/amqp/compose.yaml | 3 +-- 5 files changed, 30 insertions(+), 33 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 17df0426cb9ed..e5d322d2b8cc9 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -82,15 +82,14 @@ jobs: echo "PR_HAS_ACCESS_TO_SECRETS=false" >> "$GITHUB_ENV" fi - # Re-enable when https://github.com/vectordotdev/vector/issues/23054 is fixed. - # - if: (github.event_name == 'schedule' || needs.changes.outputs.all-e2e == 'true' || needs.changes.outputs.e2e-datadog-logs == 'true') && - # (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') - # name: e2e-datadog-logs - # uses: nick-fields/retry@v3 - # with: - # timeout_minutes: 35 - # max_attempts: 3 - # command: bash scripts/int-e2e-test.sh e2e datadog-logs + - if: (github.event_name == 'schedule' || needs.changes.outputs.all-e2e == 'true' || needs.changes.outputs.e2e-datadog-logs == 'true') && + (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') + name: e2e-datadog-logs + uses: nick-fields/retry@v3 + with: + timeout_minutes: 35 + max_attempts: 3 + command: bash scripts/int-e2e-test.sh e2e datadog-logs - if: (github.event_name == 'schedule' || needs.changes.outputs.all-e2e == 'true' || needs.changes.outputs.e2e-datadog-metrics == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 35ba4e186dc5a..74eb0bebaa696 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -111,11 +111,10 @@ jobs: if: ${{ needs.setup.outputs.can_access_secrets == 'true' }} strategy: matrix: - # TODO: Add "amqp" back once https://github.com/vectordotdev/vector/issues/22379 is fixed. # TODO: Add "splunk" back once https://github.com/vectordotdev/vector/issues/22379 is fixed. # If you modify this list, please also update the `int_tests` job in changes.yml. service: [ - "appsignal", "axiom", "aws", "azure", "clickhouse", "databend", "datadog-agent", + "amqp", "appsignal", "axiom", "aws", "azure", "clickhouse", "databend", "datadog-agent", "datadog-logs", "datadog-metrics", "datadog-traces", "dnstap", "docker-logs", "elasticsearch", "eventstoredb", "fluent", "gcp", "greptimedb", "http-client", "influxdb", "kafka", "logstash", "loki", "mongodb", "nats", "nginx", "opentelemetry", "postgres", "prometheus", "pulsar", diff --git a/scripts/e2e/datadog-logs/compose.yaml b/scripts/e2e/datadog-logs/compose.yaml index fe0bafc869012..61a9f8f2da1ae 100644 --- a/scripts/e2e/datadog-logs/compose.yaml +++ b/scripts/e2e/datadog-logs/compose.yaml @@ -32,12 +32,12 @@ services: - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false - DD_CONTAINER_EXCLUDE="name:.*" volumes: - # The Agent config file - - ${PWD}/tests/data/e2e/datadog/logs/agent_only.yaml:/etc/datadog-agent/datadog.yaml - # The custom logs check - - ${PWD}/tests/data/e2e/datadog/logs/logs.conf.d:/conf.d:ro - # The custom log to tail, created by the `log_generator` service - - log_path:/var/log/ + # The Agent config file + - ../../../tests/data/e2e/datadog/logs/agent_only.yaml:/etc/datadog-agent/datadog.yaml + # The custom logs check + - ../../../tests/data/e2e/datadog/logs/logs.conf.d:/conf.d:ro + # The custom log to tail, created by the `log_generator` service + - log_path:/var/log/ # Tails a custom log created by `log_generator` and sends log data to # the `vector` service @@ -52,12 +52,12 @@ services: - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false - DD_CONTAINER_EXCLUDE="name:.*" volumes: - # The Agent config file - - ${PWD}/tests/data/e2e/datadog/logs/agent_vector.yaml:/etc/datadog-agent/datadog.yaml - # The custom logs check - - ${PWD}/tests/data/e2e/datadog/logs/logs.conf.d:/conf.d:ro - # The custom log to tail, created by the `log_generator` service - - log_path:/var/log/ + # The Agent config file + - ../../../tests/data/e2e/datadog/logs/agent_vector.yaml:/etc/datadog-agent/datadog.yaml + # The custom logs check + - ../../../tests/data/e2e/datadog/logs/logs.conf.d:/conf.d:ro + # The custom log to tail, created by the `log_generator` service + - log_path:/var/log/ # Receives log data from the `datadog-agent-vector` service and sends # to the `fakeintake-vector` service. @@ -65,7 +65,7 @@ services: depends_on: - fakeintake-vector build: - context: ${PWD} + context: ../../.. # re-using the integration test runner image since it already has # compiled vector on it. image: ${CONFIG_VECTOR_IMAGE} @@ -79,7 +79,7 @@ services: - "-c" - "/home/vector/tests/data/e2e/datadog/logs/vector.toml" volumes: - - ${PWD}:/home/vector + - ../../..:/home/vector # Receives log data from the `datadog-agent` service. Is queried by the test runner # which does the validation of consistency with the other fakeintake service. diff --git a/scripts/e2e/datadog-metrics/compose.yaml b/scripts/e2e/datadog-metrics/compose.yaml index 5942fda1011e9..80523ac8144f4 100644 --- a/scripts/e2e/datadog-metrics/compose.yaml +++ b/scripts/e2e/datadog-metrics/compose.yaml @@ -27,8 +27,8 @@ services: - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - DD_HOSTNAME=datadog-agent volumes: - # The Agent config file - - ${PWD}/tests/data/e2e/datadog/metrics/agent_only.yaml:/etc/datadog-agent/datadog.yaml + # The Agent config file + - ../../../tests/data/e2e/datadog/metrics/agent_only.yaml:/etc/datadog-agent/datadog.yaml # Sends metric data received from the Emitter to the `vector` service datadog-agent-vector: @@ -39,8 +39,8 @@ services: - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - DD_HOSTNAME=datadog-agent-vector volumes: - # The Agent config file - - ${PWD}/tests/data/e2e/datadog/metrics/agent_vector.yaml:/etc/datadog-agent/datadog.yaml + # The Agent config file + - ../../../tests/data/e2e/datadog/metrics/agent_vector.yaml:/etc/datadog-agent/datadog.yaml # Receives metric data from the `datadog-agent-vector` service and sends # to the `fakeintake-vector` service. @@ -48,7 +48,7 @@ services: depends_on: - fakeintake-vector build: - context: ${PWD} + context: ../../.. # re-using the integration test runner image since it already has # compiled vector on it. image: ${CONFIG_VECTOR_IMAGE} @@ -62,7 +62,7 @@ services: - "-c" - "/home/vector/tests/data/e2e/datadog/metrics/vector.toml" volumes: - - ${PWD}:/home/vector + - ../../..:/home/vector # Receives metric data from the `datadog-agent` service. Is queried by the test runner # which does the validation of consistency with the other fakeintake service. diff --git a/scripts/integration/amqp/compose.yaml b/scripts/integration/amqp/compose.yaml index 865c35266f15f..a60a27155df0b 100644 --- a/scripts/integration/amqp/compose.yaml +++ b/scripts/integration/amqp/compose.yaml @@ -12,5 +12,4 @@ services: - RABBITMQ_SSL_CACERTFILE=/code/tests/data/ca/intermediate_server/certs/ca-chain.cert.pem - RABBITMQ_SSL_FAIL_IF_NO_PEER_CERT=false volumes: - - ${PWD}:/code - + - ../../..:/code From 968f3c91c3db6c1b57f4a329cb03f1978744b18d Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 16 May 2025 17:33:24 -0400 Subject: [PATCH 019/305] chore(releasing): relax prepare script branch creation (#23067) --- vdev/src/commands/release/prepare.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vdev/src/commands/release/prepare.rs b/vdev/src/commands/release/prepare.rs index 62bbbeb6da732..d2f67bce5a3f8 100644 --- a/vdev/src/commands/release/prepare.rs +++ b/vdev/src/commands/release/prepare.rs @@ -115,12 +115,12 @@ impl Prepare { git::run_and_check_output(&["fetch"])?; git::checkout_main_branch()?; - git::create_branch(self.release_branch.as_str())?; + git::checkout_or_create_branch(self.release_branch.as_str())?; git::push_and_set_upstream(self.release_branch.as_str())?; // Step 2: Create a new release preparation branch // The branch website contains 'website' to generate vector.dev preview. - git::create_branch(self.release_preparation_branch.as_str())?; + git::checkout_or_create_branch(self.release_preparation_branch.as_str())?; git::push_and_set_upstream(self.release_preparation_branch.as_str())?; Ok(()) } From e76f18ec820cf381716eca16c51ccdd11963af17 Mon Sep 17 00:00:00 2001 From: Burkov Egor Date: Mon, 19 May 2025 17:07:29 +0300 Subject: [PATCH 020/305] fix(parsing): Handling bad addresses in dsntap (#23071) * fix(parsing): Handling bad addresses in dsntap * Fix clippy warning --- .../fix_dnstap_extract_address_logic.fix.md | 3 ++ lib/dnstap-parser/src/parser.rs | 54 ++++++++++++++++++- 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fix_dnstap_extract_address_logic.fix.md diff --git a/changelog.d/fix_dnstap_extract_address_logic.fix.md b/changelog.d/fix_dnstap_extract_address_logic.fix.md new file mode 100644 index 0000000000000..60701f5b7d1d1 --- /dev/null +++ b/changelog.d/fix_dnstap_extract_address_logic.fix.md @@ -0,0 +1,3 @@ +Added checks to avoid possible panic while parsing address in `dnstap-parser::DnstapParser::parse_dnstap_message_socket_family`. + +authors: wooffie diff --git a/lib/dnstap-parser/src/parser.rs b/lib/dnstap-parser/src/parser.rs index 939f45a0b340d..14443ef581e76 100644 --- a/lib/dnstap-parser/src/parser.rs +++ b/lib/dnstap-parser/src/parser.rs @@ -438,9 +438,15 @@ impl DnstapParser { if let Some(query_address) = dnstap_message.query_address.as_ref() { let source_address = if socket_family == 1 { + if query_address.len() < 4 { + return Err(Error::from("Cannot parse query_address")); + } let address_buffer: [u8; 4] = query_address[0..4].try_into()?; IpAddr::V4(Ipv4Addr::from(address_buffer)) } else { + if query_address.len() < 16 { + return Err(Error::from("Cannot parse query_address")); + } let address_buffer: [u8; 16] = query_address[0..16].try_into()?; IpAddr::V6(Ipv6Addr::from(address_buffer)) }; @@ -464,9 +470,15 @@ impl DnstapParser { if let Some(response_address) = dnstap_message.response_address.as_ref() { let response_addr = if socket_family == 1 { + if response_address.len() < 4 { + return Err(Error::from("Cannot parse response_address")); + } let address_buffer: [u8; 4] = response_address[0..4].try_into()?; IpAddr::V4(Ipv4Addr::from(address_buffer)) } else { + if response_address.len() < 16 { + return Err(Error::from("Cannot parse response_address")); + } let address_buffer: [u8; 16] = response_address[0..16].try_into()?; IpAddr::V6(Ipv6Addr::from(address_buffer)) }; @@ -1038,7 +1050,7 @@ mod tests { use super::*; use chrono::DateTime; use dnsmsg_parser::dns_message_parser::DnsParserOptions; - use std::collections::BTreeMap; + use std::{collections::BTreeMap, vec}; #[test] fn test_parse_dnstap_data_with_query_message() { @@ -1365,6 +1377,46 @@ mod tests { ) } + #[test] + fn test_parse_dnstap_message_socket_family_bad_addr() { + // while parsing address is optional, but in this function assume otherwise + fn test_one_input(socket_family: i32, msg: DnstapMessage) -> Result<()> { + let mut event = LogEvent::default(); + let root = owned_value_path!(); + DnstapParser::parse_dnstap_message_socket_family(&mut event, &root, socket_family, &msg) + } + // all bad cases which can panic + { + let message = DnstapMessage { + query_address: Some(vec![]), + ..Default::default() + }; + assert!(test_one_input(1, message).is_err()); + } + { + let message = DnstapMessage { + query_address: Some(vec![]), + ..Default::default() + }; + assert!(test_one_input(2, message).is_err()); + } + + { + let message = DnstapMessage { + response_address: Some(vec![]), + ..Default::default() + }; + assert!(test_one_input(1, message).is_err()); + } + { + let message = DnstapMessage { + response_address: Some(vec![]), + ..Default::default() + }; + assert!(test_one_input(2, message).is_err()); + } + } + #[test] fn test_get_socket_family_name() { assert_eq!("INET", to_socket_family_name(1).unwrap()); From bf57582d5855cf7beae12eb88bf21e911680489c Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 19 May 2025 16:03:23 -0400 Subject: [PATCH 021/305] chore(releasing): update VRL changelog block insertion (#23075) --- vdev/src/commands/release/prepare.rs | 145 +++++++++++++++++---------- 1 file changed, 91 insertions(+), 54 deletions(-) diff --git a/vdev/src/commands/release/prepare.rs b/vdev/src/commands/release/prepare.rs index d2f67bce5a3f8..5b333e25ab438 100644 --- a/vdev/src/commands/release/prepare.rs +++ b/vdev/src/commands/release/prepare.rs @@ -4,7 +4,6 @@ use crate::git; use crate::util::run_command; use anyhow::{anyhow, Result}; -use indoc::indoc; use reqwest::blocking::Client; use semver::Version; use std::fs::File; @@ -364,65 +363,25 @@ impl Prepare { fn append_vrl_changelog_to_release_cue(&self) -> Result<()> { debug!("append_vrl_changelog_to_release_cue"); - let releases_path = &self.repo_root.join("website").join("cue").join("reference").join("releases"); - let vector_version = &self.new_vector_version; - let release_cue_path = releases_path.join(format!("{vector_version}.cue")); - if !release_cue_path.is_file() { - return Err(anyhow!("{release_cue_path:?} not found")); + + let releases_path = self.repo_root.join("website/cue/reference/releases"); + let version = &self.new_vector_version; + let cue_path = releases_path.join(format!("{version}.cue")); + if !cue_path.is_file() { + return Err(anyhow!("{cue_path:?} not found")); } let vrl_changelog = get_latest_vrl_tag_and_changelog()?; + let vrl_changelog_block = format_vrl_changelog_block(&vrl_changelog); - let temp_file_path = releases_path.join(format!("{vector_version}.cue.tmp")); - let input_file = File::open(&release_cue_path)?; - let reader = BufReader::new(input_file); - let mut output_file = File::create(&temp_file_path)?; - - let indent = "\t".repeat(5); - let processed_changelog: String = vrl_changelog - .lines() - .map(|line| { - let line = line.trim(); - if line.starts_with('#') { - format!("{indent}#{line}") - } else { - format!("{indent}{line}") - } - }) - .collect::>() - .join("\n"); - - // Format the new changelog entry - let vrl_cue_block = format!( - indoc! {r#" - {{ - type: "feat" - description: """ - {} - """ - }}, - "#}, - processed_changelog - ); - - let mut found_changelog = false; - let changelog_marker = "changelog: ["; + let original = fs::read_to_string(&cue_path)?; + let updated = insert_block_after_changelog(&original, &vrl_changelog_block); - // Read and write line by line - for line in reader.lines() { - let line = line?; - writeln!(output_file, "{line}")?; - - // Check if this is the changelog line - if !found_changelog && line.trim().starts_with(changelog_marker) { - // Insert the new entry after the changelog opening - writeln!(output_file, "{vrl_cue_block}")?; - found_changelog = true; - } - } + let tmp_path = cue_path.with_extension("cue.tmp"); + fs::write(&tmp_path, &updated)?; + fs::rename(&tmp_path, &cue_path)?; - fs::rename(&temp_file_path, &release_cue_path)?; - run_command(&format!("cue fmt {release_cue_path:?}")); + run_command(&format!("cue fmt {}", cue_path.display())); debug!("Successfully added VRL changelog to the release cue file."); Ok(()) } @@ -445,6 +404,45 @@ fn get_latest_version_from_vector_tags() -> Result { .map_err(|e| anyhow::anyhow!("Failed to parse version from tag '{latest_tag}': {e}")) } +fn format_vrl_changelog_block(changelog: &str) -> String { + let double_tab = "\t\t"; + let body = changelog + .lines() + .map(|line| { + let line = line.trim(); + if line.starts_with('#') { + format!("{double_tab}#{line}") + } else { + format!("{double_tab}{line}") + } + }) + .collect::>() + .join("\n"); + + let opening = "\tvrl_changelog: \"\"\""; + let closing = format!("{double_tab}\"\"\""); + + format!("{opening}\n{body}\n{closing}") +} + +fn insert_block_after_changelog(original: &str, block: &str) -> String { + let mut result = Vec::new(); + let mut inserted = false; + + for line in original.lines() { + result.push(line.to_string()); + + // Insert *after* the line containing only the closing `]` (end of changelog array) + if !inserted && line.trim() == "]" { + result.push(String::new()); // empty line before + result.push(block.to_string()); + inserted = true; + } + } + + result.join("\n") +} + fn get_latest_vrl_tag_and_changelog() -> Result { let client = Client::new(); @@ -497,3 +495,42 @@ fn get_latest_vrl_tag_and_changelog() -> Result { Ok(section.join("\n")) } + +#[cfg(test)] +mod tests { + use crate::commands::release::prepare::{format_vrl_changelog_block, insert_block_after_changelog}; + use indoc::indoc; + + #[test] + fn test_insert_block_after_changelog() { + let vrl_changelog = "### [0.2.0]\n- Feature\n- Fix"; + let vrl_changelog_block = format_vrl_changelog_block(vrl_changelog); + + let expected = concat!( + "\tvrl_changelog: \"\"\"\n", + "\t\t#### [0.2.0]\n", + "\t\t- Feature\n", + "\t\t- Fix\n", + "\t\t\"\"\"" + ); + + assert_eq!(vrl_changelog_block, expected); + + let original = indoc! {r#" + version: "1.2.3" + changelog: [ + { + type: "fix" + description: "Some fix" + }, + ] + "#}; + let updated = insert_block_after_changelog(original, &vrl_changelog_block); + + // Assert the last 5 lines match the VRL changelog block + let expected_lines_len = 5; + let updated_tail: Vec<&str> = updated.lines().rev().take(expected_lines_len).collect::>().into_iter().rev().collect(); + let expected_lines: Vec<&str> = vrl_changelog_block.lines().collect(); + assert_eq!(updated_tail, expected_lines); + } +} From eb12304ebd8b2b06fd50df242d660b056a33b8a3 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 20 May 2025 11:51:36 -0400 Subject: [PATCH 022/305] fix(ci): update git repo url for homebrew release (#23081) --- vdev/src/commands/release/homebrew.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vdev/src/commands/release/homebrew.rs b/vdev/src/commands/release/homebrew.rs index ea08786dfb9fd..213e6ce1c5ed3 100644 --- a/vdev/src/commands/release/homebrew.rs +++ b/vdev/src/commands/release/homebrew.rs @@ -42,9 +42,10 @@ impl Cli { fn clone_and_setup_git(username: &str) -> Result<()> { let github_token = env::var("GITHUB_TOKEN")?; let homebrew_repo = format!( - "https://{github_token}@github.com/{username}/homebrew-brew.git", + "https://{username}:{github_token}@github.com/{username}/homebrew-brew.git" ); + git::clone(&homebrew_repo)?; env::set_current_dir("homebrew-brew")?; git::set_config_value("user.name", "vic")?; From 0ce2de7527170e2b9792a7916391a8d5483bc949 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 20 May 2025 16:03:02 -0400 Subject: [PATCH 023/305] chore(releasing): improve release templates (#23080) * chore(releasing): improve release guide * easier setup * more improvements * reorganize release prep steps * lint * improve titles * typo --- .github/ISSUE_TEMPLATE/minor-release.md | 68 ++++++++++++++++++------- .github/ISSUE_TEMPLATE/patch-release.md | 36 ++++++++----- 2 files changed, 73 insertions(+), 31 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/minor-release.md b/.github/ISSUE_TEMPLATE/minor-release.md index 50dabfcc72139..fe1dc999ba9a7 100644 --- a/.github/ISSUE_TEMPLATE/minor-release.md +++ b/.github/ISSUE_TEMPLATE/minor-release.md @@ -5,25 +5,44 @@ title: "Vector [version] release" labels: "domain: releasing" --- -The week before the release: + +# Setup and Automation + +Note the preparation steps are now automated. You can run: + +```shell +export NEW_VERSION=0.47.0 # replace this with the actual new version +export MINOR_VERSION=$(echo "$NEW_VERSION" | cut -d. -f2) +export PREP_BRANCH=prepare-v-0-"${MINOR_VERSION}"-"${NEW_VERSION}"-website +export RELEASE_BRANCH=v0."${MINOR_VERSION}" +export NEW_VRL_VERSION=0.42.0 # replace this with the actual new VRL version +``` + +# The week before the release + +## 1. Manual Steps - [ ] Cut a new release of [VRL](https://github.com/vectordotdev/vrl) if needed -- [ ] Check for any outstanding deprecation actions in [DEPRECATIONS.md](https://github.com/vectordotdev/vector/blob/master/docs/DEPRECATIONS.md) and - take them (or have someone help you take them) + - VRL release steps: https://github.com/vectordotdev/vrl/blob/main/release/README.md + +## 2. Automated Steps + +Run the following: + +```shell +vdev release prepare --version "${NEW_VERSION_NUMBER}" --vrl-version NEW_VRL_VERSION +``` + +Automated steps include: - [ ] Create a new release branch from master to freeze commits - - `git fetch && git checkout origin/master && git checkout -b v0. && git push -u` + - `git fetch && git checkout origin/master && git checkout -b "{RELEASE_BRANCH}" && git push -u` - [ ] Create a new release preparation branch from `master` - - `git checkout -b website-prepare-v0. && git push -u` + - `git checkout -b "${PREP_BRANCH}" && git push -u` - [ ] Pin VRL to latest released version rather than `main` - [ ] Check if there is a newer version of [Alpine](https://alpinelinux.org/releases/) or [Debian](https://www.debian.org/releases/) available to update the release images in `distribution/docker/`. Update if so. - [ ] Run `cargo vdev build release-cue` to generate a new cue file for the release - - [ ] Add description key to the generated cue file with a description of the release (see - previous releases for examples). - - [ ] Ensure any breaking changes are highlighted in the release upgrade guide - - [ ] Ensure any deprecations are highlighted in the release upgrade guide - - [ ] Review generated changelog entries to ensure they are understandable to end-users - [ ] Copy VRL changelogs from the VRL version in the last Vector release as a new changelog entry ([example](https://github.com/vectordotdev/vector/blob/9c67bba358195f5018febca2f228dfcb2be794b5/website/cue/reference/releases/0.41.0.cue#L33-L64)) - [ ] Update version number in `website/cue/reference/administration/interfaces/kubectl.cue` @@ -32,29 +51,40 @@ The week before the release: - [ ] Create new release md file by copying an existing one in `./website/content/en/releases/` and updating version number - [ ] Commit these changes -- [ ] Open PR against the release branch (`v0.`) for review -- [ ] PR approval +- [ ] Open PR against the release branch (`"${RELEASE_BRANCH}"`) for review + +## 3. Manual Steps + +- [ ] Edit `website/cue/reference/releases/"${NEW_VERSION_NUMBER}".cue` + - [ ] Add description key to the generated cue file with a description of the release (see + previous releases for examples). + - [ ] Ensure any breaking changes are highlighted in the release upgrade guide + - [ ] Ensure any deprecations are highlighted in the release upgrade guide + - [ ] Review generated changelog entries to ensure they are understandable to end-users +- [ ] Check for any outstanding deprecation actions in [DEPRECATIONS.md](https://github.com/vectordotdev/vector/blob/master/docs/DEPRECATIONS.md) and + take them (or have someone help you take them) +- [ ] PR review & approval -On the day of release: +# On the day of release - [ ] Rebase the release preparation branch on the release branch - [ ] Squash the release preparation commits (but not the cherry-picked commits!) to a single - commit. This makes it easier to cherry-pick to master after the release.  + commit. This makes it easier to cherry-pick to master after the release. - [ ] Ensure release date in cue matches current date. - [ ] Merge release preparation branch into the release branch - - `git co v0. && git merge --ff-only prepare-v0.` + - `git switch "${RELEASE_BRANCH}" && git merge --ff-only "${PREP_BRANCH}"` - [ ] Tag new release - - [ ] `git tag v0..0 -a -m v0..0` - - [ ] `git push origin v0..0` + - [ ] `git tag v"${NEW_VERSION_NUMBER}" -a -m v"${NEW_VERSION_NUMBER}"` + - [ ] `git push origin v"${NEW_VERSION_NUMBER}"` - [ ] Wait for release workflow to complete - Discoverable via [https://github.com/timberio/vector/actions/workflows/release.yml](https://github.com/timberio/vector/actions/workflows/release.yml) - [ ] Reset the `website` branch to the `HEAD` of the release branch to update https://vector.dev - - [ ] `git checkout website && git reset --hard origin/v0. && git push` + - [ ] `git switch website && git reset --hard origin/"{RELEASE_BRANCH}" && git push` - [ ] Confirm that the release changelog was published to https://vector.dev/releases/ - The deployment is done by Amplify. You can see the [deployment logs here](https://dd-corpsite.datadoghq.com/logs?query=service%3Awebsites-vector%20branch%3Awebsite&agg_m=count&agg_m_source=base&agg_t=count&cols=host%2Cservice&fromUser=true&messageDisplay=inline&refresh_mode=sliding&storage=hot&stream_sort=time%2Casc&viz=stream). - [ ] Release Linux packages. See [`vector-release` usage](https://github.com/DataDog/vector-release#usage). - - Note: the pipeline inputs are the version number `v0.` and a personal GitHub token. + - Note: the pipeline inputs are the version number `v"${NEW_VERSION_NUMBER}"` and a personal GitHub token. - [ ] Manually trigger the `trigger-package-release-pipeline-prod-stable` job. - [ ] Release updated Helm chart. See [releasing Helm chart](https://github.com/vectordotdev/helm-charts#releasing). - [ ] Once Helm chart is released, updated Vector manifests diff --git a/.github/ISSUE_TEMPLATE/patch-release.md b/.github/ISSUE_TEMPLATE/patch-release.md index 185d5beff7917..afc5d8d826423 100644 --- a/.github/ISSUE_TEMPLATE/patch-release.md +++ b/.github/ISSUE_TEMPLATE/patch-release.md @@ -5,10 +5,22 @@ title: "Vector [version] release" labels: "domain: releasing" --- -Before the release: +# Setup environment + +```shell +export CURRENT_MINOR_VERSION = # e.g. 47 +export CURRENT_PATCH_VERSION = # e.g. 0 +export CURRENT_VERSION="${RELEASE_BRANCH}"."${CURRENT_PATCH_VERSION}" +export NEW_PATCH_VERSION = # e.g. 1 +export NEW_VERSION="${RELEASE_BRANCH}"."${NEW_PATCH_VERSION}" +export RELEASE_BRANCH=v0."${CURRENT_MINOR_VERSION}" +export PREP_BRANCH=prepare-v-0-"${CURRENT_MINOR_VERSION}"-"${NEW_PATCH_VERSION}"-website +``` + +# Before the release - [ ] Create a new release preparation branch from the current release branch - - `git fetch --all && git checkout v0. && git checkout -b website-prepare-v0-- + - `git fetch --all && git checkout "${RELEASE_BRANCH}" && git checkout -b "${PREP_BRANCH}""` - [ ] Cherry-pick in all commits to be released from the associated release milestone - If any merge conflicts occur, attempt to solve them and if needed enlist the aid of those familiar with the conflicting commits. - [ ] Bump the release number in the `Cargo.toml` to the current version number @@ -18,31 +30,31 @@ Before the release: - [ ] Update version number in `distribution/install.sh` - [ ] Add new version to `website/cue/reference/versions.cue` - [ ] Create new release md file by copying an existing one in `./website/content/en/releases/`. - - Update the version number to `v0..` and increase the `weight` by 1. + - Update the version number to `"${NEW_VERSION}"` and increase the `weight` by 1. - [ ] Run `cargo check` to regenerate `Cargo.lock` file - [ ] Commit these changes -- [ ] Open PR against the release branch (`v0.`) for review +- [ ] Open PR against the release branch (`"${RELEASE_BRANCH}"`) for review - [ ] PR approval -On the day of release: +# On the day of release - [ ] Ensure release date in cue matches current date. - [ ] Rebase the release preparation branch on the release branch - Squash the release preparation commits (but not the cherry-picked commits!) to a single commit. This makes it easier to cherry-pick to master after the release. - - `git fetch --all && git checkout website-prepare-v0-- && git rebase -i v0.` + - `git fetch --all && git checkout website-prepare-v0-"${CURRENT_MINOR_VERSION}"-"${NEW_PATCH_VERSION}" && git rebase -i "${RELEASE_BRANCH}"` - [ ] Merge release preparation branch into the release branch - - `git checkout v0. && git merge --ff-only website-prepare-v0--` + - `git checkout "${RELEASE_BRANCH}" && git merge --ff-only website-prepare-v0-"${CURRENT_MINOR_VERSION}"-"${NEW_PATCH_VERSION}"` - [ ] Tag new release - - [ ] `git tag v0.. -a -m v0..` - - [ ] `git push origin v0..` + - [ ] `git tag "${NEW_VERSION}" -a -m "${NEW_VERSION}"` + - [ ] `git push origin "${NEW_VERSION}"` - [ ] Wait for release workflow to complete - Discoverable via [https://github.com/timberio/vector/actions/workflows/release.yml](https://github.com/timberio/vector/actions/workflows/release.yml) - [ ] Release Linux packages. See [`vector-release` usage](https://github.com/DataDog/vector-release#usage). - - Note: the pipeline inputs are the version number `v0.` and a personal GitHub token. + - Note: the pipeline inputs are the version number `"${CURRENT_VERSION}"` and a personal GitHub token. - [ ] Manually trigger the `trigger-package-release-pipeline-prod-stable` job. - [ ] Push the release branch to update the remote (This should close the preparation branch PR). - - `git checkout v0. && git push` + - `git checkout "${RELEASE_BRANCH}" && git push` - [ ] Release updated Helm chart. See [releasing Helm chart](https://github.com/vectordotdev/helm-charts#releasing). - [ ] Once Helm chart is released, updated Vector manifests - Run `cargo vdev build manifests` and open a PR with changes @@ -50,5 +62,5 @@ On the day of release: - Follow the [instructions at the top of the mirror.yaml file](https://github.com/DataDog/images/blob/fbf12868e90d52e513ebca0389610dea8a3c7e1a/mirror.yaml#L33-L49). - [ ] Cherry-pick any release commits from the release branch that are not on `master`, to `master` - [ ] Reset the `website` branch to the `HEAD` of the release branch to update https://vector.dev - - [ ] `git checkout website && git reset --hard origin/v0.. && git push` + - [ ] `git checkout website && git reset --hard origin/"${RELEASE_BRANCH}" && git push` - [ ] Kick-off post-mortems for any regressions resolved by the release From 34db5b0d784392e8d1c687edb6682ca980dcace8 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 20 May 2025 17:20:19 -0400 Subject: [PATCH 024/305] chore(releasing): final release steps (#23085) * chore(releasing): final release steps * chore(releasing): Pinned VRL version to 0.24.0 chore(releasing): Generated release CUE file chore(releasing): Updated website/cue/reference/administration/interfaces/kubectl.cue vector version to 0.47.0 chore(releasing): Updated distribution/install.sh vector version to 0.47.0 chore(releasing): Add 0.47.0 to versions.cue chore(releasing): Created release md file changelog editing user facing text, rm accidentally comitted files Apply suggestions from code review Co-authored-by: domalessi <111786334+domalessi@users.noreply.github.com> Apply suggestions from code review Co-authored-by: domalessi <111786334+domalessi@users.noreply.github.com> Apply suggestions from code review Co-authored-by: domalessi <111786334+domalessi@users.noreply.github.com> Update website/cue/reference/releases/0.47.0.cue Co-authored-by: domalessi <111786334+domalessi@users.noreply.github.com> wording merge sentences bump the date * bump vector version --- .gitignore | 3 + Cargo.lock | 10 +- Cargo.toml | 4 +- .../14082_add_mermaid_support.feature.md | 3 - ...group_id_groups_max_limit_increased.fix.md | 3 - changelog.d/19931_add_mqtt_source.feature.md | 3 - changelog.d/22125.feature.md | 3 - .../22212_fluent_unix_socket.feature.md | 3 - changelog.d/22533_zlib-rs.enhancement.md | 3 - ...et_server_buffering_ack_support.feature.md | 3 - .../22651_greptimedb_logs_headers.breaking.md | 32 -- .../22656_statsd_timer_conversion.feature.md | 3 - .../22786_remove_empty_file_after_secs.fix.md | 3 - ...2827_cloudwatch_auth_missing_region.fix.md | 3 - ...rease_cloudwatch_log_event_max_size.fix.md | 3 - ...2850_custom_auth_address_access.feature.md | 3 - ...cket_server_subprotocol_support.feature.md | 3 - ...57_junit_reports_for_unit_tests.feature.md | 3 - .../22865_framing_performance.enhancement.md | 3 - ...is_sink_supports_encoding_input.feature.md | 3 - ...remove_limit_if_use_apiserver_cache.fix.md | 3 - ...2922_healthcheck_timeout_config.feature.md | 3 - ...e_bounded_date_range_search.enhancement.md | 3 - ...1_add_relaxed_wildcard_matching.feature.md | 24 - ..._add_a6_prefix_check_in_dns_parsing.fix.md | 3 - .../23048_fix_arith_logic_dnstap_parse.fix.md | 3 - ...19_watch_remap_external_vrl.enhancement.md | 3 - .../add_priority_amqp_sink.enhancement.md | 3 - changelog.d/datadog_agent_http_header.fix.md | 31 -- .../datadog_logs_sink_logs_namespace.fix.md | 3 - changelog.d/gelf-error.fix.md | 3 - changelog.d/opentelemetry_metrics.feature.md | 3 - .../s3-source-defer-or-delete.enhancement.md | 3 - changelog.d/window-transform.feature.md | 5 - distribution/install.sh | 2 +- .../kubernetes/vector-agent/README.md | 2 +- .../kubernetes/vector-agent/configmap.yaml | 2 +- .../kubernetes/vector-agent/daemonset.yaml | 4 +- .../kubernetes/vector-agent/rbac.yaml | 4 +- .../vector-agent/service-headless.yaml | 2 +- .../vector-agent/serviceaccount.yaml | 2 +- .../kubernetes/vector-aggregator/README.md | 2 +- .../vector-aggregator/configmap.yaml | 2 +- .../vector-aggregator/service-headless.yaml | 2 +- .../kubernetes/vector-aggregator/service.yaml | 2 +- .../vector-aggregator/serviceaccount.yaml | 2 +- .../vector-aggregator/statefulset.yaml | 4 +- .../vector-stateless-aggregator/README.md | 2 +- .../configmap.yaml | 2 +- .../deployment.yaml | 4 +- .../service-headless.yaml | 2 +- .../vector-stateless-aggregator/service.yaml | 2 +- .../serviceaccount.yaml | 2 +- docker_logs.txt | 0 .../reference/configuration/sources/mqtt.md | 2 +- website/content/en/releases/0.47.0.md | 4 + .../administration/interfaces/kubectl.cue | 2 +- website/cue/reference/releases/0.47.0.cue | 419 ++++++++++++++++++ website/cue/reference/versions.cue | 1 + website/layouts/releases/single.html | 11 +- 60 files changed, 465 insertions(+), 210 deletions(-) delete mode 100644 changelog.d/14082_add_mermaid_support.feature.md delete mode 100644 changelog.d/17294_allocation_group_id_groups_max_limit_increased.fix.md delete mode 100644 changelog.d/19931_add_mqtt_source.feature.md delete mode 100644 changelog.d/22125.feature.md delete mode 100644 changelog.d/22212_fluent_unix_socket.feature.md delete mode 100644 changelog.d/22533_zlib-rs.enhancement.md delete mode 100644 changelog.d/22540_websocket_server_buffering_ack_support.feature.md delete mode 100644 changelog.d/22651_greptimedb_logs_headers.breaking.md delete mode 100644 changelog.d/22656_statsd_timer_conversion.feature.md delete mode 100644 changelog.d/22786_remove_empty_file_after_secs.fix.md delete mode 100644 changelog.d/22827_cloudwatch_auth_missing_region.fix.md delete mode 100644 changelog.d/22831_increase_cloudwatch_log_event_max_size.fix.md delete mode 100644 changelog.d/22850_custom_auth_address_access.feature.md delete mode 100644 changelog.d/22854_websocket_server_subprotocol_support.feature.md delete mode 100644 changelog.d/22857_junit_reports_for_unit_tests.feature.md delete mode 100644 changelog.d/22865_framing_performance.enhancement.md delete mode 100644 changelog.d/22909_redis_sink_supports_encoding_input.feature.md delete mode 100644 changelog.d/22921_remove_limit_if_use_apiserver_cache.fix.md delete mode 100644 changelog.d/22922_healthcheck_timeout_config.feature.md delete mode 100644 changelog.d/22926_enrichment_function_single_bounded_date_range_search.enhancement.md delete mode 100644 changelog.d/23011_add_relaxed_wildcard_matching.feature.md delete mode 100644 changelog.d/23047_add_a6_prefix_check_in_dns_parsing.fix.md delete mode 100644 changelog.d/23048_fix_arith_logic_dnstap_parse.fix.md delete mode 100644 changelog.d/9819_watch_remap_external_vrl.enhancement.md delete mode 100644 changelog.d/add_priority_amqp_sink.enhancement.md delete mode 100644 changelog.d/datadog_agent_http_header.fix.md delete mode 100644 changelog.d/datadog_logs_sink_logs_namespace.fix.md delete mode 100644 changelog.d/gelf-error.fix.md delete mode 100644 changelog.d/opentelemetry_metrics.feature.md delete mode 100644 changelog.d/s3-source-defer-or-delete.enhancement.md delete mode 100644 changelog.d/window-transform.feature.md create mode 100644 docker_logs.txt create mode 100644 website/content/en/releases/0.47.0.md create mode 100644 website/cue/reference/releases/0.47.0.cue diff --git a/.gitignore b/.gitignore index 24b0593d51d6c..2a57f31cc6f40 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,6 @@ tilt_modules/ # Jetbrains .idea/ + +# Ignore generated temporary Docker Compose files +**/compose-temp-*.yaml diff --git a/Cargo.lock b/Cargo.lock index 24d055bbce81a..3c465db7ff350 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11434,7 +11434,7 @@ dependencies = [ [[package]] name = "vector" -version = "0.47.0" +version = "0.48.0" dependencies = [ "apache-avro", "approx", @@ -11994,8 +11994,9 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "vrl" -version = "0.23.0" -source = "git+https://github.com/vectordotdev/vrl?branch=main#7f8ed50af55b936c6da62a19dbbd966a8af903b6" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9ceadaa40aef567a26079ff014ca7a567ba85344f1b81090b5ec7d7bb16a219" dependencies = [ "aes", "aes-siv", @@ -12003,7 +12004,7 @@ dependencies = [ "arbitrary", "base16", "base62", - "base64 0.22.1", + "base64-simd", "bytes 1.10.1", "cbc", "cfb-mode", @@ -12022,7 +12023,6 @@ dependencies = [ "crypto_secretbox", "csv", "ctr", - "data-encoding", "digest", "dns-lookup", "domain", diff --git a/Cargo.toml b/Cargo.toml index c999e7b270d74..4bf10447c73ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vector" -version = "0.47.0" +version = "0.48.0" authors = ["Vector Contributors "] edition = "2021" description = "A lightweight and ultra-fast tool for building observability pipelines" @@ -174,7 +174,7 @@ vector-lib = { path = "lib/vector-lib", default-features = false, features = ["v vector-config = { path = "lib/vector-config" } vector-config-common = { path = "lib/vector-config-common" } vector-config-macros = { path = "lib/vector-config-macros" } -vrl = { git = "https://github.com/vectordotdev/vrl", branch = "main", features = ["arbitrary", "cli", "test", "test_framework"] } +vrl = { features = ["arbitrary", "cli", "test", "test_framework"], version = "0.24.0" } [dependencies] cfg-if.workspace = true diff --git a/changelog.d/14082_add_mermaid_support.feature.md b/changelog.d/14082_add_mermaid_support.feature.md deleted file mode 100644 index 912652f9bc0d8..0000000000000 --- a/changelog.d/14082_add_mermaid_support.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Add support for rendering to Mermaid format in `vector graph` - -authors: Firehed diff --git a/changelog.d/17294_allocation_group_id_groups_max_limit_increased.fix.md b/changelog.d/17294_allocation_group_id_groups_max_limit_increased.fix.md deleted file mode 100644 index 58791882556ba..0000000000000 --- a/changelog.d/17294_allocation_group_id_groups_max_limit_increased.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fix a Vector crash that occurred when the internal metrics generated a lot of groups by increasing groups max limit from 128 to 256. - -authors: triggerhappy17 diff --git a/changelog.d/19931_add_mqtt_source.feature.md b/changelog.d/19931_add_mqtt_source.feature.md deleted file mode 100644 index 24e9dbebc6aed..0000000000000 --- a/changelog.d/19931_add_mqtt_source.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Adds a new `mqtt` source enabling Vector to receive logs from a MQTT broker. - -authors: mladedav pront StormStake diff --git a/changelog.d/22125.feature.md b/changelog.d/22125.feature.md deleted file mode 100644 index a2fb312ef23ed..0000000000000 --- a/changelog.d/22125.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Allows users to specify AWS authentication and the AWS service name for HTTP sinks to support AWS API endpoints that require SigV4. - -authors: johannesfloriangeiger diff --git a/changelog.d/22212_fluent_unix_socket.feature.md b/changelog.d/22212_fluent_unix_socket.feature.md deleted file mode 100644 index be21ffb0c042a..0000000000000 --- a/changelog.d/22212_fluent_unix_socket.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Add support for fluentd forwarding over a unix socket - -authors: tustvold diff --git a/changelog.d/22533_zlib-rs.enhancement.md b/changelog.d/22533_zlib-rs.enhancement.md deleted file mode 100644 index cbc922fcc52b5..0000000000000 --- a/changelog.d/22533_zlib-rs.enhancement.md +++ /dev/null @@ -1,3 +0,0 @@ -Replaced miniz_oxide backend for zlib compression with zlib-rs for significantly improved compression performance. zlib-rs provides 2-3x performance improvements for compression, and ~20% improvements for decompression than miniz_oxide. - -authors: JakubOnderka diff --git a/changelog.d/22540_websocket_server_buffering_ack_support.feature.md b/changelog.d/22540_websocket_server_buffering_ack_support.feature.md deleted file mode 100644 index 0ce4cadf6740c..0000000000000 --- a/changelog.d/22540_websocket_server_buffering_ack_support.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Add ACK support to message buffering feature of `websocket_server` sink, allowing this component to cache latest received messages per client. - -authors: esensar Quad9DNS diff --git a/changelog.d/22651_greptimedb_logs_headers.breaking.md b/changelog.d/22651_greptimedb_logs_headers.breaking.md deleted file mode 100644 index 674bf5d69bacb..0000000000000 --- a/changelog.d/22651_greptimedb_logs_headers.breaking.md +++ /dev/null @@ -1,32 +0,0 @@ -Added a new `extra_headers` option to `greptimedb_logs` sink configuration to set additional headers for outgoing requests. - -change `greptimedb_logs` sink default content type to `application/x-ndjson` to match the default content type of `greptimedb` sink -if you use the greptimedb version v0.12 or earlier, you need to set the content type to `application/json` in the sink configuration - -Example: - -```yaml -sinks: - greptime_logs: - type: greptimedb_logs - inputs: ["my_source_id"] - endpoint: "http://localhost:4000" - table: "demo_logs" - dbname: "public" - extra_headers: - x-source: vector -``` - -```toml -[sinks.greptime_logs] -type = "greptimedb_logs" -inputs = ["my_source_id"] -endpoint = "http://localhost:4000" -table = "demo_logs" -dbname = "public" - -[sinks.greptime_logs.extra_headers] -x-source = "vector" -``` - -authors: greptimedb diff --git a/changelog.d/22656_statsd_timer_conversion.feature.md b/changelog.d/22656_statsd_timer_conversion.feature.md deleted file mode 100644 index fa55207ebd3b6..0000000000000 --- a/changelog.d/22656_statsd_timer_conversion.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -This change introduces a new configuration option in the StatsD source: `convert_to` of type `ConversionUnit`. By default, timing values in milliseconds (`ms`) are converted to seconds (`s`). Users can set `convert_to` to `"milliseconds"` to preserve the original millisecond values. - -authors: devkoriel diff --git a/changelog.d/22786_remove_empty_file_after_secs.fix.md b/changelog.d/22786_remove_empty_file_after_secs.fix.md deleted file mode 100644 index 70b9d5177e956..0000000000000 --- a/changelog.d/22786_remove_empty_file_after_secs.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed `file` source bug where known small files were not deleted after the specified `remove_after_secs`. - -authors: linw1995 diff --git a/changelog.d/22827_cloudwatch_auth_missing_region.fix.md b/changelog.d/22827_cloudwatch_auth_missing_region.fix.md deleted file mode 100644 index 7cfacdc2191c1..0000000000000 --- a/changelog.d/22827_cloudwatch_auth_missing_region.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a `AwsAuthentication::File` bug where `region` was missing from the `STS` authentication endpoint. - -authors: cahartma diff --git a/changelog.d/22831_increase_cloudwatch_log_event_max_size.fix.md b/changelog.d/22831_increase_cloudwatch_log_event_max_size.fix.md deleted file mode 100644 index dddae5c17b134..0000000000000 --- a/changelog.d/22831_increase_cloudwatch_log_event_max_size.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Increase the max event size for `aws_cloudwatch_logs` sink based on newly increased limit from `AWS` - -authors: cahartma diff --git a/changelog.d/22850_custom_auth_address_access.feature.md b/changelog.d/22850_custom_auth_address_access.feature.md deleted file mode 100644 index a955cec1b0aa5..0000000000000 --- a/changelog.d/22850_custom_auth_address_access.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Enabled IP address access in VRL scripts of custom auth strategy for server components. - -authors: esensar Quad9DNS diff --git a/changelog.d/22854_websocket_server_subprotocol_support.feature.md b/changelog.d/22854_websocket_server_subprotocol_support.feature.md deleted file mode 100644 index 322aa36a8b4d9..0000000000000 --- a/changelog.d/22854_websocket_server_subprotocol_support.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Added support for the `Sec-WebSocket-Protocol` header in the `websocket_server` sink to better accommodate clients that require it. - -authors: esensar Quad9DNS diff --git a/changelog.d/22857_junit_reports_for_unit_tests.feature.md b/changelog.d/22857_junit_reports_for_unit_tests.feature.md deleted file mode 100644 index 1c72c4919937e..0000000000000 --- a/changelog.d/22857_junit_reports_for_unit_tests.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Added the ability to generate JUnit reports for unit tests. - -authors: simplepad diff --git a/changelog.d/22865_framing_performance.enhancement.md b/changelog.d/22865_framing_performance.enhancement.md deleted file mode 100644 index 00a8479f662dd..0000000000000 --- a/changelog.d/22865_framing_performance.enhancement.md +++ /dev/null @@ -1,3 +0,0 @@ -Reduced unnecessary buffer reallocation when using `framing.method = length_delimited` in sinks for significantly improved performance with large (more than 10MB) batches. - -authors: Ilmarii diff --git a/changelog.d/22909_redis_sink_supports_encoding_input.feature.md b/changelog.d/22909_redis_sink_supports_encoding_input.feature.md deleted file mode 100644 index 52f5514516570..0000000000000 --- a/changelog.d/22909_redis_sink_supports_encoding_input.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -The redis sink now supports any input event type that the configured encoding supports. It previously only supported log events. - -authors: ynachi diff --git a/changelog.d/22921_remove_limit_if_use_apiserver_cache.fix.md b/changelog.d/22921_remove_limit_if_use_apiserver_cache.fix.md deleted file mode 100644 index e0f68868b5ce1..0000000000000 --- a/changelog.d/22921_remove_limit_if_use_apiserver_cache.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a `kubernetes source` bug where `use_apiserver_cache=true` but there is no `resourceVersion=0` parameters in list request. As [the issue](https://github.com/kube-rs/kube/issues/1743) mentioned, when there are `resourceVersion =0` and `!page_size.is_none` in the `ListParams` fields, the parameter `resourceVersion=0` will be ignored by `kube-rs` sdk. If no parameter `resourceVersion` passed to the apiserver, the apiserver will list pods from ETCD instead of in memory cache. - -authors: xiaozongyang diff --git a/changelog.d/22922_healthcheck_timeout_config.feature.md b/changelog.d/22922_healthcheck_timeout_config.feature.md deleted file mode 100644 index 4865e0f047876..0000000000000 --- a/changelog.d/22922_healthcheck_timeout_config.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Added `timeout` config option to the `healthcheck` sink configuration. Previously it was hardcoded to 10 seconds across all components, but now it can be configured per component. - -authors: esensar Quad9DNS diff --git a/changelog.d/22926_enrichment_function_single_bounded_date_range_search.enhancement.md b/changelog.d/22926_enrichment_function_single_bounded_date_range_search.enhancement.md deleted file mode 100644 index e20a27c160a15..0000000000000 --- a/changelog.d/22926_enrichment_function_single_bounded_date_range_search.enhancement.md +++ /dev/null @@ -1,3 +0,0 @@ -The [enrichment functions](https://vector.dev/docs/reference/vrl/functions/#enrichment-functions) now support bounded date range filtering using optional `from` and `to` parameters. There are no changes to the function signatures. - -authors: nzxwang diff --git a/changelog.d/23011_add_relaxed_wildcard_matching.feature.md b/changelog.d/23011_add_relaxed_wildcard_matching.feature.md deleted file mode 100644 index adaf44fd03177..0000000000000 --- a/changelog.d/23011_add_relaxed_wildcard_matching.feature.md +++ /dev/null @@ -1,24 +0,0 @@ -Added `wildcard_matching` global config option to set wildcard matching mode for inputs. Relaxed mode allows configurations with wildcards that do not match any inputs to be accepted without causing an error. - -Example config: - -```yaml -wildcard_matching: relaxed - -sources: - stdin: - type: stdin - -# note - no transforms - -sinks: - stdout: - type: console - encoding: - codec: json - inputs: - - "runtime-added-transform-*" - -``` - -authors: simplepad diff --git a/changelog.d/23047_add_a6_prefix_check_in_dns_parsing.fix.md b/changelog.d/23047_add_a6_prefix_check_in_dns_parsing.fix.md deleted file mode 100644 index 7de9609bce5d7..0000000000000 --- a/changelog.d/23047_add_a6_prefix_check_in_dns_parsing.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a bug that allowed a record to be transmitted with an IPv6 prefix larger than allowed (128) in `dnsmsg_parser::DnsMessageParser` parse functions. - -authors: wooffie diff --git a/changelog.d/23048_fix_arith_logic_dnstap_parse.fix.md b/changelog.d/23048_fix_arith_logic_dnstap_parse.fix.md deleted file mode 100644 index 1a213d2bd4e4a..0000000000000 --- a/changelog.d/23048_fix_arith_logic_dnstap_parse.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Added checks for integer operations with timestamps which used in `dnstap-parser::DnstapParser::parse`. - -authors: wooffie diff --git a/changelog.d/9819_watch_remap_external_vrl.enhancement.md b/changelog.d/9819_watch_remap_external_vrl.enhancement.md deleted file mode 100644 index a77b4c1809d7f..0000000000000 --- a/changelog.d/9819_watch_remap_external_vrl.enhancement.md +++ /dev/null @@ -1,3 +0,0 @@ -Files specified in the `file` and `files` fields of `remap` transforms are now watched when `--watch-config` is enabled. Changes to these files automatically trigger a configuration reload, so there's no need to restart Vector. - -authors: nekorro diff --git a/changelog.d/add_priority_amqp_sink.enhancement.md b/changelog.d/add_priority_amqp_sink.enhancement.md deleted file mode 100644 index 6b665ebef64b1..0000000000000 --- a/changelog.d/add_priority_amqp_sink.enhancement.md +++ /dev/null @@ -1,3 +0,0 @@ -The `amqp` sink now supports setting the `priority` for messages. The priority value can be templated to an integer between 0 and 255 (inclusive). Also, this new field is template-able and Vector templates can now render numerical values. - -authors: aramperes diff --git a/changelog.d/datadog_agent_http_header.fix.md b/changelog.d/datadog_agent_http_header.fix.md deleted file mode 100644 index a075a4976e157..0000000000000 --- a/changelog.d/datadog_agent_http_header.fix.md +++ /dev/null @@ -1,31 +0,0 @@ -Adding an option in the `datadog_logs` sink to allow Vector to mutate the record to conform to the -protocol used by the Datadog Agent itself. To enable use the `conforms_as_agent` option or have the -appropriate agent header (`DD-PROTOCOL: agent-json`) within the additional HTTP Headers list. - -The log will be mutated so that any data that is a reserved keyword that exists at the root will be -moved under a new object keyed at the field 'message'. One will be created if it already does not -exist. As an example: - -```json -{ - "key1": "value1", - "key2": { "key2-1" : "value2" }, - "message" : "Hello world", - ... rest of reserved fields -} -``` - -will be modified to: - -```json -{ - "message" : { - "message" : "Hello world", - "key1": "value1", - "key2": { "key2-1" : "value2" } - }, - ... rest of reserved fields -} -``` - -authors: graphcareful diff --git a/changelog.d/datadog_logs_sink_logs_namespace.fix.md b/changelog.d/datadog_logs_sink_logs_namespace.fix.md deleted file mode 100644 index c430a21b4f448..0000000000000 --- a/changelog.d/datadog_logs_sink_logs_namespace.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fix bug in the `datadog_logs` sink where the content of the log message is dropped when logs namespacing is enabled. - -authors: graphcareful diff --git a/changelog.d/gelf-error.fix.md b/changelog.d/gelf-error.fix.md deleted file mode 100644 index 9f0133e6d550c..0000000000000 --- a/changelog.d/gelf-error.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed misleading error message for invalid field name in gelf encoder. - -authors: mprasil diff --git a/changelog.d/opentelemetry_metrics.feature.md b/changelog.d/opentelemetry_metrics.feature.md deleted file mode 100644 index 824383d14747b..0000000000000 --- a/changelog.d/opentelemetry_metrics.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -The `opentelemetry` source now supports metrics ingestion. - -authors: cmcmacs diff --git a/changelog.d/s3-source-defer-or-delete.enhancement.md b/changelog.d/s3-source-defer-or-delete.enhancement.md deleted file mode 100644 index c4d0b54ccce4a..0000000000000 --- a/changelog.d/s3-source-defer-or-delete.enhancement.md +++ /dev/null @@ -1,3 +0,0 @@ -Add deferred processing options (`deferred.max_age_secs`, `deferred.queue_url`) to automatically route older event notifications to a separate queue, enabling prioritized processing of recent files - -authors: akutta diff --git a/changelog.d/window-transform.feature.md b/changelog.d/window-transform.feature.md deleted file mode 100644 index b3f366108c625..0000000000000 --- a/changelog.d/window-transform.feature.md +++ /dev/null @@ -1,5 +0,0 @@ -Add a new `window` transform, a variant of ring buffer or backtrace logging implemented as a sliding window. -Allows for reduction of log volume by filtering out logs when the system is healthy, but preserving detailed -logs when they are most relevant. - -authors: ilinas diff --git a/distribution/install.sh b/distribution/install.sh index 13d8f847b6e06..e96aff7fe4c4e 100755 --- a/distribution/install.sh +++ b/distribution/install.sh @@ -13,7 +13,7 @@ set -u # If PACKAGE_ROOT is unset or empty, default it. PACKAGE_ROOT="${PACKAGE_ROOT:-"https://packages.timber.io/vector"}" # If VECTOR_VERSION is unset or empty, default it. -VECTOR_VERSION="${VECTOR_VERSION:-"0.46.1"}" +VECTOR_VERSION="${VECTOR_VERSION:-"0.47.0"}" _divider="--------------------------------------------------------------------------------" _prompt=">>>" _indent=" " diff --git a/distribution/kubernetes/vector-agent/README.md b/distribution/kubernetes/vector-agent/README.md index 27da769cff90c..95b20cde3fb8d 100644 --- a/distribution/kubernetes/vector-agent/README.md +++ b/distribution/kubernetes/vector-agent/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.42.1 with the following `values.yaml`: +version 0.43.0 with the following `values.yaml`: ```yaml role: Agent diff --git a/distribution/kubernetes/vector-agent/configmap.yaml b/distribution/kubernetes/vector-agent/configmap.yaml index 6db2a2ea95a6b..1eb3b3a2b8f88 100644 --- a/distribution/kubernetes/vector-agent/configmap.yaml +++ b/distribution/kubernetes/vector-agent/configmap.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.47.0-distroless-libc" data: agent.yaml: | data_dir: /vector-data-dir diff --git a/distribution/kubernetes/vector-agent/daemonset.yaml b/distribution/kubernetes/vector-agent/daemonset.yaml index 41d64e003a753..b1ef77d0dd828 100644 --- a/distribution/kubernetes/vector-agent/daemonset.yaml +++ b/distribution/kubernetes/vector-agent/daemonset.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.47.0-distroless-libc" spec: selector: matchLabels: @@ -30,7 +30,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "timberio/vector:0.46.1-distroless-libc" + image: "timberio/vector:0.47.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir diff --git a/distribution/kubernetes/vector-agent/rbac.yaml b/distribution/kubernetes/vector-agent/rbac.yaml index 4c776a65970e3..f1e1f7351ddd5 100644 --- a/distribution/kubernetes/vector-agent/rbac.yaml +++ b/distribution/kubernetes/vector-agent/rbac.yaml @@ -10,7 +10,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.47.0-distroless-libc" rules: - apiGroups: - "" @@ -31,7 +31,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.47.0-distroless-libc" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole diff --git a/distribution/kubernetes/vector-agent/service-headless.yaml b/distribution/kubernetes/vector-agent/service-headless.yaml index 45fc274219f22..c7a5c971fc2f7 100644 --- a/distribution/kubernetes/vector-agent/service-headless.yaml +++ b/distribution/kubernetes/vector-agent/service-headless.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.47.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-agent/serviceaccount.yaml b/distribution/kubernetes/vector-agent/serviceaccount.yaml index 6582dfe20ca82..18198d14032b0 100644 --- a/distribution/kubernetes/vector-agent/serviceaccount.yaml +++ b/distribution/kubernetes/vector-agent/serviceaccount.yaml @@ -9,5 +9,5 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.47.0-distroless-libc" automountServiceAccountToken: true diff --git a/distribution/kubernetes/vector-aggregator/README.md b/distribution/kubernetes/vector-aggregator/README.md index f50f97fa32989..2940d28c48a1a 100644 --- a/distribution/kubernetes/vector-aggregator/README.md +++ b/distribution/kubernetes/vector-aggregator/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.42.1 with the following `values.yaml`: +version 0.43.0 with the following `values.yaml`: ```yaml diff --git a/distribution/kubernetes/vector-aggregator/configmap.yaml b/distribution/kubernetes/vector-aggregator/configmap.yaml index c3b6e0ac25ada..5548ef74f65ed 100644 --- a/distribution/kubernetes/vector-aggregator/configmap.yaml +++ b/distribution/kubernetes/vector-aggregator/configmap.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.47.0-distroless-libc" data: aggregator.yaml: | data_dir: /vector-data-dir diff --git a/distribution/kubernetes/vector-aggregator/service-headless.yaml b/distribution/kubernetes/vector-aggregator/service-headless.yaml index c88a67f2b310b..b3e79b9d67718 100644 --- a/distribution/kubernetes/vector-aggregator/service-headless.yaml +++ b/distribution/kubernetes/vector-aggregator/service-headless.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.47.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-aggregator/service.yaml b/distribution/kubernetes/vector-aggregator/service.yaml index f85a74c2acdab..fe65a612a1fc0 100644 --- a/distribution/kubernetes/vector-aggregator/service.yaml +++ b/distribution/kubernetes/vector-aggregator/service.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.47.0-distroless-libc" annotations: spec: ports: diff --git a/distribution/kubernetes/vector-aggregator/serviceaccount.yaml b/distribution/kubernetes/vector-aggregator/serviceaccount.yaml index ca52b73bb4fca..cab03421b9df7 100644 --- a/distribution/kubernetes/vector-aggregator/serviceaccount.yaml +++ b/distribution/kubernetes/vector-aggregator/serviceaccount.yaml @@ -9,5 +9,5 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.47.0-distroless-libc" automountServiceAccountToken: true diff --git a/distribution/kubernetes/vector-aggregator/statefulset.yaml b/distribution/kubernetes/vector-aggregator/statefulset.yaml index 03a554ba2568c..a8f500eec9656 100644 --- a/distribution/kubernetes/vector-aggregator/statefulset.yaml +++ b/distribution/kubernetes/vector-aggregator/statefulset.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.47.0-distroless-libc" annotations: {} spec: replicas: 1 @@ -34,7 +34,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "timberio/vector:0.46.1-distroless-libc" + image: "timberio/vector:0.47.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir diff --git a/distribution/kubernetes/vector-stateless-aggregator/README.md b/distribution/kubernetes/vector-stateless-aggregator/README.md index 773c7190c08e7..73042c237387b 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/README.md +++ b/distribution/kubernetes/vector-stateless-aggregator/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.42.1 with the following `values.yaml`: +version 0.43.0 with the following `values.yaml`: ```yaml role: Stateless-Aggregator diff --git a/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml b/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml index 6e2277f4b0b8d..9285d34da1486 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.47.0-distroless-libc" data: aggregator.yaml: | data_dir: /vector-data-dir diff --git a/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml b/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml index abd9830d1e9a2..90e636d234352 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.47.0-distroless-libc" annotations: {} spec: replicas: 1 @@ -32,7 +32,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "timberio/vector:0.46.1-distroless-libc" + image: "timberio/vector:0.47.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir diff --git a/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml b/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml index ce28b3f972de6..a79fa284683e8 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.47.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-stateless-aggregator/service.yaml b/distribution/kubernetes/vector-stateless-aggregator/service.yaml index 0a5a0cb3eea98..d4ec508ea0188 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/service.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/service.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.47.0-distroless-libc" annotations: spec: ports: diff --git a/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml b/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml index 9c6e57f0f5155..ab1579b288218 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml @@ -9,5 +9,5 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.46.1-distroless-libc" + app.kubernetes.io/version: "0.47.0-distroless-libc" automountServiceAccountToken: true diff --git a/docker_logs.txt b/docker_logs.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/website/content/en/docs/reference/configuration/sources/mqtt.md b/website/content/en/docs/reference/configuration/sources/mqtt.md index 179b0bce2e9d5..86560c96d566f 100644 --- a/website/content/en/docs/reference/configuration/sources/mqtt.md +++ b/website/content/en/docs/reference/configuration/sources/mqtt.md @@ -1,7 +1,7 @@ --- title: MQTT description: Receive logs from an [MQTT](https://mqtt.org) broker -kind: source +component_kind: source layout: component tags: ["mqtt", "component", "source"] --- diff --git a/website/content/en/releases/0.47.0.md b/website/content/en/releases/0.47.0.md new file mode 100644 index 0000000000000..5bad0ddc3fbdb --- /dev/null +++ b/website/content/en/releases/0.47.0.md @@ -0,0 +1,4 @@ +--- +title: Vector v0.47.0 release notes +weight: 25 +--- diff --git a/website/cue/reference/administration/interfaces/kubectl.cue b/website/cue/reference/administration/interfaces/kubectl.cue index 67b39c183d675..ea5f5b2b9c4a1 100644 --- a/website/cue/reference/administration/interfaces/kubectl.cue +++ b/website/cue/reference/administration/interfaces/kubectl.cue @@ -19,7 +19,7 @@ administration: interfaces: kubectl: { role_implementations: [Name=string]: { commands: { _deployment_variant: string - _vector_version: "0.46" + _vector_version: "0.47" _namespace: string | *"vector" _controller_resource_type: string _controller_resource_name: string | *_deployment_variant diff --git a/website/cue/reference/releases/0.47.0.cue b/website/cue/reference/releases/0.47.0.cue new file mode 100644 index 0000000000000..d6edbe6388ade --- /dev/null +++ b/website/cue/reference/releases/0.47.0.cue @@ -0,0 +1,419 @@ +package metadata + +releases: "0.47.0": { + date: "2025-05-20" + codename: "" + + whats_next: [] + + description: """ + The Vector team is excited to announce version `0.47.0`! + + Release highlights: + + - The `opentelemetry` source now supports metrics ingestion. + - A new `window` transform has been introduced which enables log noise reduction by filtering out events when the system is in a healthy state. + - A new `mqtt` source is now available, enabling ingestion from MQTT brokers. + - The `datadog_logs` sink now supports a new `conforms_as_agent` option to format logs like the Datadog Agent, ensuring compatibility with reserved fields. + """ + + vrl_changelog: """ + VRL was updated to `v0.24.0`. This includes the following changes: + + #### Enhancements + + - The `encode_gzip`, `decode_gzip`, `encode_zlib`, and `decode_zlib` methods now use the [zlib-rs](https://github.com/trifectatechfoundation/zlib-rs) backend. + which is much faster than the previous backend `miniz_oxide`. + + - The `decode_base64`, `encode_base64`, and `decode_mime_q` functions now use the SIMD backend. + which is faster than the previous backend. + + #### Fixes + + - Add BOM stripping logic to the parse_json function. + """ + + changelog: [ + { + type: "feat" + description: """ + The `opentelemetry` source now supports metrics ingestion. + """ + contributors: ["cmcmacs"] + }, + { + type: "feat" + description: """ + Add a new `window` transform, a variant of ring buffer or backtrace logging implemented as a sliding window. + Allows for reduction of log volume by filtering out logs when the system is healthy, but preserving detailed + logs when they are most relevant. + """ + contributors: ["ilinas"] + }, + { + type: "feat" + description: """ + Add a new `mqtt` source enabling Vector to receive logs from a MQTT broker. + """ + contributors: ["mladedav", "pront", "StormStake"] + }, + { + type: "feat" + description: """ + Add support for rendering to Mermaid format in `vector graph` + """ + contributors: ["Firehed"] + }, + { + type: "fix" + description: """ + Fix a Vector crash that occurred when the internal metrics generated too many groups by increasing groups max limit from 128 to 256. + """ + contributors: ["triggerhappy17"] + }, + { + type: "feat" + description: """ + Allow users to specify AWS authentication and the AWS service name for HTTP sinks to support AWS API endpoints that require SigV4. + """ + contributors: ["johannesfloriangeiger"] + }, + { + type: "feat" + description: """ + Add support for fluentd forwarding over a Unix socket + """ + contributors: ["tustvold"] + }, + { + type: "enhancement" + description: """ + Zlib compression and decompression are now more efficient by using [zlib-rs](https://github.com/trifectatechfoundation/zlib-rs). + """ + contributors: ["JakubOnderka"] + }, + { + type: "feat" + description: """ + Add ACK support to message buffering feature of `websocket_server` sink, allowing this component to cache latest received messages per client. + """ + contributors: ["esensar", "Quad9DNS"] + }, + { + type: "chore" + description: """ + Add a new `extra_headers` option to `greptimedb_logs` sink configuration to set additional headers for outgoing requests. + + Change `greptimedb_logs` sink default content type to `application/x-ndjson` to match the default content type of `greptimedb` sink. + If you use the greptimedb version v0.12 or earlier, you need to set the content type to `application/json` in the sink configuration. + + Example: + + ```yaml + sinks: + greptime_logs: + type: greptimedb_logs + inputs: ["my_source_id"] + endpoint: "http://localhost:4000" + table: "demo_logs" + dbname: "public" + extra_headers: + x-source: vector + ``` + + ```toml + [sinks.greptime_logs] + type = "greptimedb_logs" + inputs = ["my_source_id"] + endpoint = "http://localhost:4000" + table = "demo_logs" + dbname = "public" + + [sinks.greptime_logs.extra_headers] + x-source = "vector" + ``` + """ + contributors: ["greptimedb"] + }, + { + type: "feat" + description: """ + Introduce a configuration option in the StatsD source: `convert_to` of type `ConversionUnit`. By default, timing values in milliseconds (`ms`) are converted to seconds (`s`). Users can set `convert_to` to `milliseconds` to preserve the original millisecond values. + """ + contributors: ["devkoriel"] + }, + { + type: "fix" + description: """ + Fix `file` source bug where known small files were not deleted after the specified `remove_after_secs`. + """ + contributors: ["linw1995"] + }, + { + type: "fix" + description: """ + Fix an AWS authentication bug where `region` was missing from the `STS` authentication endpoint. + """ + contributors: ["cahartma"] + }, + { + type: "fix" + description: """ + Increase the max event size for `aws_cloudwatch_logs` sink to ~1MB. + """ + contributors: ["cahartma"] + }, + { + type: "feat" + description: """ + The `address` field is now available within VRL scripts when using the `auth.strategy.custom` authentication method. + """ + contributors: ["esensar", "Quad9DNS"] + }, + { + type: "feat" + description: """ + Add support for the `Sec-WebSocket-Protocol` header in the `websocket_server` sink to better accommodate clients that require it. + """ + contributors: ["esensar", "Quad9DNS"] + }, + { + type: "enhancement" + description: """ + Reduce unnecessary buffer reallocation when using `framing.method = length_delimited` in sinks for significantly improved performance with large (more than 10MB) batches. + """ + contributors: ["Ilmarii"] + }, + { + type: "feat" + description: """ + The redis sink now supports any input event type that the configured encoding supports. It previously only supported log events. + """ + contributors: ["ynachi"] + }, + { + type: "fix" + description: """ + Fix a `kubernetes source` bug where `use_apiserver_cache=true` but there is no `resourceVersion=0` parameter in list request. Per [this issue](https://github.com/kube-rs/kube/issues/1743), when `resourceVersion =0` and `!page_size.is_none` in`ListParams`, the parameter `resourceVersion=0` will be ignored by `kube-rs` sdk. If no parameter `resourceVersion` passed to the apiserver, the apiserver will list pods from ETCD instead of in memory cache. + """ + contributors: ["xiaozongyang"] + }, + { + type: "feat" + description: """ + Add `timeout` config option to the `healthcheck` sink configuration. Previously it was hardcoded to 10 seconds across all components, but now it can be configured per component. + """ + contributors: ["esensar", "Quad9DNS"] + }, + { + type: "enhancement" + description: """ + The [enrichment functions](https://vector.dev/docs/reference/vrl/functions/#enrichment-functions) now support bounded date range filtering using optional `from` and `to` parameters. There are no changes to the function signatures. + """ + contributors: ["nzxwang"] + }, + { + type: "feat" + description: """ + Add `wildcard_matching` global config option to set wildcard matching mode for inputs. Relaxed mode allows configurations with wildcards that do not match any inputs to be accepted without causing an error. + + Example config: + + ```yaml + wildcard_matching: relaxed + + sources: + stdin: + type: stdin + + # note - no transforms + + sinks: + stdout: + type: console + encoding: + codec: json + inputs: + - "runtime-added-transform-*" + + ``` + """ + contributors: ["simplepad"] + }, + { + type: "fix" + description: """ + Fix a bug that allows DNS records with an IPv6 prefix length greater than 128 to be transmitted; invalid prefixes are now rejected during parsing. + """ + contributors: ["wooffie"] + }, + { + type: "fix" + description: """ + Add checks to prevent invalid timestamps operations during DNS tap parsing; such operations are now validated to ensure correctness. + """ + contributors: ["wooffie"] + }, + { + type: "enhancement" + description: """ + Files specified in the `file` and `files` fields of `remap` transforms are now watched when `--watch-config` is enabled. Changes to these files automatically trigger a configuration reload, so there's no need to restart Vector. + """ + contributors: ["nekorro"] + }, + { + type: "enhancement" + description: """ + The `amqp` sink now supports setting the `priority` for messages. The value can be templated to an integer 0-255 (inclusive). + """ + contributors: ["aramperes"] + }, + { + type: "fix" + description: """ + Add an option in the `datadog_logs` sink to allow Vector to mutate the record to conform to the + protocol used by the Datadog Agent itself. To enable, use the `conforms_as_agent` option or have the + appropriate agent header (`DD-PROTOCOL: agent-json`) within the additional HTTP Headers list. + + Any top-level fields that use Datadog-reserved keywords are moved into a new object named `message`. If `message` doesn’t exist, it is created first. For example: + + ```json + { + "key1": "value1", + "key2": { "key2-1" : "value2" }, + "message" : "Hello world", + ... rest of reserved fields + } + ``` + + will be modified to: + + ```json + { + "message" : { + "message" : "Hello world", + "key1": "value1", + "key2": { "key2-1" : "value2" } + }, + ... rest of reserved fields + } + ``` + """ + contributors: ["graphcareful"] + }, + { + type: "fix" + description: """ + Fix a bug in the `datadog_logs` sink where the content of the log message is dropped when logs namespacing is enabled. + """ + contributors: ["graphcareful"] + }, + { + type: "fix" + description: """ + Fix misleading error message for invalid field name in gelf encoder. + """ + contributors: ["mprasil"] + }, + { + type: "enhancement" + description: """ + Add `deferred.max_age_secs` and `deferred.queue_url` options to the `aws_s3` and `aws_sqs` sinks, to automatically + route older event notifications to a separate queue, allowing prioritized processing of recent files. + """ + contributors: ["akutta"] + }, + ] + + commits: [ + {sha: "37803453653444ce1b210b66cdc1e64e500a970f", date: "2025-04-07 17:59:45 UTC", description: "Bump indexmap from 2.8.0 to 2.9.0", pr_number: 22814, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 3, insertions_count: 27, deletions_count: 27}, + {sha: "bb3680636d9ae058d608999f0c4424e3e7f1d9f7", date: "2025-04-07 18:28:41 UTC", description: "Bump smallvec from 1.14.0 to 1.15.0", pr_number: 22817, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 2, deletions_count: 2}, + {sha: "1547c15f4c7bebae56d0e8d595ece5789be3b54e", date: "2025-04-07 22:43:10 UTC", description: "Bump openssl from 0.10.71 to 0.10.72 in the cargo group", pr_number: 22802, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 3, insertions_count: 6, deletions_count: 6}, + {sha: "e74b60cd8d468b19c683d7152699067d1c043781", date: "2025-04-07 23:11:34 UTC", description: "Bump crossterm from 0.28.1 to 0.29.0", pr_number: 22815, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 3, insertions_count: 59, deletions_count: 25}, + {sha: "92e2a2f6a9b701e8e6d501cb2452a5850d5e19f5", date: "2025-04-07 19:43:58 UTC", description: "update tokio to fix RUSTSEC-2025-0023", pr_number: 22820, scopes: ["deps"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 3, deletions_count: 3}, + {sha: "081b3044ea8e11763374e49958208e618ee54408", date: "2025-04-08 01:06:44 UTC", description: "Bump vrl from `7020ba2` to `048253d`", pr_number: 22812, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 68, deletions_count: 48}, + {sha: "b0823b6364c7d6702a84cae1698b5b6b51b0664d", date: "2025-04-07 21:20:10 UTC", description: "update interval changed to 'monthly'", pr_number: 22822, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 4, deletions_count: 4}, + {sha: "5d9297b35bd9d132292663bc6623c9c34c3efef2", date: "2025-04-08 12:41:58 UTC", description: "add documentation for Lz4 encode and decode features in vrl", pr_number: 22702, scopes: ["external"], type: "docs", breaking_change: false, author: "James Lamb", files_count: 2, insertions_count: 62, deletions_count: 0}, + {sha: "5ccca8ae3098c1aefc65e7222e60ea4819f0b152", date: "2025-04-07 23:17:57 UTC", description: "update CODEOWNERS files for cue docs PRs", pr_number: 22824, scopes: ["administration"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 1, deletions_count: 0}, + {sha: "60048c814bffac01ac49b84045bce37ad5b4190b", date: "2025-04-08 19:55:19 UTC", description: "fix website branch name", pr_number: 22830, scopes: ["releasing"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 14, deletions_count: 13}, + {sha: "cf9185f1e47bad13ae6ffbe14f2f2cbcb0c051f4", date: "2025-04-09 04:15:11 UTC", description: "Use zlib-rs for much faster zlib decoding/encoding", pr_number: 22533, scopes: ["deps"], type: "enhancement", breaking_change: false, author: "Jakub Onderka", files_count: 6, insertions_count: 32, deletions_count: 9}, + {sha: "e1491d84812a1892053d57482bc322bb061bad90", date: "2025-04-09 00:03:21 UTC", description: "Refactor reduce transform logic", pr_number: 22829, scopes: ["reduce transform"], type: "enhancement", breaking_change: false, author: "ArunPiduguDD", files_count: 2, insertions_count: 40, deletions_count: 26}, + {sha: "7d9242aaa13814b2eaf9157b3858ce6bc5d59572", date: "2025-04-09 00:26:01 UTC", description: "update homebrew repo link", pr_number: 22832, scopes: ["releasing"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "1de7e9d6703487c23e74312bafd130ff324dd3ff", date: "2025-04-09 00:57:41 UTC", description: "#22827 missing region error when using AWS credentials file auth", pr_number: 22831, scopes: ["auth"], type: "fix", breaking_change: false, author: "Casey Hartman", files_count: 2, insertions_count: 21, deletions_count: 1}, + {sha: "6a59a873a520bbe736f23d09108f019ba29a6d88", date: "2025-04-09 05:10:57 UTC", description: "Bump sysinfo from 0.32.1 to 0.34.2", pr_number: 22816, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 4, insertions_count: 15, deletions_count: 8}, + {sha: "c2e082052c1c637d66ecdc8b5b2a5e30b2749d77", date: "2025-04-09 01:39:38 UTC", description: "cherry pick release prep commit", pr_number: 22836, scopes: ["releasing"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 30, insertions_count: 398, deletions_count: 96}, + {sha: "050743b0134ecc2e88436f903c705aa31b8b6cf1", date: "2025-04-09 02:25:51 UTC", description: "cargo vdev build manifests", pr_number: 22833, scopes: ["releasing"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 18, insertions_count: 37, deletions_count: 37}, + {sha: "668caf89b64c5c412631909bfbf971098d5747ed", date: "2025-04-10 00:04:10 UTC", description: "add ACK support for message buffering", pr_number: 22540, scopes: ["websocket_server sink"], type: "feat", breaking_change: false, author: "Ensar Sarajčić", files_count: 6, insertions_count: 652, deletions_count: 47}, + {sha: "906a9bd900e7df402417a5544392b8792bf34c86", date: "2025-04-09 18:19:35 UTC", description: "PR template and CONTRIBUTING.md enhancements", pr_number: 22788, scopes: ["dev"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 2, insertions_count: 43, deletions_count: 4}, + {sha: "6a39e6a42042761586aa115975926a65538fb76f", date: "2025-04-09 22:44:14 UTC", description: "add known issue to v0.46.0", pr_number: 22842, scopes: ["releasing"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 7, deletions_count: 0}, + {sha: "238e8b8c26718ab19f4c5d97ef9dc45e3c6f1e06", date: "2025-04-11 12:17:53 UTC", description: "fix dead links to configuration reference", pr_number: 22846, scopes: ["external"], type: "docs", breaking_change: false, author: "Shin Seunghun", files_count: 6, insertions_count: 15, deletions_count: 15}, + {sha: "c1ad3421e1215d83a376c5541412c0359682752a", date: "2025-04-12 00:58:38 UTC", description: "properly display client count in websocket_server debug logs", pr_number: 22855, scopes: ["websocket_server sink"], type: "fix", breaking_change: false, author: "Ensar Sarajčić", files_count: 1, insertions_count: 12, deletions_count: 2}, + {sha: "eb60aa05d6e3d5c390de0f581e24cebaf31cca9c", date: "2025-04-12 07:42:50 UTC", description: "Bump OpenDAL to v0.53.0", pr_number: 21493, scopes: ["deps"], type: "chore", breaking_change: false, author: "Xuanwo", files_count: 5, insertions_count: 45, deletions_count: 31}, + {sha: "0a613370046c1f25ed747a829bbf3f665282a41d", date: "2025-04-12 04:13:29 UTC", description: "Bump maxminddb from 0.25.0 to 0.26.0 ", pr_number: 22804, scopes: ["deps"], type: "chore", breaking_change: false, author: "Jakub Onderka", files_count: 4, insertions_count: 13, deletions_count: 12}, + {sha: "ee6dc973ed600e1ba4e657a0de1dd65a7f56020a", date: "2025-04-12 07:19:26 UTC", description: "add support for websocket subprotocol", pr_number: 22854, scopes: ["websocket_server sink"], type: "feat", breaking_change: false, author: "Ensar Sarajčić", files_count: 4, insertions_count: 105, deletions_count: 2}, + {sha: "9dfa36bafbeefe6841d443f0430473ee4216fb87", date: "2025-04-12 05:55:27 UTC", description: "Bump tokio from 1.44.1 to 1.44.2 in the cargo group", pr_number: 22861, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 11, insertions_count: 18, deletions_count: 18}, + {sha: "b30a36cf42c283ef8128c5720b6fc1315dad6eb5", date: "2025-04-15 04:09:56 UTC", description: "add ability to generate JUnit reports", pr_number: 22857, scopes: ["unit tests"], type: "feat", breaking_change: false, author: "simplepad", files_count: 4, insertions_count: 98, deletions_count: 0}, + {sha: "07b4c71b4d35746664fcb5bf8dca7d941fef65c8", date: "2025-04-14 21:43:30 UTC", description: "enhancements to the patch release template", pr_number: 22871, scopes: ["dev"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 6, deletions_count: 6}, + {sha: "c8b57ded8591ff70dd0bd65f0ba3d77f55675dd0", date: "2025-04-15 00:47:28 UTC", description: "update Cargo.lock", pr_number: 22873, scopes: ["deps"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 34, deletions_count: 0}, + {sha: "cb62efe251e78470c7e3f7e7b718f2c87d45db27", date: "2025-04-14 23:50:12 UTC", description: "enhancements to the patch release template", pr_number: 22872, scopes: ["dev"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 2, deletions_count: 2}, + {sha: "f35398a7c2689f5cc4f5bb6c0545f0bb2cc7940c", date: "2025-04-15 05:15:54 UTC", description: "adding metric ingestion to opentelemetry source", pr_number: 22746, scopes: ["opentelemetry source"], type: "feat", breaking_change: false, author: "Kirill Mikhailov", files_count: 16, insertions_count: 1833, deletions_count: 292}, + {sha: "7cfc9c54bffba9a603d58eb01134305b50ef5f4a", date: "2025-04-15 01:06:11 UTC", description: "add v0.46.1 patch commits", pr_number: 22874, scopes: ["releasing"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 4, insertions_count: 27, deletions_count: 1}, + {sha: "2c038c3b2600c24520be6d8e63be0f6cb57de9c0", date: "2025-04-15 17:44:48 UTC", description: "fix rendering of 0.46.1.cue", pr_number: 22882, scopes: ["website"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "3f7026f05c7bd485297ef6469fcd6d512d6c8518", date: "2025-04-15 23:12:18 UTC", description: "bump darling version", pr_number: 22883, scopes: ["deps"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 5, insertions_count: 19, deletions_count: 18}, + {sha: "8029c109fbaf8af5822dce72a678c54db02810c1", date: "2025-04-16 06:17:20 UTC", description: "support all formats: YAML,TOML,JSON", pr_number: 22864, scopes: ["config provider"], type: "enhancement", breaking_change: false, author: "Kirill Nazarov", files_count: 3, insertions_count: 39, deletions_count: 8}, + {sha: "dc614442826c829acf45a554c9cace766a9f5a31", date: "2025-04-15 23:29:45 UTC", description: "build manifests - 0.46.1", pr_number: 22887, scopes: ["releasing"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 18, insertions_count: 22, deletions_count: 22}, + {sha: "3e1327890e27b9f62ac7631fa72198fe11c8922a", date: "2025-04-15 23:44:39 UTC", description: "increase cloudwatch logs max event size to match new aws limit of 1MB", pr_number: 22886, scopes: ["aws_cloudwatch_logs sink"], type: "fix", breaking_change: false, author: "Casey Hartman", files_count: 2, insertions_count: 29, deletions_count: 6}, + {sha: "82ffecbf4b869922643b8ac51aecf972c05b64d1", date: "2025-04-15 20:57:59 UTC", description: "Add support for rendering to mermaid with `vector graph` command", pr_number: 22787, scopes: ["cli"], type: "feat", breaking_change: false, author: "Eric Stern", files_count: 2, insertions_count: 66, deletions_count: 0}, + {sha: "e293b574a3141deff09fb809e87d8cb66f3de608", date: "2025-04-16 22:55:25 UTC", description: "Improve performance with `length_delimited` framing and large batches", pr_number: 22877, scopes: ["codecs"], type: "enhancement", breaking_change: false, author: "Alex Savitskii", files_count: 2, insertions_count: 20, deletions_count: 5}, + {sha: "ce170d5e6062083af476277579361373e23af97a", date: "2025-04-17 00:22:11 UTC", description: "Apply agent-json header on events from agent", pr_number: 22701, scopes: ["datadog_logs sink"], type: "fix", breaking_change: false, author: "Rob Blafford", files_count: 6, insertions_count: 454, deletions_count: 107}, + {sha: "3ea8c86f9461f1e3d403c3c6820fdf19b280fe75", date: "2025-04-17 06:40:08 UTC", description: "Add new `window` transform", pr_number: 22609, scopes: ["new transform"], type: "feat", breaking_change: false, author: "Linas Zvirblis", files_count: 13, insertions_count: 1142, deletions_count: 0}, + {sha: "42caf99f7bb027a02ed9fef622f2d5a7a1767816", date: "2025-04-19 00:13:45 UTC", description: "add access to client address in custom VRL auth", pr_number: 22850, scopes: ["sources"], type: "feat", breaking_change: false, author: "Ensar Sarajčić", files_count: 5, insertions_count: 82, deletions_count: 32}, + {sha: "8057e633eb0b97f1286848d6f3f393524ae4c643", date: "2025-04-19 00:57:52 UTC", description: "Add support for forwarding over unix socket", pr_number: 22212, scopes: ["fluent source"], type: "enhancement", breaking_change: false, author: "Raphael Taylor-Davies", files_count: 5, insertions_count: 339, deletions_count: 93}, + {sha: "3c10200690febf375cc80c57cecbcac77a19186d", date: "2025-04-22 08:39:59 UTC", description: "Remove known small files based on remove_after_secs", pr_number: 22786, scopes: ["file source"], type: "fix", breaking_change: false, author: "林玮 (Jade Lin)", files_count: 3, insertions_count: 54, deletions_count: 18}, + {sha: "8861538469ce8f002493ff4699dae64376d62ded", date: "2025-04-22 11:50:43 UTC", description: "add `convert_to` option for controlling timing unit conversion", pr_number: 22716, scopes: ["statsd source"], type: "feat", breaking_change: false, author: "Jinsoo Heo", files_count: 7, insertions_count: 165, deletions_count: 29}, + {sha: "d7c27eb3ac131b7631391330a402bb7797a2ebba", date: "2025-04-21 23:38:52 UTC", description: "optionally configure source to defer or delete older notifications", pr_number: 22691, scopes: ["aws_s3 source"], type: "enhancement", breaking_change: false, author: "Andrew Kutta", files_count: 4, insertions_count: 317, deletions_count: 7}, + {sha: "ca9fa2e61e26243b12c54ea0a9aaf6997acadd31", date: "2025-04-23 04:43:17 UTC", description: "`AllocationGroupId::register` groups max limit increased to 256", pr_number: 22897, scopes: ["internal"], type: "fix", breaking_change: false, author: "triggerh@ppy", files_count: 2, insertions_count: 7, deletions_count: 7}, + {sha: "18d6867a64ec0884f9d3a5f9af6ae25a06041fb2", date: "2025-04-23 03:47:28 UTC", description: "add a config option for setting the healthcheck timeout", pr_number: 22922, scopes: ["config"], type: "feat", breaking_change: false, author: "Ensar Sarajčić", files_count: 7, insertions_count: 58, deletions_count: 15}, + {sha: "e3d7ad090af6b34f46a88f2f774ab715813253e5", date: "2025-04-23 01:48:37 UTC", description: "Remove datadog specific telemetry", pr_number: 22927, scopes: ["datadog service"], type: "docs", breaking_change: false, author: "Jesse Szwedko", files_count: 1, insertions_count: 0, deletions_count: 12}, + {sha: "997beb8919284bbbfa630ec51ac28788ab20a1cc", date: "2025-04-23 20:44:13 UTC", description: "guide page improvements", pr_number: 22936, scopes: ["website"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 9, insertions_count: 45, deletions_count: 39}, + {sha: "b0bac1e1ac2ef5dfae7a2a39d6d9b542ba3556e3", date: "2025-04-23 19:25:01 UTC", description: "fix typo in CONTRIBUTING.md", pr_number: 22930, scopes: ["external"], type: "docs", breaking_change: false, author: "Nick Wang", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "861075f2a69d8c6213e358527e9ab90b84763341", date: "2025-04-25 03:23:10 UTC", description: "Implement components selection in diff.rs and tests", pr_number: 22678, scopes: ["config"], type: "enhancement", breaking_change: false, author: "Guillaume Le Blanc", files_count: 10, insertions_count: 172, deletions_count: 98}, + {sha: "1e7da76fc0f178284d2a4e0291128fc8ddce67c3", date: "2025-04-25 10:03:29 UTC", description: "add headers field to greptimedb_logs config", pr_number: 22651, scopes: ["greptimedb_logs sink"], type: "enhancement", breaking_change: false, author: "localhost", files_count: 5, insertions_count: 71, deletions_count: 2}, + {sha: "e2f84ea86ffe29a01f919dcab4bbfa6df7d4744f", date: "2025-04-29 02:37:27 UTC", description: "Allows users to specify AWS authentication and the AWS service name for HTTP sinks to support AWS API endpoints that require SigV4.", pr_number: 22744, scopes: ["http sink"], type: "feat", breaking_change: false, author: "Johannes Geiger", files_count: 19, insertions_count: 1390, deletions_count: 27}, + {sha: "d771ab1b753544a2b4e382c09bb13b7e58ce29be", date: "2025-04-28 23:01:36 UTC", description: "add priority property", pr_number: 22243, scopes: ["amqp sink"], type: "feat", breaking_change: false, author: "Aram Peres", files_count: 8, insertions_count: 529, deletions_count: 16}, + {sha: "96bc5942ac5a0d082e14ba706133631d094f3061", date: "2025-04-29 01:08:52 UTC", description: "updates concepts/traces page", pr_number: 22955, scopes: ["external"], type: "docs", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 19, deletions_count: 5}, + {sha: "4b4b16e31cfdf9fc7dac67ecf48192939067742a", date: "2025-04-30 01:01:56 UTC", description: "fix reduce transfrom examples", pr_number: 22962, scopes: ["external"], type: "docs", breaking_change: false, author: "Pavlos Rontidis", files_count: 3, insertions_count: 15, deletions_count: 4}, + {sha: "4ad3fdc54c2c3b56db94a8613417783458a96111", date: "2025-04-30 18:53:50 UTC", description: "Remove `_collisions` field in agent normalization routine", pr_number: 22956, scopes: ["datadog_logs sink"], type: "fix", breaking_change: false, author: "Rob Blafford", files_count: 1, insertions_count: 16, deletions_count: 136}, + {sha: "f4c15e33b6fccdf5c7ce44f05dc2782790c9a9d2", date: "2025-05-01 17:37:03 UTC", description: "Bump vrl from `d802b30` to `7f8ed50`", pr_number: 22977, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 3, deletions_count: 3}, + {sha: "b6b9c677ac8e7ee158ba5f210750fd3ce97ff88b", date: "2025-05-01 17:37:32 UTC", description: "Bump smallvec from 1.14.0 to 1.15.0", pr_number: 22982, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 2, deletions_count: 2}, + {sha: "b70e555a1e2c563de826683eb1942ca8d88cb1bb", date: "2025-05-01 18:26:48 UTC", description: "Bump lru from 0.13.0 to 0.14.0", pr_number: 22981, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 4, deletions_count: 4}, + {sha: "4f3aad5b6ad68fc4a7514d5959575d96da167552", date: "2025-05-01 22:27:20 UTC", description: "Bump bstr from 1.11.3 to 1.12.0", pr_number: 22980, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 6, deletions_count: 6}, + {sha: "db929085e7b7ffb6adf4b16b6fd0009c4afe1b25", date: "2025-05-01 22:27:40 UTC", description: "Bump data-encoding from 2.8.0 to 2.9.0", pr_number: 22979, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 3, deletions_count: 3}, + {sha: "0cf34ee74b9868a8362e90f8b0d48fb8d7846305", date: "2025-05-01 22:29:22 UTC", description: "Bump openssl-src from 300.4.2+3.4.1 to 300.5.0+3.5.0", pr_number: 22978, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 2, deletions_count: 2}, + {sha: "f84c36c8be42352f74221aa8c9d66309a3047c9b", date: "2025-05-01 23:32:50 UTC", description: "Bump the patches group with 26 updates", pr_number: 22973, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 7, insertions_count: 433, deletions_count: 311}, + {sha: "e5726e1250d2fbbc524a76110999d49517317507", date: "2025-05-02 10:12:44 UTC", description: "remove limit if use apiserver cache", pr_number: 22921, scopes: ["kubernetes_logs source"], type: "fix", breaking_change: false, author: "Xiao", files_count: 3, insertions_count: 18, deletions_count: 2}, + {sha: "b8c24c9bebcf898cfeed257db6ca95e4e0664ba6", date: "2025-05-02 02:58:36 UTC", description: "Bump docker/build-push-action from 6.15.0 to 6.16.0", pr_number: 22984, scopes: ["ci"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 3, deletions_count: 3}, + {sha: "61c418f88a83bad665529cab82462e15034df730", date: "2025-05-05 18:46:17 UTC", description: "introduce log namespace guide", pr_number: 22985, scopes: ["external"], type: "docs", breaking_change: false, author: "Pavlos Rontidis", files_count: 5, insertions_count: 196, deletions_count: 3}, + {sha: "9b824f97516ea92c3a7a894e4bedddb9ad9b59f1", date: "2025-05-06 01:44:29 UTC", description: "support input based on encoding type", pr_number: 22989, scopes: ["redis sink"], type: "feat", breaking_change: false, author: "Yao Noel Achi", files_count: 4, insertions_count: 201, deletions_count: 6}, + {sha: "9a3d2e83a904c36972e2aa7e5457a65454b7bc36", date: "2025-05-07 08:21:27 UTC", description: "fix reduce transfrom examples", pr_number: 22992, scopes: ["external"], type: "docs", breaking_change: false, author: "Pomin Wu", files_count: 1, insertions_count: 6, deletions_count: 7}, + {sha: "9daa00cbec230ba0d89ae85ab3b001b15bc9db97", date: "2025-05-07 20:07:34 UTC", description: "run markdownlint on git files only", pr_number: 23004, scopes: ["dev"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 2, insertions_count: 29, deletions_count: 15}, + {sha: "91245a4dd9ac2be20acb9ce53d2d59905e3513bb", date: "2025-05-10 01:05:49 UTC", description: "add `wildcard_matching` config option", pr_number: 23011, scopes: ["config"], type: "feat", breaking_change: false, author: "simplepad", files_count: 9, insertions_count: 156, deletions_count: 13}, + {sha: "9129d95c56ad530989c238dd0d380de6a0c95ca8", date: "2025-05-13 15:31:55 UTC", description: "Remove slash from max_bytes text", pr_number: 23035, scopes: ["sinks"], type: "docs", breaking_change: false, author: "Jed Laundry", files_count: 47, insertions_count: 47, deletions_count: 47}, + {sha: "a39d60acbfb60a3b740c3da45a3fadfc3e1052a4", date: "2025-05-14 20:42:31 UTC", description: "Initial MQTT Source, #19931", pr_number: 22752, scopes: ["new source"], type: "feat", breaking_change: false, author: "StormStake", files_count: 18, insertions_count: 1296, deletions_count: 103}, + {sha: "689a65d51a2f8034f220afcf5e65e1b924019687", date: "2025-05-14 22:45:15 UTC", description: "e2e dd logs failure", pr_number: 23038, scopes: ["ci"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 8, insertions_count: 50, deletions_count: 31}, + {sha: "822ed0d3f6a752637a33c60b983bd158ad27cce6", date: "2025-05-15 00:48:00 UTC", description: "revert deps bump", pr_number: 23039, scopes: ["azure service"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 5, insertions_count: 60, deletions_count: 114}, + {sha: "346b1ba993f7cdb4e1608fcfb730267c90a85496", date: "2025-05-15 03:46:54 UTC", description: "gelf encoder error message", pr_number: 23021, scopes: ["codecs"], type: "fix", breaking_change: false, author: "Miro Prasil", files_count: 2, insertions_count: 11, deletions_count: 1}, + {sha: "1a1050dbe585bf4a79c37b2183d075ac49bbf254", date: "2025-05-15 00:10:33 UTC", description: "request ARG in every build stage", pr_number: 23049, scopes: ["ci"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 2, insertions_count: 3, deletions_count: 3}, + {sha: "8f10b37a38daaec7bf48f91f00ebea6afa4eb183", date: "2025-05-15 16:37:20 UTC", description: "disable E2E datadog-logs test", pr_number: 23055, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 9, deletions_count: 8}, + {sha: "c1cf664ecbbdf5abe945a0b0c74001829a25fea5", date: "2025-05-15 16:54:10 UTC", description: "temporarily disable failing AMQP tests", pr_number: 23057, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 2, deletions_count: 1}, + {sha: "f2b4e29191e5cbdd6d94111dec1122dc3f9da5d7", date: "2025-05-15 17:13:27 UTC", description: "remove redundant rustup prefix", pr_number: 23059, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "4643acd78cdaf051c3088f7fc46be1488c109563", date: "2025-05-15 17:22:23 UTC", description: "provide default RUST_VERSION if one is not specified", pr_number: 23060, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 3, insertions_count: 3, deletions_count: 3}, + {sha: "84e5fe9bec1221c58e663602a2aef10e90c146d5", date: "2025-05-15 17:57:56 UTC", description: "bump nextest to latest version", pr_number: 23058, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 4, insertions_count: 5, deletions_count: 5}, + {sha: "2c63454e394cdd18451bd3432b87f1b65d516ede", date: "2025-05-16 00:07:01 UTC", description: "Handling wrong timestamps in dnstap-parser", pr_number: 23048, scopes: ["parsing"], type: "fix", breaking_change: false, author: "Burkov Egor", files_count: 2, insertions_count: 52, deletions_count: 5}, + {sha: "ace0dd21b542cfd522a18f3f3943a643a91e0e79", date: "2025-05-16 00:42:26 UTC", description: "Handle underflow in A6 parsing", pr_number: 23047, scopes: ["parsing"], type: "fix", breaking_change: false, author: "Burkov Egor", files_count: 2, insertions_count: 20, deletions_count: 0}, + {sha: "9cdab60b320ad14c06214b4cf7b7e0f8a441f373", date: "2025-05-16 01:26:16 UTC", description: "make `--watch-config` watch external VRL files in `remap` transforms", pr_number: 23010, scopes: ["remap transform"], type: "feat", breaking_change: false, author: "nekorro", files_count: 5, insertions_count: 35, deletions_count: 2}, + {sha: "914d1568dd1363d2d95b9e8dc4b2b1a808885d12", date: "2025-05-16 20:59:37 UTC", description: "install mold (no need to build it)", pr_number: 23064, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 3, deletions_count: 11}, + {sha: "1d232fb5057786b704b5ac953aa0fd7a74493c26", date: "2025-05-16 18:53:31 UTC", description: "add from and to date search to enrichment tables", pr_number: 22926, scopes: ["enriching"], type: "enhancement", breaking_change: false, author: "Nick Wang", files_count: 5, insertions_count: 177, deletions_count: 3}, + {sha: "c9791e3db80659a5f763e51b4ad0824e366cbf1f", date: "2025-05-17 01:15:42 UTC", description: "path resolution for docker compose files", pr_number: 23066, scopes: ["ci"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 5, insertions_count: 30, deletions_count: 33}, + ] +} diff --git a/website/cue/reference/versions.cue b/website/cue/reference/versions.cue index ff39842f506e1..fcd606b214c63 100644 --- a/website/cue/reference/versions.cue +++ b/website/cue/reference/versions.cue @@ -2,6 +2,7 @@ package metadata // This has to be maintained manually because there's currently no way to sort versions programmatically versions: [string, ...string] & [ + "0.47.0", "0.46.1", "0.46.0", "0.45.0", diff --git a/website/layouts/releases/single.html b/website/layouts/releases/single.html index 9c3a962ed773e..fe4cb606aacdf 100644 --- a/website/layouts/releases/single.html +++ b/website/layouts/releases/single.html @@ -126,12 +126,13 @@ {{ if gt (len .contributors) 0 }}
- Thanks to - {{ range .contributors}} - {{ . }} + Thanks to + {{ $len := len .contributors }} + {{ range $i, $contributor := .contributors }} + {{ $contributor }}{{ if lt (add $i 1) $len }}, {{ end }} {{ end }} - for contributing this change! - + for contributing this change! + {{ end }} {{ end }} From bb6a91404cdc3f662b037cc4d5f52237b28d215d Mon Sep 17 00:00:00 2001 From: Bruce Guenter Date: Wed, 21 May 2025 17:31:12 -0400 Subject: [PATCH 025/305] chore(sources): Refactor `struct SourceSender` a bit (#23089) * Clean up `inner` handling, as it should never be `None` * Rename some fields for clarity * Handle the unsent event count decrementing in one place --- src/source_sender/mod.rs | 130 +++++++++++++++++---------------------- 1 file changed, 58 insertions(+), 72 deletions(-) diff --git a/src/source_sender/mod.rs b/src/source_sender/mod.rs index b79dba5684862..bf0ee0525597f 100644 --- a/src/source_sender/mod.rs +++ b/src/source_sender/mod.rs @@ -94,22 +94,27 @@ impl From for EventArray { pub struct Builder { buf_size: usize, - inner: Option, - named_inners: HashMap, + default_output: Option, + named_outputs: HashMap, lag_time: Option, } -impl Builder { - // https://github.com/rust-lang/rust/issues/73255 - #[allow(clippy::missing_const_for_fn)] - pub fn with_buffer(self, n: usize) -> Self { +impl Default for Builder { + fn default() -> Self { Self { - buf_size: n, - inner: self.inner, - named_inners: self.named_inners, - lag_time: self.lag_time, + buf_size: CHUNK_SIZE, + default_output: None, + named_outputs: Default::default(), + lag_time: Some(histogram!(LAG_TIME_NAME)), } } +} + +impl Builder { + pub const fn with_buffer(mut self, n: usize) -> Self { + self.buf_size = n; + self + } pub fn add_source_output( &mut self, @@ -124,54 +129,47 @@ impl Builder { }; match output.port { None => { - let (inner, rx) = Inner::new_with_buffer( + let (output, rx) = Output::new_with_buffer( self.buf_size, DEFAULT_OUTPUT.to_owned(), lag_time, log_definition, output_id, ); - self.inner = Some(inner); + self.default_output = Some(output); rx } Some(name) => { - let (inner, rx) = Inner::new_with_buffer( + let (output, rx) = Output::new_with_buffer( self.buf_size, name.clone(), lag_time, log_definition, output_id, ); - self.named_inners.insert(name, inner); + self.named_outputs.insert(name, output); rx } } } - // https://github.com/rust-lang/rust/issues/73255 - #[allow(clippy::missing_const_for_fn)] pub fn build(self) -> SourceSender { SourceSender { - inner: self.inner, - named_inners: self.named_inners, + default_output: self.default_output.expect("no default output"), + named_outputs: self.named_outputs, } } } #[derive(Debug, Clone)] pub struct SourceSender { - inner: Option, - named_inners: HashMap, + default_output: Output, + named_outputs: HashMap, } impl SourceSender { pub fn builder() -> Builder { - Builder { - buf_size: CHUNK_SIZE, - inner: None, - named_inners: Default::default(), - lag_time: Some(histogram!(LAG_TIME_NAME)), - } + Builder::default() } #[cfg(any(test, feature = "test-utils"))] @@ -181,12 +179,12 @@ impl SourceSender { component: "test".to_string().into(), port: None, }; - let (inner, rx) = - Inner::new_with_buffer(n, DEFAULT_OUTPUT.to_owned(), lag_time, None, output_id); + let (default_output, rx) = + Output::new_with_buffer(n, DEFAULT_OUTPUT.to_owned(), lag_time, None, output_id); ( Self { - inner: Some(inner), - named_inners: Default::default(), + default_output, + named_outputs: Default::default(), }, rx, ) @@ -254,7 +252,7 @@ impl SourceSender { component: "test".to_string().into(), port: Some(name.clone()), }; - let (inner, recv) = Inner::new_with_buffer(100, name.clone(), None, None, output_id); + let (output, recv) = Output::new_with_buffer(100, name.clone(), None, None, output_id); let recv = recv.into_stream().map(move |mut item| { item.events.iter_events_mut().for_each(|mut event| { let metadata = event.metadata_mut(); @@ -263,7 +261,7 @@ impl SourceSender { }); item }); - self.named_inners.insert(name, inner); + self.named_outputs.insert(name, output); recv } @@ -271,11 +269,7 @@ impl SourceSender { /// /// This internally handles emitting [EventsSent] and [ComponentEventsDropped] events. pub async fn send_event(&mut self, event: impl Into) -> Result<(), ClosedError> { - self.inner - .as_mut() - .expect("no default output") - .send_event(event) - .await + self.default_output.send_event(event).await } /// Send a stream of events to the default output. @@ -286,11 +280,7 @@ impl SourceSender { S: Stream + Unpin, E: Into + ByteSizeOf, { - self.inner - .as_mut() - .expect("no default output") - .send_event_stream(events) - .await + self.default_output.send_event_stream(events).await } /// Send a batch of events to the default output. @@ -302,11 +292,7 @@ impl SourceSender { I: IntoIterator, ::IntoIter: ExactSizeIterator, { - self.inner - .as_mut() - .expect("no default output") - .send_batch(events) - .await + self.default_output.send_batch(events).await } /// Send a batch of events event to a named output. @@ -318,7 +304,7 @@ impl SourceSender { I: IntoIterator, ::IntoIter: ExactSizeIterator, { - self.named_inners + self.named_outputs .get_mut(name) .expect("unknown output") .send_batch(events) @@ -368,9 +354,8 @@ impl Drop for UnsentEventCount { } #[derive(Clone)] -struct Inner { - inner: LimitedSender, - output: String, +struct Output { + sender: LimitedSender, lag_time: Option, events_sent: Registered, /// The schema definition that will be attached to Log events sent through here @@ -380,17 +365,17 @@ struct Inner { output_id: Arc, } -impl fmt::Debug for Inner { +impl fmt::Debug for Output { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt.debug_struct("Inner") - .field("inner", &self.inner) - .field("output", &self.output) + fmt.debug_struct("Output") + .field("sender", &self.sender) + .field("output_id", &self.output_id) // `metrics::Histogram` is missing `impl Debug` .finish() } } -impl Inner { +impl Output { fn new_with_buffer( n: usize, output: String, @@ -401,8 +386,7 @@ impl Inner { let (tx, rx) = channel::limited(n); ( Self { - inner: tx, - output: output.clone(), + sender: tx, lag_time, events_sent: register!(EventsSent::from(internal_event::Output(Some( output.into() @@ -414,7 +398,11 @@ impl Inner { ) } - async fn send(&mut self, mut events: EventArray) -> Result<(), ClosedError> { + async fn send( + &mut self, + mut events: EventArray, + unsent_event_count: &mut UnsentEventCount, + ) -> Result<(), ClosedError> { let send_reference = Instant::now(); let reference = Utc::now().timestamp_millis(); events @@ -433,7 +421,7 @@ impl Inner { let byte_size = events.estimated_json_encoded_size_of(); let count = events.len(); - self.inner + self.sender .send(SourceSenderItem { events, send_reference, @@ -441,6 +429,7 @@ impl Inner { .await .map_err(|_| ClosedError)?; self.events_sent.emit(CountByteSize(count, byte_size)); + unsent_event_count.decr(count); Ok(()) } @@ -449,11 +438,8 @@ impl Inner { // It's possible that the caller stops polling this future while it is blocked waiting // on `self.send()`. When that happens, we use `UnsentEventCount` to correctly emit // `ComponentEventsDropped` events. - let count = event.len(); - let mut unsent_event_count = UnsentEventCount::new(count); - let res = self.send(event).await; - unsent_event_count.discard(); - res + let mut unsent_event_count = UnsentEventCount::new(event.len()); + self.send(event, &mut unsent_event_count).await } async fn send_event_stream(&mut self, events: S) -> Result<(), ClosedError> @@ -480,13 +466,13 @@ impl Inner { let events = events.into_iter().map(Into::into); let mut unsent_event_count = UnsentEventCount::new(events.len()); for events in array::events_into_arrays(events, Some(CHUNK_SIZE)) { - let count = events.len(); - self.send(events).await.inspect_err(|_| { - // The unsent event count is discarded here because the caller emits the - // `StreamClosedError`. - unsent_event_count.discard(); - })?; - unsent_event_count.decr(count); + self.send(events, &mut unsent_event_count) + .await + .inspect_err(|_| { + // The unsent event count is discarded here because the callee emits the + // `StreamClosedError`. + unsent_event_count.discard(); + })?; } Ok(()) } From 3528fc19162a6c722209423ea1db0e2e754bfaf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20AGBEKODO?= <70853334+20agbekodo@users.noreply.github.com> Date: Wed, 28 May 2025 12:43:47 -0400 Subject: [PATCH 026/305] chore(transforms): Move `TransformOutputsBuf` functions out of test flag (#23116) * chore(remap transform) Move take_primary() outside of test flag * chore(remap transform): Remove other take/drain functions from the test FF * fix: also remove called function from test FF --- lib/vector-core/src/transform/mod.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/vector-core/src/transform/mod.rs b/lib/vector-core/src/transform/mod.rs index 37d694e6c1b9b..87fac605f6b12 100644 --- a/lib/vector-core/src/transform/mod.rs +++ b/lib/vector-core/src/transform/mod.rs @@ -376,7 +376,6 @@ impl TransformOutputsBuf { /// # Panics /// /// Panics if there is no default output. - #[cfg(any(feature = "test", test))] pub fn drain(&mut self) -> impl Iterator + '_ { self.primary_buffer .as_mut() @@ -389,7 +388,6 @@ impl TransformOutputsBuf { /// # Panics /// /// Panics if there is no output with the given name. - #[cfg(any(feature = "test", test))] pub fn drain_named(&mut self, name: &str) -> impl Iterator + '_ { self.named_buffers .get_mut(name) @@ -402,12 +400,10 @@ impl TransformOutputsBuf { /// # Panics /// /// Panics if there is no default output. - #[cfg(any(feature = "test", test))] pub fn take_primary(&mut self) -> OutputBuffer { std::mem::take(self.primary_buffer.as_mut().expect("no default output")) } - #[cfg(any(feature = "test", test))] pub fn take_all_named(&mut self) -> HashMap { std::mem::take(&mut self.named_buffers) } @@ -482,7 +478,6 @@ impl OutputBuffer { }) } - #[cfg(any(feature = "test", test))] pub fn drain(&mut self) -> impl Iterator + '_ { self.0.drain(..).flat_map(EventArray::into_events) } From 43550f31b039f39707c7f2060016a197d9e19077 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Wed, 4 Jun 2025 11:39:58 -0700 Subject: [PATCH 027/305] chore(ci): Add axiom sink to list of scopes --- .github/workflows/semantic.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/semantic.yml b/.github/workflows/semantic.yml index 47a53618e4eef..f59a9403c4d86 100644 --- a/.github/workflows/semantic.yml +++ b/.github/workflows/semantic.yml @@ -192,6 +192,7 @@ jobs: aws_kinesis_streams sink aws_s3 sink aws_sqs sink + axiom sink azure_blob sink azure_monitor_logs sink blackhole sink From 5ac8263dc400033552cc86a1885a6f8a2d3cfd0a Mon Sep 17 00:00:00 2001 From: Darach Ennis Date: Wed, 4 Jun 2025 21:20:48 +0200 Subject: [PATCH 028/305] feat(axiom sink): Add trace data support to Axiom Configuration for the axiom sink (#22935) * Add trace data support to Axiom Configuration for the axiom sink Signed-off-by: Darach Ennis * Add changelog entry for axiom sink trace support feature addition Signed-off-by: Darach Ennis --------- Signed-off-by: Darach Ennis Signed-off-by: Jesse Szwedko Co-authored-by: Jesse Szwedko --- changelog.d/22935_axiom_sink_trace_data_support.feature.md | 5 +++++ src/sinks/axiom.rs | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changelog.d/22935_axiom_sink_trace_data_support.feature.md diff --git a/changelog.d/22935_axiom_sink_trace_data_support.feature.md b/changelog.d/22935_axiom_sink_trace_data_support.feature.md new file mode 100644 index 0000000000000..08d117b51f672 --- /dev/null +++ b/changelog.d/22935_axiom_sink_trace_data_support.feature.md @@ -0,0 +1,5 @@ +Adds trace data type support to the Axiom sink allowing propagation of +OpenTelemetry traces received via the `opentelemetry` or compatible trace +emitting components. + +authors: darach diff --git a/src/sinks/axiom.rs b/src/sinks/axiom.rs index dfc4ab124dcc1..5d4cf96076ab3 100644 --- a/src/sinks/axiom.rs +++ b/src/sinks/axiom.rs @@ -139,7 +139,7 @@ impl SinkConfig for AxiomConfig { } fn input(&self) -> Input { - Input::new(DataType::Metric | DataType::Log) + Input::new(DataType::Metric | DataType::Log | DataType::Trace) } fn acknowledgements(&self) -> &AcknowledgementsConfig { From c6cc716d12cd3e16fa3504ae8353ee294f53c0b7 Mon Sep 17 00:00:00 2001 From: Scott Opell Date: Thu, 5 Jun 2025 18:53:40 -0400 Subject: [PATCH 029/305] chore(ci): Updates SMP cli client version (#23125) updates SMP cli client version --- .github/workflows/regression.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index af2426ce30429..712d9a200c939 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -105,7 +105,7 @@ jobs: - name: Set SMP version id: experimental-meta run: | - export SMP_CRATE_VERSION="0.21.0" + export SMP_CRATE_VERSION="0.22.0" echo "smp crate version: ${SMP_CRATE_VERSION}" echo "SMP_CRATE_VERSION=${SMP_CRATE_VERSION}" >> $GITHUB_OUTPUT From 2f8f1c3474f7f5a0717ab8f1879750844e07ff05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jun 2025 09:54:22 -0400 Subject: [PATCH 030/305] chore(ci): Bump check-spelling/check-spelling from 0.0.24 to 0.0.25 (#23143) Bumps [check-spelling/check-spelling](https://github.com/check-spelling/check-spelling) from 0.0.24 to 0.0.25. - [Release notes](https://github.com/check-spelling/check-spelling/releases) - [Commits](https://github.com/check-spelling/check-spelling/compare/v0.0.24...v0.0.25) --- updated-dependencies: - dependency-name: check-spelling/check-spelling dependency-version: 0.0.25 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/spelling.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml index 5976f1a9fedb4..e7649b1e9f4cb 100644 --- a/.github/workflows/spelling.yml +++ b/.github/workflows/spelling.yml @@ -87,7 +87,7 @@ jobs: steps: - name: check-spelling id: spelling - uses: check-spelling/check-spelling@v0.0.24 + uses: check-spelling/check-spelling@v0.0.25 with: suppress_push_for_open_pull_request: ${{ github.actor != 'dependabot[bot]' && 1 }} checkout: true From 03e6215ee4bdb37790e2356ce89990d461079ada Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jun 2025 09:54:49 -0400 Subject: [PATCH 031/305] chore(ci): Bump aws-actions/configure-aws-credentials from 4.1.0 to 4.2.1 (#23142) chore(ci): Bump aws-actions/configure-aws-credentials Bumps [aws-actions/configure-aws-credentials](https://github.com/aws-actions/configure-aws-credentials) from 4.1.0 to 4.2.1. - [Release notes](https://github.com/aws-actions/configure-aws-credentials/releases) - [Changelog](https://github.com/aws-actions/configure-aws-credentials/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws-actions/configure-aws-credentials/compare/v4.1.0...v4.2.1) --- updated-dependencies: - dependency-name: aws-actions/configure-aws-credentials dependency-version: 4.2.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/regression.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 712d9a200c939..b0aa8378df9c4 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -269,7 +269,7 @@ jobs: - resolve-inputs steps: - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.1.0 + uses: aws-actions/configure-aws-credentials@v4.2.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -303,7 +303,7 @@ jobs: docker load --input baseline-image.tar - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.1.0 + uses: aws-actions/configure-aws-credentials@v4.2.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -343,7 +343,7 @@ jobs: docker load --input comparison-image.tar - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.1.0 + uses: aws-actions/configure-aws-credentials@v4.2.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -378,7 +378,7 @@ jobs: ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.1.0 + uses: aws-actions/configure-aws-credentials@v4.2.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -451,7 +451,7 @@ jobs: - uses: actions/checkout@v4 - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.1.0 + uses: aws-actions/configure-aws-credentials@v4.2.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -490,7 +490,7 @@ jobs: ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.1.0 + uses: aws-actions/configure-aws-credentials@v4.2.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} From a34ab883c1682444c55e5f15757e20c024f763db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jun 2025 09:55:33 -0400 Subject: [PATCH 032/305] chore(ci): Bump ossf/scorecard-action from 2.4.1 to 2.4.2 (#23141) Bumps [ossf/scorecard-action](https://github.com/ossf/scorecard-action) from 2.4.1 to 2.4.2. - [Release notes](https://github.com/ossf/scorecard-action/releases) - [Changelog](https://github.com/ossf/scorecard-action/blob/main/RELEASE.md) - [Commits](https://github.com/ossf/scorecard-action/compare/v2.4.1...v2.4.2) --- updated-dependencies: - dependency-name: ossf/scorecard-action dependency-version: 2.4.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index ba19d832ca3fb..294b5ecaedd3f 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -37,7 +37,7 @@ jobs: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@v2.4.1 + uses: ossf/scorecard-action@v2.4.2 with: results_file: results.sarif results_format: sarif From 59adb1aa767d471d2de3c3ac705e8dfe35e3c5d9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jun 2025 13:55:58 +0000 Subject: [PATCH 033/305] chore(ci): Bump docker/build-push-action from 6.16.0 to 6.18.0 (#23140) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.16.0 to 6.18.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v6.16.0...v6.18.0) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-version: 6.18.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/environment.yml | 2 +- .github/workflows/regression.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/environment.yml b/.github/workflows/environment.yml index 500e7db8c8425..edbbce826dd50 100644 --- a/.github/workflows/environment.yml +++ b/.github/workflows/environment.yml @@ -62,7 +62,7 @@ jobs: org.opencontainers.image.title=Vector development environment org.opencontainers.image.url=https://github.com/vectordotdev/vector - name: Build and push - uses: docker/build-push-action@v6.16.0 + uses: docker/build-push-action@v6.18.0 with: context: . file: ./scripts/environment/Dockerfile diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index b0aa8378df9c4..88b3a4637043f 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -204,7 +204,7 @@ jobs: uses: docker/setup-buildx-action@v3.10.0 - name: Build 'vector' target image - uses: docker/build-push-action@v6.16.0 + uses: docker/build-push-action@v6.18.0 with: context: baseline-vector/ cache-from: type=gha @@ -243,7 +243,7 @@ jobs: uses: docker/setup-buildx-action@v3.10.0 - name: Build 'vector' target image - uses: docker/build-push-action@v6.16.0 + uses: docker/build-push-action@v6.18.0 with: context: comparison-vector/ cache-from: type=gha From a04b4d3bd094902a2aa92feed981b21687e3fd68 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jun 2025 14:01:18 +0000 Subject: [PATCH 034/305] chore(deps): Bump tokio from 1.44.2 to 1.45.1 (#23136) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.44.2 to 1.45.1. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.44.2...tokio-1.45.1) --- updated-dependencies: - dependency-name: tokio dependency-version: 1.45.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 6 +++--- lib/file-source/Cargo.toml | 2 +- lib/k8s-e2e-tests/Cargo.toml | 2 +- lib/k8s-test-framework/Cargo.toml | 2 +- lib/vector-api-client/Cargo.toml | 2 +- lib/vector-buffers/Cargo.toml | 2 +- lib/vector-common/Cargo.toml | 4 ++-- lib/vector-core/Cargo.toml | 2 +- lib/vector-stream/Cargo.toml | 2 +- lib/vector-tap/Cargo.toml | 4 ++-- 11 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c465db7ff350..46a6bb16b8ca6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10412,9 +10412,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.44.2" +version = "1.45.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" +checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" dependencies = [ "backtrace", "bytes 1.10.1", diff --git a/Cargo.toml b/Cargo.toml index 4bf10447c73ac..faccd2a7dd0d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -165,7 +165,7 @@ serde_json = { version = "1.0.140", default-features = false, features = ["raw_v serde = { version = "1.0.219", default-features = false, features = ["alloc", "derive", "rc"] } snafu = { version = "0.7.5", default-features = false, features = ["futures", "std"] } tempfile = "3.19.1" -tokio = { version = "1.44.2", default-features = false, features = ["full"] } +tokio = { version = "1.45.1", default-features = false, features = ["full"] } toml = { version = "0.8.22", default-features = false, features = ["display", "parse"] } tonic = { version = "0.11", default-features = false, features = ["transport", "codegen", "prost", "tls", "tls-roots", "gzip"] } tonic-build = { version = "0.11", default-features = false, features = ["transport", "prost"] } @@ -207,7 +207,7 @@ loki-logproto = { path = "lib/loki-logproto", optional = true } async-stream = { version = "0.3.6", default-features = false } async-trait = { version = "0.1.88", default-features = false } futures.workspace = true -tokio = { version = "1.44.2", default-features = false, features = ["full"] } +tokio = { version = "1.45.1", default-features = false, features = ["full"] } tokio-openssl = { version = "0.6.5", default-features = false } tokio-stream = { version = "0.1.17", default-features = false, features = ["net", "sync", "time"] } tokio-util = { version = "0.7", default-features = false, features = ["io", "time"] } @@ -443,7 +443,7 @@ reqwest = { version = "0.11", features = ["json"] } rstest = { version = "0.25.0" } tempfile.workspace = true test-generator = "0.3.1" -tokio = { version = "1.44.2", features = ["test-util"] } +tokio = { version = "1.45.1", features = ["test-util"] } tokio-test = "0.4.4" tower-test = "0.4.0" vector-lib = { workspace = true, features = ["test"] } diff --git a/lib/file-source/Cargo.toml b/lib/file-source/Cargo.toml index f9c5c178d5651..e5de5004697c4 100644 --- a/lib/file-source/Cargo.toml +++ b/lib/file-source/Cargo.toml @@ -68,7 +68,7 @@ default-features = false features = [] [dependencies.tokio] -version = "1.44.2" +version = "1.45.1" default-features = false features = ["full"] diff --git a/lib/k8s-e2e-tests/Cargo.toml b/lib/k8s-e2e-tests/Cargo.toml index 954675e9b3b76..fc5339e054e79 100644 --- a/lib/k8s-e2e-tests/Cargo.toml +++ b/lib/k8s-e2e-tests/Cargo.toml @@ -14,7 +14,7 @@ k8s-test-framework = { version = "0.1", path = "../k8s-test-framework" } regex = "1" reqwest = { version = "0.11.26", features = ["json"] } serde_json.workspace = true -tokio = { version = "1.44.2", features = ["full"] } +tokio = { version = "1.45.1", features = ["full"] } indoc.workspace = true env_logger = "0.11" tracing = { version = "0.1", features = ["log"] } diff --git a/lib/k8s-test-framework/Cargo.toml b/lib/k8s-test-framework/Cargo.toml index fa7b47e3bf5f5..c5a4a213500a3 100644 --- a/lib/k8s-test-framework/Cargo.toml +++ b/lib/k8s-test-framework/Cargo.toml @@ -11,5 +11,5 @@ license = "MPL-2.0" k8s-openapi = { version = "0.16.0", default-features = false, features = ["v1_19"] } serde_json.workspace = true tempfile.workspace = true -tokio = { version = "1.44.2", features = ["full"] } +tokio = { version = "1.45.1", features = ["full"] } log = "0.4" diff --git a/lib/vector-api-client/Cargo.toml b/lib/vector-api-client/Cargo.toml index be614b8ccbb7f..30a7c9d80f990 100644 --- a/lib/vector-api-client/Cargo.toml +++ b/lib/vector-api-client/Cargo.toml @@ -17,7 +17,7 @@ anyhow = { version = "1.0.98", default-features = false, features = ["std"] } # Tokio / Futures futures.workspace = true -tokio = { version = "1.44.2", default-features = false, features = ["macros", "rt", "sync"] } +tokio = { version = "1.45.1", default-features = false, features = ["macros", "rt", "sync"] } tokio-stream = { version = "0.1.17", default-features = false, features = ["sync"] } # GraphQL diff --git a/lib/vector-buffers/Cargo.toml b/lib/vector-buffers/Cargo.toml index 24e6c90ff0862..6d2f40cf91b3a 100644 --- a/lib/vector-buffers/Cargo.toml +++ b/lib/vector-buffers/Cargo.toml @@ -28,7 +28,7 @@ rkyv = { version = "0.7.45", default-features = false, features = ["size_32", "s serde.workspace = true snafu.workspace = true tokio-util = { version = "0.7.0", default-features = false } -tokio = { version = "1.44.2", default-features = false, features = ["rt", "macros", "rt-multi-thread", "sync", "fs", "io-util", "time"] } +tokio = { version = "1.45.1", default-features = false, features = ["rt", "macros", "rt-multi-thread", "sync", "fs", "io-util", "time"] } tracing = { version = "0.1.34", default-features = false, features = ["attributes"] } vector-config = { path = "../vector-config", default-features = false } vector-common = { path = "../vector-common", default-features = false, features = ["byte_size_of"] } diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index 17ad79cdc73f1..07d09e7d45a48 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -48,11 +48,11 @@ serde.workspace = true serde_json.workspace = true smallvec = { version = "1", default-features = false } stream-cancel = { version = "0.8.2", default-features = false } -tokio = { version = "1.44.2", default-features = false, features = ["macros", "time"] } +tokio = { version = "1.45.1", default-features = false, features = ["macros", "time"] } tracing = { version = "0.1.34", default-features = false } vrl.workspace = true vector-config = { path = "../vector-config" } [dev-dependencies] futures = { version = "0.3.31", default-features = false, features = ["async-await"] } -tokio = { version = "1.44.2", default-features = false, features = ["rt", "time"] } +tokio = { version = "1.45.1", default-features = false, features = ["rt", "time"] } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index af3c2839e8dfe..2c805ae62ae3d 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -49,7 +49,7 @@ serde_with = { version = "3.12.0", default-features = false, features = ["std", smallvec = { version = "1", default-features = false, features = ["serde", "const_generics"] } snafu.workspace = true socket2 = { version = "0.5.9", default-features = false } -tokio = { version = "1.44.2", default-features = false, features = ["net"] } +tokio = { version = "1.45.1", default-features = false, features = ["net"] } tokio-openssl = { version = "0.6.5", default-features = false } tokio-stream = { version = "0.1", default-features = false, features = ["time"], optional = true } tokio-util = { version = "0.7.0", default-features = false, features = ["time"] } diff --git a/lib/vector-stream/Cargo.toml b/lib/vector-stream/Cargo.toml index 63493bea41cd3..87908f771ad13 100644 --- a/lib/vector-stream/Cargo.toml +++ b/lib/vector-stream/Cargo.toml @@ -10,7 +10,7 @@ async-stream = { version = "0.3.6", default-features = false } futures.workspace = true futures-util = { version = "0.3.29", default-features = false, features = ["std"] } pin-project.workspace = true -tokio = { version = "1.44.2", default-features = false, features = ["net"] } +tokio = { version = "1.45.1", default-features = false, features = ["net"] } tokio-util = { version = "0.7.0", default-features = false, features = ["time"] } tower = { version = "0.4", default-features = false, features = ["util"] } tracing = { version = "0.1.34", default-features = false } diff --git a/lib/vector-tap/Cargo.toml b/lib/vector-tap/Cargo.toml index 83a89dd9c8fcf..154e7a3f08e33 100644 --- a/lib/vector-tap/Cargo.toml +++ b/lib/vector-tap/Cargo.toml @@ -15,7 +15,7 @@ colored = { version = "3.0.0", default-features = false } futures.workspace = true glob.workspace = true serde_yaml = { version = "0.9.34", default-features = false } -tokio = { version = "1.44.2", default-features = false, features = ["time"] } +tokio = { version = "1.45.1", default-features = false, features = ["time"] } tokio-stream = { version = "0.1.17", default-features = false, features = ["sync"] } tokio-tungstenite = { version = "0.20.1", default-features = false } tracing = { version = "0.1.34", default-features = false } @@ -31,4 +31,4 @@ futures-util = "0.3.30" chrono = { workspace = true } portpicker = { path = "../portpicker" } serde_json = { workspace = true } -tokio = { version = "1.44.2", default-features = false, features = ["test-util"] } +tokio = { version = "1.45.1", default-features = false, features = ["test-util"] } From e5f533e86c6914369b3b34c9e68a0f23ec85e7d9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jun 2025 14:01:39 +0000 Subject: [PATCH 035/305] chore(deps): Bump quickcheck_macros from 1.0.0 to 1.1.0 (#23135) Bumps [quickcheck_macros](https://github.com/BurntSushi/quickcheck) from 1.0.0 to 1.1.0. - [Commits](https://github.com/BurntSushi/quickcheck/compare/quickcheck_macros-1.0.0...quickcheck_macros-1.1.0) --- updated-dependencies: - dependency-name: quickcheck_macros dependency-version: 1.1.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 46a6bb16b8ca6..a414558938c32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8034,13 +8034,13 @@ dependencies = [ [[package]] name = "quickcheck_macros" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b22a693222d716a9587786f37ac3f6b4faedb5b80c23914e7303ff5a1d8016e9" +checksum = "f71ee38b42f8459a88d3362be6f9b841ad2d5421844f61eb1c59c11bff3ac14a" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 1.0.109", + "syn 2.0.101", ] [[package]] From 563a9f048195f10e81044edca867d7788e755bd8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jun 2025 14:01:59 +0000 Subject: [PATCH 036/305] chore(deps): Bump uuid from 1.16.0 to 1.17.0 (#23134) Bumps [uuid](https://github.com/uuid-rs/uuid) from 1.16.0 to 1.17.0. - [Release notes](https://github.com/uuid-rs/uuid/releases) - [Commits](https://github.com/uuid-rs/uuid/compare/v1.16.0...v1.17.0) --- updated-dependencies: - dependency-name: uuid dependency-version: 1.17.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 9 +++++---- Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a414558938c32..7f9961ae40d19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5393,10 +5393,11 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.65" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -11377,9 +11378,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" +checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" dependencies = [ "getrandom 0.3.1", "js-sys", diff --git a/Cargo.toml b/Cargo.toml index faccd2a7dd0d4..b031dbfac8336 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -169,7 +169,7 @@ tokio = { version = "1.45.1", default-features = false, features = ["full"] } toml = { version = "0.8.22", default-features = false, features = ["display", "parse"] } tonic = { version = "0.11", default-features = false, features = ["transport", "codegen", "prost", "tls", "tls-roots", "gzip"] } tonic-build = { version = "0.11", default-features = false, features = ["transport", "prost"] } -uuid = { version = "1.16.0", features = ["v4", "v7", "serde"] } +uuid = { version = "1.17.0", features = ["v4", "v7", "serde"] } vector-lib = { path = "lib/vector-lib", default-features = false, features = ["vrl"] } vector-config = { path = "lib/vector-config" } vector-config-common = { path = "lib/vector-config-common" } From ddb412160664cee505619e75f69a3d56fd9f337b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jun 2025 14:03:55 +0000 Subject: [PATCH 037/305] chore(deps): Bump crc from 3.2.1 to 3.3.0 (#23132) Bumps [crc](https://github.com/mrhooray/crc-rs) from 3.2.1 to 3.3.0. - [Release notes](https://github.com/mrhooray/crc-rs/releases) - [Commits](https://github.com/mrhooray/crc-rs/commits) --- updated-dependencies: - dependency-name: crc dependency-version: 3.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- lib/file-source/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7f9961ae40d19..3ab8b9c34cc70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2526,9 +2526,9 @@ dependencies = [ [[package]] name = "crc" -version = "3.2.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" dependencies = [ "crc-catalog", ] diff --git a/lib/file-source/Cargo.toml b/lib/file-source/Cargo.toml index e5de5004697c4..b61e0b036d3a4 100644 --- a/lib/file-source/Cargo.toml +++ b/lib/file-source/Cargo.toml @@ -11,7 +11,7 @@ libc = "0.2" winapi = { version = "0.3", features = ["winioctl"] } [dependencies] -crc = "3.2.1" +crc = "3.3.0" glob.workspace = true scan_fmt = "0.2.6" vector-common = { path = "../vector-common", default-features = false } From fcc546a4ad7229f949852044900e6ff997016905 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jun 2025 14:04:41 +0000 Subject: [PATCH 038/305] chore(deps): Bump confy from 0.6.1 to 1.0.0 (#23130) Bumps [confy](https://github.com/rust-cli/confy) from 0.6.1 to 1.0.0. - [Release notes](https://github.com/rust-cli/confy/releases) - [Commits](https://github.com/rust-cli/confy/compare/v0.6.1...v1.0.0) --- updated-dependencies: - dependency-name: confy dependency-version: 1.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 33 ++++++--------------------------- vdev/Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3ab8b9c34cc70..ef618c62c8f27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2395,13 +2395,13 @@ dependencies = [ [[package]] name = "confy" -version = "0.6.1" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45b1f4c00870f07dc34adcac82bb6a72cc5aabca8536ba1797e01df51d2ce9a0" +checksum = "f29222b549d4e3ded127989d523da9e928918d0d0d7f7c1690b439d0d538bae9" dependencies = [ - "directories 5.0.1", + "directories", "serde", - "thiserror 1.0.68", + "thiserror 2.0.3", "toml", ] @@ -3087,22 +3087,13 @@ dependencies = [ "subtle", ] -[[package]] -name = "directories" -version = "5.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" -dependencies = [ - "dirs-sys 0.4.1", -] - [[package]] name = "directories" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d" dependencies = [ - "dirs-sys 0.5.0", + "dirs-sys", ] [[package]] @@ -3115,18 +3106,6 @@ dependencies = [ "dirs-sys-next", ] -[[package]] -name = "dirs-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" -dependencies = [ - "libc", - "option-ext", - "redox_users 0.4.3", - "windows-sys 0.48.0", -] - [[package]] name = "dirs-sys" version = "0.5.0" @@ -11411,7 +11390,7 @@ dependencies = [ "clap-verbosity-flag", "clap_complete", "confy", - "directories 6.0.0", + "directories", "dunce", "glob", "hex", diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index a0e2b3bd863b8..b35fe0fefce54 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -13,7 +13,7 @@ chrono.workspace = true clap.workspace = true clap-verbosity-flag = "3.0.2" clap_complete = "4.5.48" -confy = "0.6.1" +confy = "1.0.0" directories = "6.0.0" # remove this when stabilized https://doc.rust-lang.org/stable/std/path/fn.absolute.html dunce = "1.0.5" From a03f45c511ef3893317153b07ba0fc2f51cf12ef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jun 2025 14:23:41 +0000 Subject: [PATCH 039/305] chore(deps): Bump windows-service from 0.7.0 to 0.8.0 (#22557) Bumps [windows-service](https://github.com/mullvad/windows-service-rs) from 0.7.0 to 0.8.0. - [Release notes](https://github.com/mullvad/windows-service-rs/releases) - [Changelog](https://github.com/mullvad/windows-service-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/mullvad/windows-service-rs/compare/v0.7.0...v0.8.0) --- updated-dependencies: - dependency-name: windows-service dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Pavlos Rontidis --- Cargo.lock | 6 +++--- Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ef618c62c8f27..eb0faad3c3dff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12495,13 +12495,13 @@ dependencies = [ [[package]] name = "windows-service" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d24d6bcc7f734a4091ecf8d7a64c5f7d7066f45585c1861eba06449909609c8a" +checksum = "193cae8e647981c35bc947fdd57ba7928b1fa0d4a79305f6dd2dc55221ac35ac" dependencies = [ "bitflags 2.9.0", "widestring 1.0.2", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b031dbfac8336..4424dbd79edfc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -407,7 +407,7 @@ sysinfo = "0.34.2" byteorder = "1.5.0" [target.'cfg(windows)'.dependencies] -windows-service = "0.7.0" +windows-service = "0.8.0" [target.'cfg(unix)'.dependencies] nix = { version = "0.26.2", default-features = false, features = ["socket", "signal"] } From 2e067e7054dd4cd88f4b2416407b49b902ccb447 Mon Sep 17 00:00:00 2001 From: Bruce Guenter Date: Mon, 9 Jun 2025 13:23:45 -0600 Subject: [PATCH 040/305] fix(sources): Handle sources with no default output (#23172) * fix(sources): Handle sources with no default output There exist sources that can be configured to output events entirely to non-default ports. The refactoring to `struct SourceSender` in #23089 failed to take this into account. This PR reverts the changes to the `default_output` handling in the builder and `SourceSender` with the addition of a small cleanup of its own. The sources that can have this condition are `datadog_agent` when `multiple_outputs` is set and `opentelemetry`. * Generalize comment --- src/source_sender/mod.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/source_sender/mod.rs b/src/source_sender/mod.rs index bf0ee0525597f..719d55968385b 100644 --- a/src/source_sender/mod.rs +++ b/src/source_sender/mod.rs @@ -155,7 +155,7 @@ impl Builder { pub fn build(self) -> SourceSender { SourceSender { - default_output: self.default_output.expect("no default output"), + default_output: self.default_output, named_outputs: self.named_outputs, } } @@ -163,7 +163,9 @@ impl Builder { #[derive(Debug, Clone)] pub struct SourceSender { - default_output: Output, + // The default output is optional because some sources, e.g. `datadog_agent` + // and `opentelemetry`, can be configured to only output to named outputs. + default_output: Option, named_outputs: HashMap, } @@ -183,7 +185,7 @@ impl SourceSender { Output::new_with_buffer(n, DEFAULT_OUTPUT.to_owned(), lag_time, None, output_id); ( Self { - default_output, + default_output: Some(default_output), named_outputs: Default::default(), }, rx, @@ -265,11 +267,16 @@ impl SourceSender { recv } + /// Get a mutable reference to the default output, panicking if none exists. + fn default_output_mut(&mut self) -> &mut Output { + self.default_output.as_mut().expect("no default output") + } + /// Send an event to the default output. /// /// This internally handles emitting [EventsSent] and [ComponentEventsDropped] events. pub async fn send_event(&mut self, event: impl Into) -> Result<(), ClosedError> { - self.default_output.send_event(event).await + self.default_output_mut().send_event(event).await } /// Send a stream of events to the default output. @@ -280,7 +287,7 @@ impl SourceSender { S: Stream + Unpin, E: Into + ByteSizeOf, { - self.default_output.send_event_stream(events).await + self.default_output_mut().send_event_stream(events).await } /// Send a batch of events to the default output. @@ -292,7 +299,7 @@ impl SourceSender { I: IntoIterator, ::IntoIter: ExactSizeIterator, { - self.default_output.send_batch(events).await + self.default_output_mut().send_batch(events).await } /// Send a batch of events event to a named output. From 4539b58964ab8bf23fb8f51a34718e233f7555db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ensar=20Saraj=C4=8Di=C4=87?= Date: Mon, 9 Jun 2025 20:47:04 +0000 Subject: [PATCH 041/305] fix(dnstap source): fix rcode 16 handling in dnsmsg-parser (#23106) * fix(dnstap source): fix rcode 16 handling in dnsmsg-parser Changes RCODE 16 description to BADVERS instead of BADSIG, since BADSIG is used for TSIG RR (which is not parsed currently). Related: https://www.iana.org/go/rfc6891 Related: https://www.iana.org/go/rfc8945 Closes: #22949 * Add changelog entry * Allow BADVERS in spelling --- .github/actions/spelling/allow.txt | 1 + changelog.d/23106_rcode_16_dnsmsg_parser_handling.fix.md | 3 +++ lib/dnsmsg-parser/src/dns_message_parser.rs | 4 ++-- lib/dnstap-parser/src/vrl_functions/parse_dnstap.rs | 4 ++-- website/cue/reference/components/sources/dnstap.cue | 1 + website/cue/reference/remap/functions/parse_dnstap.cue | 2 +- 6 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 changelog.d/23106_rcode_16_dnsmsg_parser_handling.fix.md diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index 36e53e7b8f9af..341a2ed509dae 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -50,6 +50,7 @@ Comcast Consolas Coolpad BADDCAFE +BADVERS DEBHELPER Danew dkr diff --git a/changelog.d/23106_rcode_16_dnsmsg_parser_handling.fix.md b/changelog.d/23106_rcode_16_dnsmsg_parser_handling.fix.md new file mode 100644 index 0000000000000..59506f1fb6ff3 --- /dev/null +++ b/changelog.d/23106_rcode_16_dnsmsg_parser_handling.fix.md @@ -0,0 +1,3 @@ +Fixed description for RCODE 16 (it was BADSIG previously, but it should be BADVERS). + +authors: esensar Quad9DNS diff --git a/lib/dnsmsg-parser/src/dns_message_parser.rs b/lib/dnsmsg-parser/src/dns_message_parser.rs index fadb27109ae63..e80c380324fe1 100644 --- a/lib/dnsmsg-parser/src/dns_message_parser.rs +++ b/lib/dnsmsg-parser/src/dns_message_parser.rs @@ -945,8 +945,8 @@ fn parse_response_code(rcode: u16) -> Option<&'static str> { 9 => Some("NotAuth"), // 9 NotAuth Server Not Authoritative for zone [RFC2136] 10 => Some("NotZone"), // 10 NotZone Name not contained in zone [RFC2136] // backwards compat for 4 bit ResponseCodes so far. - // 16 BADVERS Bad OPT Version [RFC6891] - 16 => Some("BADSIG"), // 16 BADSIG TSIG Signature Failure [RFC2845] + 16 => Some("BADVERS"), // 16 BADVERS Bad OPT Version [RFC6891] + // 16 BADSIG TSIG Signature Failure [RFC2845] 17 => Some("BADKEY"), // 17 BADKEY Key not recognized [RFC2845] 18 => Some("BADTIME"), // 18 BADTIME Signature out of time window [RFC2845] 19 => Some("BADMODE"), // 19 BADMODE Bad TKEY Mode [RFC2930] diff --git a/lib/dnstap-parser/src/vrl_functions/parse_dnstap.rs b/lib/dnstap-parser/src/vrl_functions/parse_dnstap.rs index ee6c5d52bc8b8..3b397478d00b8 100644 --- a/lib/dnstap-parser/src/vrl_functions/parse_dnstap.rs +++ b/lib/dnstap-parser/src/vrl_functions/parse_dnstap.rs @@ -120,7 +120,7 @@ impl Function for ParseDnstap { "questionTypeId": 6 } ], - "rcodeName": "BADSIG" + "rcodeName": "BADVERS" }, "responseAddress": "2001:502:7094::30", "responsePort": 53, @@ -295,7 +295,7 @@ mod tests { questionTypeId: 6, } ], - rcodeName: "BADSIG", + rcodeName: "BADVERS", }, responseAddress: "2001:502:7094::30", responsePort: 53, diff --git a/website/cue/reference/components/sources/dnstap.cue b/website/cue/reference/components/sources/dnstap.cue index 19235f95e0678..836f37ad78667 100644 --- a/website/cue/reference/components/sources/dnstap.cue +++ b/website/cue/reference/components/sources/dnstap.cue @@ -658,6 +658,7 @@ components: sources: dnstap: { NXRRSet: "RR Set that should exist does not" NotAuth: "Server Not Authoritative for zone" NotZone: "Name not contained in zone" + BADVERS: "Bad OPT Version" BADSIG: "TSIG Signature Failure" BADKEY: "Key not recognized" BADTIME: "Signature out of time window" diff --git a/website/cue/reference/remap/functions/parse_dnstap.cue b/website/cue/reference/remap/functions/parse_dnstap.cue index 55fc381adc767..de91822af17d7 100644 --- a/website/cue/reference/remap/functions/parse_dnstap.cue +++ b/website/cue/reference/remap/functions/parse_dnstap.cue @@ -123,7 +123,7 @@ remap: functions: parse_dnstap: { "questionTypeId": 6 }, ] - "rcodeName": "BADSIG" + "rcodeName": "BADVERS" } "responseAddress": "2001:502:7094::30" "responsePort": 53 From 1bf3328af6c4f4d01a9830a34a7078906431ecff Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 9 Jun 2025 17:54:25 -0400 Subject: [PATCH 042/305] chore(ci): add docker compose debug logs (if a test fails) (#23176) * chore(e2e): output docker compose logs when in debug mode * support long form vdev - read_only set to true now * always output logs on failure * more edits * expand int --- scripts/e2e/datadog-logs/compose.yaml | 42 +++++++++++++++++++++------ scripts/int-e2e-test.sh | 17 +++++++++++ vdev/src/testing/config.rs | 25 ++++++++++++++-- vdev/src/testing/integration.rs | 19 +++++++----- 4 files changed, 84 insertions(+), 19 deletions(-) diff --git a/scripts/e2e/datadog-logs/compose.yaml b/scripts/e2e/datadog-logs/compose.yaml index 61a9f8f2da1ae..ae004588f46f3 100644 --- a/scripts/e2e/datadog-logs/compose.yaml +++ b/scripts/e2e/datadog-logs/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: '3.8' services: # Generates random log data for consumption by the custom Agent check @@ -17,7 +17,9 @@ services: - "-o" - "/var/log/a_custom.log" volumes: - - log_path:/var/log/ + - type: volume + source: log_path + target: /var/log/ # Tails a custom log created by `log_generator` and sends log data to # the `fakeintake-agent` service @@ -33,11 +35,21 @@ services: - DD_CONTAINER_EXCLUDE="name:.*" volumes: # The Agent config file - - ../../../tests/data/e2e/datadog/logs/agent_only.yaml:/etc/datadog-agent/datadog.yaml + - type: bind + source: ../../../tests/data/e2e/datadog/logs/agent_only.yaml + target: /etc/datadog-agent/datadog.yaml + read_only: true + # The custom logs check - - ../../../tests/data/e2e/datadog/logs/logs.conf.d:/conf.d:ro + - type: bind + source: ../../../tests/data/e2e/datadog/logs/logs.conf.d + target: /conf.d + read_only: true + # The custom log to tail, created by the `log_generator` service - - log_path:/var/log/ + - type: volume + source: log_path + target: /var/log/ # Tails a custom log created by `log_generator` and sends log data to # the `vector` service @@ -53,11 +65,21 @@ services: - DD_CONTAINER_EXCLUDE="name:.*" volumes: # The Agent config file - - ../../../tests/data/e2e/datadog/logs/agent_vector.yaml:/etc/datadog-agent/datadog.yaml + - type: bind + source: ../../../tests/data/e2e/datadog/logs/agent_vector.yaml + target: /etc/datadog-agent/datadog.yaml + read_only: true + # The custom logs check - - ../../../tests/data/e2e/datadog/logs/logs.conf.d:/conf.d:ro + - type: bind + source: ../../../tests/data/e2e/datadog/logs/logs.conf.d + target: /conf.d + read_only: true + # The custom log to tail, created by the `log_generator` service - - log_path:/var/log/ + - type: volume + source: log_path + target: /var/log/ # Receives log data from the `datadog-agent-vector` service and sends # to the `fakeintake-vector` service. @@ -79,7 +101,9 @@ services: - "-c" - "/home/vector/tests/data/e2e/datadog/logs/vector.toml" volumes: - - ../../..:/home/vector + - type: bind + source: ../../.. + target: /home/vector # Receives log data from the `datadog-agent` service. Is queried by the test runner # which does the validation of consistency with the other fakeintake service. diff --git a/scripts/int-e2e-test.sh b/scripts/int-e2e-test.sh index f795c352e3d8e..51f41999aec13 100755 --- a/scripts/int-e2e-test.sh +++ b/scripts/int-e2e-test.sh @@ -21,6 +21,23 @@ cargo vdev -v "${TEST_TYPE}" start -a "${TEST_NAME}" sleep 30 cargo vdev -v "${TEST_TYPE}" test --retries 2 -a "${TEST_NAME}" RET=$? + +# Output docker compose logs on failure +if [[ $RET -ne 0 ]]; then + SCRIPT_DIR=$(realpath "$(dirname "${BASH_SOURCE[0]}")") + + case "$TEST_TYPE" in + int) TYPE_DIR="integration" ;; + e2e) TYPE_DIR="e2e" ;; + *) TYPE_DIR="" ;; + esac + + if [[ -n "$TYPE_DIR" ]]; then + COMPOSE_DIR="${SCRIPT_DIR}/${TYPE_DIR}/${TEST_NAME}" + (cd "${COMPOSE_DIR}" && docker compose logs) + fi +fi + cargo vdev -v "${TEST_TYPE}" stop -a "${TEST_NAME}" # Only upload test results if CI is defined diff --git a/vdev/src/testing/config.rs b/vdev/src/testing/config.rs index 487da26a8489e..f8bfaf549000d 100644 --- a/vdev/src/testing/config.rs +++ b/vdev/src/testing/config.rs @@ -47,11 +47,32 @@ pub fn get_rust_version() -> String { } } +#[derive(Debug, Deserialize, Serialize)] +#[serde(untagged)] +pub enum VolumeMount { + Short(String), + Long { + #[serde(default)] + r#type: Option, + source: String, + target: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + read_only: Option, + }, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(untagged)] +pub enum VolumeDefinition { + Empty, + WithOptions(BTreeMap), +} + #[derive(Debug, Deserialize, Serialize)] pub struct ComposeConfig { pub services: BTreeMap, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub volumes: BTreeMap, + pub volumes: BTreeMap, #[serde(default)] pub networks: BTreeMap>, } @@ -73,7 +94,7 @@ pub struct ComposeService { #[serde(default, skip_serializing_if = "Option::is_none")] pub env_file: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] - pub volumes: Option>, + pub volumes: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub environment: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index 15202bc6730dc..a2d55aca07c98 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -338,9 +338,9 @@ mod unix { use std::os::unix::fs::PermissionsExt as _; use std::path::{Path, PathBuf}; - use anyhow::{Context, Result}; - use super::super::config::ComposeConfig; + use crate::testing::config::VolumeMount; + use anyhow::{Context, Result}; /// Unix permissions mask to allow everybody to read a file const ALL_READ: u32 = 0o444; @@ -350,14 +350,17 @@ mod unix { /// Fix up potential issues before starting a compose container pub fn prepare_compose_volumes(config: &ComposeConfig, test_dir: &Path) -> Result<()> { for service in config.services.values() { - // Make sure all volume files are world readable if let Some(volumes) = &service.volumes { for volume in volumes { - let source = volume - .split_once(':') - .expect("Invalid volume in compose file") - .0; - // Only fixup relative paths, i.e. within our source tree. + let source = match volume { + VolumeMount::Short(s) => { + s.split_once(':').map(|(s, _)| s).ok_or_else(|| { + anyhow::anyhow!("Invalid short volume mount format: {s}") + })? + } + VolumeMount::Long { source, .. } => source, + }; + if !config.volumes.contains_key(source) && !source.starts_with('/') && !source.starts_with('$') From 6c1922e2769c49b70db179f94e9969802e319ef8 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 10 Jun 2025 10:16:38 -0400 Subject: [PATCH 043/305] chore(deps): update Rust to 1.86 (#23175) * chore(deps): update to rust 1.86 * cargo fmt * one more fix --- lib/codecs/src/encoding/mod.rs | 4 ++-- lib/vector-buffers/src/config.rs | 2 +- lib/vector-buffers/src/test/variant.rs | 2 +- lib/vector-buffers/src/topology/builder.rs | 2 +- .../src/topology/channel/receiver.rs | 1 - .../src/topology/channel/sender.rs | 2 +- .../src/variants/disk_v2/reader.rs | 2 +- .../src/variants/disk_v2/writer.rs | 10 +--------- lib/vector-common/src/request_metadata.rs | 2 +- lib/vector-core/src/event/discriminant.rs | 2 +- lib/vector-core/src/event/metadata.rs | 2 +- lib/vector-core/src/event/util/log/all_fields.rs | 4 ++-- lib/vector-core/src/tls/settings.rs | 2 +- lib/vector-stream/src/driver.rs | 2 +- lib/vector-tap/src/controller.rs | 2 +- rust-toolchain.toml | 2 +- src/api/schema/relay.rs | 2 +- src/async_read.rs | 2 +- src/codecs/ready_frames.rs | 2 +- src/components/validation/resources/event.rs | 2 +- src/config/mod.rs | 2 +- src/sinks/aws_kinesis/firehose/record.rs | 2 +- src/sinks/aws_kinesis/streams/record.rs | 2 +- src/sinks/datadog/metrics/request_builder.rs | 5 ++++- src/sinks/util/buffer/metrics/normalize.rs | 2 +- src/sinks/util/buffer/mod.rs | 16 +++++++++------- src/sinks/util/retries.rs | 2 +- src/sinks/util/service.rs | 8 ++++---- src/source_sender/mod.rs | 6 +++--- src/sources/aws_kinesis_firehose/errors.rs | 1 + src/sources/aws_kinesis_firehose/handlers.rs | 11 ++++++----- src/sources/docker_logs/mod.rs | 4 ++-- src/sources/opentelemetry/status.rs | 1 + src/sources/socket/tcp.rs | 8 ++++---- src/sources/socket/udp.rs | 2 +- src/sources/util/wrappers.rs | 2 +- src/template.rs | 1 + src/topology/builder.rs | 4 ++-- src/topology/task.rs | 1 + src/transforms/log_to_metric.rs | 2 +- src/transforms/remap.rs | 2 +- 41 files changed, 68 insertions(+), 67 deletions(-) diff --git a/lib/codecs/src/encoding/mod.rs b/lib/codecs/src/encoding/mod.rs index 411f3f692515f..bd71a4f3c5e02 100644 --- a/lib/codecs/src/encoding/mod.rs +++ b/lib/codecs/src/encoding/mod.rs @@ -397,7 +397,7 @@ impl SerializerConfig { } SerializerConfig::Cef(config) => config.input_type(), SerializerConfig::Csv(config) => config.input_type(), - SerializerConfig::Gelf { .. } => GelfSerializerConfig::input_type(), + SerializerConfig::Gelf => GelfSerializerConfig::input_type(), SerializerConfig::Json(config) => config.input_type(), SerializerConfig::Logfmt => LogfmtSerializerConfig.input_type(), SerializerConfig::Native => NativeSerializerConfig.input_type(), @@ -416,7 +416,7 @@ impl SerializerConfig { } SerializerConfig::Cef(config) => config.schema_requirement(), SerializerConfig::Csv(config) => config.schema_requirement(), - SerializerConfig::Gelf { .. } => GelfSerializerConfig::schema_requirement(), + SerializerConfig::Gelf => GelfSerializerConfig::schema_requirement(), SerializerConfig::Json(config) => config.schema_requirement(), SerializerConfig::Logfmt => LogfmtSerializerConfig.schema_requirement(), SerializerConfig::Native => NativeSerializerConfig.schema_requirement(), diff --git a/lib/vector-buffers/src/config.rs b/lib/vector-buffers/src/config.rs index 14826b6c204c0..9264685d4fa28 100644 --- a/lib/vector-buffers/src/config.rs +++ b/lib/vector-buffers/src/config.rs @@ -290,7 +290,7 @@ impl BufferType { let data_dir = data_dir.ok_or(BufferBuildError::RequiresDataDir)?; builder.stage(DiskV2Buffer::new(id, data_dir, max_size), when_full); } - }; + } Ok(()) } diff --git a/lib/vector-buffers/src/test/variant.rs b/lib/vector-buffers/src/test/variant.rs index a14995d86b436..0d58ce388d3d7 100644 --- a/lib/vector-buffers/src/test/variant.rs +++ b/lib/vector-buffers/src/test/variant.rs @@ -67,7 +67,7 @@ impl Variant { *when_full, ); } - }; + } let (sender, receiver) = builder .build(String::from("benches"), Span::none()) diff --git a/lib/vector-buffers/src/topology/builder.rs b/lib/vector-buffers/src/topology/builder.rs index 51f52900eca2f..9f13b6d17a228 100644 --- a/lib/vector-buffers/src/topology/builder.rs +++ b/lib/vector-buffers/src/topology/builder.rs @@ -129,7 +129,7 @@ impl TopologyBuilder { return Err(TopologyError::NextStageNotUsed { stage_idx }); } } - }; + } // Create the buffer usage handle for this stage and initialize it as we create the // sender/receiver. This is slightly awkward since we just end up actually giving diff --git a/lib/vector-buffers/src/topology/channel/receiver.rs b/lib/vector-buffers/src/topology/channel/receiver.rs index d21aa1ed67e4f..4afacc3bb0dff 100644 --- a/lib/vector-buffers/src/topology/channel/receiver.rs +++ b/lib/vector-buffers/src/topology/channel/receiver.rs @@ -54,7 +54,6 @@ where // If we've hit a recoverable error, we'll emit an event to indicate as much but we'll still // keep trying to read the next available record. emit(re); - continue; } None => panic!("Reader encountered unrecoverable error: {e:?}"), }, diff --git a/lib/vector-buffers/src/topology/channel/sender.rs b/lib/vector-buffers/src/topology/channel/sender.rs index 15191ec02eead..7e4b09dad2753 100644 --- a/lib/vector-buffers/src/topology/channel/sender.rs +++ b/lib/vector-buffers/src/topology/channel/sender.rs @@ -221,7 +221,7 @@ impl BufferSender { .await?; } } - }; + } if sent_to_base || was_dropped { if let (Some(send_duration), Some(send_reference)) = diff --git a/lib/vector-buffers/src/variants/disk_v2/reader.rs b/lib/vector-buffers/src/variants/disk_v2/reader.rs index 8aea89645ca64..a584af5d38d6f 100644 --- a/lib/vector-buffers/src/variants/disk_v2/reader.rs +++ b/lib/vector-buffers/src/variants/disk_v2/reader.rs @@ -1035,7 +1035,7 @@ where return Err(e); } - }; + } // Fundamentally, when `try_read_record` returns `None`, there's three possible // scenarios: diff --git a/lib/vector-buffers/src/variants/disk_v2/writer.rs b/lib/vector-buffers/src/variants/disk_v2/writer.rs index fc8f60672e1af..173ccd1248b14 100644 --- a/lib/vector-buffers/src/variants/disk_v2/writer.rs +++ b/lib/vector-buffers/src/variants/disk_v2/writer.rs @@ -1161,12 +1161,7 @@ where /// If an error occurred while writing the record, an error variant will be returned describing /// the error. pub async fn try_write_record(&mut self, record: T) -> Result, WriterError> { - self.try_write_record_inner(record) - .await - .map(|result| match result { - Ok(_) => None, - Err(record) => Some(record), - }) + self.try_write_record_inner(record).await.map(Result::err) } #[instrument(skip_all, level = "debug")] @@ -1219,8 +1214,6 @@ where last_attempted_write_size = serialized_len, "Current data file reached maximum size. Rolling to the next data file." ); - - continue; } e => return Err(e), }, @@ -1294,7 +1287,6 @@ where Err(old_record) => { record = old_record; self.ledger.wait_for_reader().await; - continue; } } } diff --git a/lib/vector-common/src/request_metadata.rs b/lib/vector-common/src/request_metadata.rs index 12a164d23e437..d38321a476e40 100644 --- a/lib/vector-common/src/request_metadata.rs +++ b/lib/vector-common/src/request_metadata.rs @@ -196,7 +196,7 @@ impl AddAssign for GroupedCountByteSize { } } (Self::Untagged { .. }, Self::Tagged { .. }) => unreachable!(), - }; + } } } diff --git a/lib/vector-core/src/event/discriminant.rs b/lib/vector-core/src/event/discriminant.rs index 20770506562db..08f7f913222ab 100644 --- a/lib/vector-core/src/event/discriminant.rs +++ b/lib/vector-core/src/event/discriminant.rs @@ -77,7 +77,7 @@ fn f64_eq(this: f64, other: f64) -> bool { } if this != other { return false; - }; + } if (this.is_sign_positive() && other.is_sign_negative()) || (this.is_sign_negative() && other.is_sign_positive()) { diff --git a/lib/vector-core/src/event/metadata.rs b/lib/vector-core/src/event/metadata.rs index c1dd5790236bb..b7be59900503e 100644 --- a/lib/vector-core/src/event/metadata.rs +++ b/lib/vector-core/src/event/metadata.rs @@ -356,7 +356,7 @@ impl EventMetadata { inner.source_event_id = Some(uuid2); } _ => {} // Keep the existing value. - }; + } } /// Update the finalizer(s) status. diff --git a/lib/vector-core/src/event/util/log/all_fields.rs b/lib/vector-core/src/event/util/log/all_fields.rs index 10afadfede4cc..5194f05cb707d 100644 --- a/lib/vector-core/src/event/util/log/all_fields.rs +++ b/lib/vector-core/src/event/util/log/all_fields.rs @@ -144,7 +144,7 @@ impl<'a> FieldsIter<'a> { None => break res.into(), Some(PathComponent::Key(key)) => { if self.quote_invalid_fields && !IS_VALID_PATH_SEGMENT.is_match(key) { - res.push_str(&format!("\"{key}\"")); + write!(res, "\"{key}\"").expect("write to String never fails"); } else { res.push_str(key); } @@ -191,7 +191,7 @@ impl<'a> Iterator for FieldsIter<'a> { *visited = true; break result; } - }; + } } } } diff --git a/lib/vector-core/src/tls/settings.rs b/lib/vector-core/src/tls/settings.rs index e0c84dc51308c..0fece816894a9 100644 --- a/lib/vector-core/src/tls/settings.rs +++ b/lib/vector-core/src/tls/settings.rs @@ -370,7 +370,7 @@ impl TlsConfig { |der| X509::from_der(&der).map(|x509| vec![x509]), |pem| { pem.match_indices(PEM_START_MARKER) - .map(|(start, _)| X509::from_pem(pem[start..].as_bytes())) + .map(|(start, _)| X509::from_pem(&pem.as_bytes()[start..])) .collect() }, ) diff --git a/lib/vector-stream/src/driver.rs b/lib/vector-stream/src/driver.rs index b142deba814f2..45ec1af74741a 100644 --- a/lib/vector-stream/src/driver.rs +++ b/lib/vector-stream/src/driver.rs @@ -226,7 +226,7 @@ where finalizers.update_status(EventStatus::Rejected); } } - }; + } drop(finalizers); // suppress "argument not consumed" warning } diff --git a/lib/vector-tap/src/controller.rs b/lib/vector-tap/src/controller.rs index 656f77c108bc2..c07318d224507 100644 --- a/lib/vector-tap/src/controller.rs +++ b/lib/vector-tap/src/controller.rs @@ -26,7 +26,7 @@ type TapSender = tokio_mpsc::Sender; type ShutdownTx = oneshot::Sender<()>; type ShutdownRx = oneshot::Receiver<()>; -const TAP_BUFFER_SIZE: NonZeroUsize = unsafe { NonZeroUsize::new_unchecked(100) }; +const TAP_BUFFER_SIZE: NonZeroUsize = NonZeroUsize::new(100).unwrap(); /// Clients can supply glob patterns to find matched topology components. trait GlobMatcher { diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 5bedd373fc370..bd493b61329bf 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.85" +channel = "1.86" profile = "default" diff --git a/src/api/schema/relay.rs b/src/api/schema/relay.rs index 366a71723aafe..83f397a9d8f76 100644 --- a/src/api/schema/relay.rs +++ b/src/api/schema/relay.rs @@ -49,7 +49,7 @@ impl Base64Cursor { let cursor = String::from_utf8(bytes).map_err(|_| Base64CursorError::Invalid)?; let index = cursor .split(':') - .last() + .next_back() .map(|s| s.parse::()) .ok_or(Base64CursorError::Invalid)? .map_err(|_| Base64CursorError::Invalid)?; diff --git a/src/async_read.rs b/src/async_read.rs index 00a3c53024684..6dc0f0f7d83dd 100644 --- a/src/async_read.rs +++ b/src/async_read.rs @@ -39,7 +39,7 @@ impl AllowReadUntil { &self.reader } - pub fn get_mut(&mut self) -> &mut S { + pub const fn get_mut(&mut self) -> &mut S { &mut self.reader } } diff --git a/src/codecs/ready_frames.rs b/src/codecs/ready_frames.rs index 4aed021b57e83..a707140b23001 100644 --- a/src/codecs/ready_frames.rs +++ b/src/codecs/ready_frames.rs @@ -52,7 +52,7 @@ where } /// Returns a mutable reference to the underlying stream. - pub fn get_mut(&mut self) -> &mut T { + pub const fn get_mut(&mut self) -> &mut T { &mut self.inner } diff --git a/src/components/validation/resources/event.rs b/src/components/validation/resources/event.rs index b7a6e102ea799..46ae48378ae1f 100644 --- a/src/components/validation/resources/event.rs +++ b/src/components/validation/resources/event.rs @@ -101,7 +101,7 @@ impl TestEvent { } } - pub fn get_event(&mut self) -> &mut Event { + pub const fn get_event(&mut self) -> &mut Event { match self { Self::Passthrough(event) => event, Self::FailWithAlternateEncoder(event) => event, diff --git a/src/config/mod.rs b/src/config/mod.rs index 8da0ce75c4255..8f3b247db57a0 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -265,7 +265,7 @@ impl HealthcheckOptions { } } - fn merge(&mut self, other: Self) { + const fn merge(&mut self, other: Self) { self.enabled &= other.enabled; self.require_healthy |= other.require_healthy; } diff --git a/src/sinks/aws_kinesis/firehose/record.rs b/src/sinks/aws_kinesis/firehose/record.rs index 24704aa3b50d5..5cd6c44fddc31 100644 --- a/src/sinks/aws_kinesis/firehose/record.rs +++ b/src/sinks/aws_kinesis/firehose/record.rs @@ -28,7 +28,7 @@ impl Record for KinesisFirehoseRecord { fn encoded_length(&self) -> usize { let data_len = self.record.data.as_ref().len(); // data is simply base64 encoded, quoted, and comma separated - (data_len + 2) / 3 * 4 + 3 + data_len.div_ceil(3) * 4 + 3 } fn get(self) -> Self::T { diff --git a/src/sinks/aws_kinesis/streams/record.rs b/src/sinks/aws_kinesis/streams/record.rs index 16bf89515c277..ebd59fd81a890 100644 --- a/src/sinks/aws_kinesis/streams/record.rs +++ b/src/sinks/aws_kinesis/streams/record.rs @@ -38,7 +38,7 @@ impl Record for KinesisStreamRecord { let data_len = self.record.data.as_ref().len(); let key_len = self.record.partition_key.len(); - (data_len + 2) / 3 * 4 + hash_key_size + key_len + 10 + data_len.div_ceil(3) * 4 + hash_key_size + key_len + 10 } fn get(self) -> Self::T { diff --git a/src/sinks/datadog/metrics/request_builder.rs b/src/sinks/datadog/metrics/request_builder.rs index 794d8b5a58a2e..83b53b8d2b6ed 100644 --- a/src/sinks/datadog/metrics/request_builder.rs +++ b/src/sinks/datadog/metrics/request_builder.rs @@ -91,7 +91,10 @@ impl DatadogMetricsRequestBuilder { }) } - fn get_encoder(&mut self, endpoint: DatadogMetricsEndpoint) -> &mut DatadogMetricsEncoder { + const fn get_encoder( + &mut self, + endpoint: DatadogMetricsEndpoint, + ) -> &mut DatadogMetricsEncoder { match endpoint { DatadogMetricsEndpoint::Series { .. } => &mut self.series_encoder, DatadogMetricsEndpoint::Sketches => &mut self.sketches_encoder, diff --git a/src/sinks/util/buffer/metrics/normalize.rs b/src/sinks/util/buffer/metrics/normalize.rs index 57b77413b7f41..edb63f5f8a4d7 100644 --- a/src/sinks/util/buffer/metrics/normalize.rs +++ b/src/sinks/util/buffer/metrics/normalize.rs @@ -47,7 +47,7 @@ pub struct MetricNormalizer { impl MetricNormalizer { /// Gets a mutable reference to the current metric state for this normalizer. - pub fn get_state_mut(&mut self) -> &mut MetricSet { + pub const fn get_state_mut(&mut self) -> &mut MetricSet { &mut self.state } } diff --git a/src/sinks/util/buffer/mod.rs b/src/sinks/util/buffer/mod.rs index 25760b345388e..a5797e6eb14f9 100644 --- a/src/sinks/util/buffer/mod.rs +++ b/src/sinks/util/buffer/mod.rs @@ -198,10 +198,12 @@ mod test { batch_settings.timeout, ); - let input = std::iter::repeat(BytesMut::from( - "It's going down, I'm yelling timber, You better move, you better dance", - )) - .take(100_000); + let input = std::iter::repeat_n( + BytesMut::from( + "It's going down, I'm yelling timber, You better move, you better dance", + ), + 100_000, + ); buffered .sink_map_err(drop) @@ -228,10 +230,10 @@ mod test { decompressed }); - assert!(decompressed.eq(std::iter::repeat( - b"It's going down, I'm yelling timber, You better move, you better dance".to_vec() + assert!(decompressed.eq(std::iter::repeat_n( + b"It's going down, I'm yelling timber, You better move, you better dance".to_vec(), + 100_000 ) - .take(100_000) .flatten())); } } diff --git a/src/sinks/util/retries.rs b/src/sinks/util/retries.rs index e265be25a9089..32ac8fd251f2e 100644 --- a/src/sinks/util/retries.rs +++ b/src/sinks/util/retries.rs @@ -283,7 +283,7 @@ impl ExponentialBackoff { } /// Resents the exponential back-off strategy to its initial state. - pub fn reset(&mut self) { + pub const fn reset(&mut self) { self.current = self.base; } } diff --git a/src/sinks/util/service.rs b/src/sinks/util/service.rs index 283637b74b595..5960250034144 100644 --- a/src/sinks/util/service.rs +++ b/src/sinks/util/service.rs @@ -90,8 +90,8 @@ pub trait TowerRequestConfigDefaults { const RATE_LIMIT_DURATION_SECS: u64 = 1; const RATE_LIMIT_NUM: u64 = i64::MAX as u64; // i64 avoids TOML deserialize issue const RETRY_ATTEMPTS: usize = isize::MAX as usize; // isize avoids TOML deserialize issue - const RETRY_MAX_DURATION_SECS: NonZeroU64 = unsafe { NonZeroU64::new_unchecked(30) }; - const RETRY_INITIAL_BACKOFF_SECS: NonZeroU64 = unsafe { NonZeroU64::new_unchecked(1) }; + const RETRY_MAX_DURATION_SECS: NonZeroU64 = NonZeroU64::new(30).unwrap(); + const RETRY_INITIAL_BACKOFF_SECS: NonZeroU64 = NonZeroU64::new(1).unwrap(); } #[derive(Clone, Copy, Debug)] @@ -474,8 +474,8 @@ mod tests { const RATE_LIMIT_DURATION_SECS: u64 = 2; const RATE_LIMIT_NUM: u64 = 3; const RETRY_ATTEMPTS: usize = 4; - const RETRY_MAX_DURATION_SECS: NonZeroU64 = unsafe { NonZeroU64::new_unchecked(5) }; - const RETRY_INITIAL_BACKOFF_SECS: NonZeroU64 = unsafe { NonZeroU64::new_unchecked(6) }; + const RETRY_MAX_DURATION_SECS: NonZeroU64 = NonZeroU64::new(5).unwrap(); + const RETRY_INITIAL_BACKOFF_SECS: NonZeroU64 = NonZeroU64::new(6).unwrap(); } #[test] diff --git a/src/source_sender/mod.rs b/src/source_sender/mod.rs index 719d55968385b..243eb7ad8ecf7 100644 --- a/src/source_sender/mod.rs +++ b/src/source_sender/mod.rs @@ -268,7 +268,7 @@ impl SourceSender { } /// Get a mutable reference to the default output, panicking if none exists. - fn default_output_mut(&mut self) -> &mut Output { + const fn default_output_mut(&mut self) -> &mut Output { self.default_output.as_mut().expect("no default output") } @@ -339,11 +339,11 @@ impl UnsentEventCount { } } - fn decr(&mut self, count: usize) { + const fn decr(&mut self, count: usize) { self.count = self.count.saturating_sub(count); } - fn discard(&mut self) { + const fn discard(&mut self) { self.count = 0; } } diff --git a/src/sources/aws_kinesis_firehose/errors.rs b/src/sources/aws_kinesis_firehose/errors.rs index fb96487d33801..0aaa518244414 100644 --- a/src/sources/aws_kinesis_firehose/errors.rs +++ b/src/sources/aws_kinesis_firehose/errors.rs @@ -76,6 +76,7 @@ impl RequestError { } } + #[allow(clippy::missing_const_for_fn)] // Adding `const` results in https://doc.rust-lang.org/error_codes/E0015.html pub fn request_id(&self) -> Option<&str> { use RequestError::*; match *self { diff --git a/src/sources/aws_kinesis_firehose/handlers.rs b/src/sources/aws_kinesis_firehose/handlers.rs index cd424d97a336a..4ad8ddf50121b 100644 --- a/src/sources/aws_kinesis_firehose/handlers.rs +++ b/src/sources/aws_kinesis_firehose/handlers.rs @@ -78,13 +78,14 @@ pub(super) async fn firehose( events.estimated_json_encoded_size_of(), )); - let (batch, receiver) = context - .acknowledgements - .then(|| { + let (batch, receiver) = if context.acknowledgements { + { let (batch, receiver) = BatchNotifier::new_with_receiver(); (Some(batch), Some(receiver)) - }) - .unwrap_or((None, None)); + } + } else { + (None, None) + }; let now = Utc::now(); for event in &mut events { diff --git a/src/sources/docker_logs/mod.rs b/src/sources/docker_logs/mod.rs index 634cf45a1d936..4920c72bfe1af 100644 --- a/src/sources/docker_logs/mod.rs +++ b/src/sources/docker_logs/mod.rs @@ -904,12 +904,12 @@ impl ContainerState { } } - fn running(&mut self) { + const fn running(&mut self) { self.running = true; self.generation += 1; } - fn stopped(&mut self) { + const fn stopped(&mut self) { self.running = false; } diff --git a/src/sources/opentelemetry/status.rs b/src/sources/opentelemetry/status.rs index 7aa6fddae8e0d..e7d9c038896d5 100644 --- a/src/sources/opentelemetry/status.rs +++ b/src/sources/opentelemetry/status.rs @@ -1,3 +1,4 @@ +#![allow(clippy::doc_overindented_list_items)] // The generated code has this minor issue. include!(concat!(env!("OUT_DIR"), "/google.rpc.rs")); impl warp::reject::Reject for Status {} diff --git a/src/sources/socket/tcp.rs b/src/sources/socket/tcp.rs index 1aea8b0332d48..98b10dbf2105e 100644 --- a/src/sources/socket/tcp.rs +++ b/src/sources/socket/tcp.rs @@ -158,12 +158,12 @@ impl TcpConfig { self.max_connection_duration_secs } - pub fn set_max_connection_duration_secs(&mut self, val: Option) -> &mut Self { + pub const fn set_max_connection_duration_secs(&mut self, val: Option) -> &mut Self { self.max_connection_duration_secs = val; self } - pub fn set_shutdown_timeout_secs(&mut self, val: u64) -> &mut Self { + pub const fn set_shutdown_timeout_secs(&mut self, val: u64) -> &mut Self { self.shutdown_timeout_secs = Duration::from_secs(val); self } @@ -173,7 +173,7 @@ impl TcpConfig { self } - pub fn set_framing(&mut self, val: Option) -> &mut Self { + pub const fn set_framing(&mut self, val: Option) -> &mut Self { self.framing = val; self } @@ -183,7 +183,7 @@ impl TcpConfig { self } - pub fn set_log_namespace(&mut self, val: Option) -> &mut Self { + pub const fn set_log_namespace(&mut self, val: Option) -> &mut Self { self.log_namespace = val; self } diff --git a/src/sources/socket/udp.rs b/src/sources/socket/udp.rs index a1b39c2fbcd33..a5d57207ea0a0 100644 --- a/src/sources/socket/udp.rs +++ b/src/sources/socket/udp.rs @@ -128,7 +128,7 @@ impl UdpConfig { } } - pub fn set_log_namespace(&mut self, val: Option) -> &mut Self { + pub const fn set_log_namespace(&mut self, val: Option) -> &mut Self { self.log_namespace = val; self } diff --git a/src/sources/util/wrappers.rs b/src/sources/util/wrappers.rs index aab0a650293d8..848ecac9c4c8e 100644 --- a/src/sources/util/wrappers.rs +++ b/src/sources/util/wrappers.rs @@ -39,7 +39,7 @@ impl AfterRead { } #[cfg(all(unix, feature = "sources-utils-net-unix"))] - pub fn get_mut_ref(&mut self) -> &mut T { + pub const fn get_mut_ref(&mut self) -> &mut T { &mut self.inner } } diff --git a/src/template.rs b/src/template.rs index ec989f1968159..dab20876a1ac7 100644 --- a/src/template.rs +++ b/src/template.rs @@ -228,6 +228,7 @@ impl Template { (!parts.is_empty()).then_some(parts) } + #[allow(clippy::missing_const_for_fn)] // Adding `const` results in https://doc.rust-lang.org/error_codes/E0015.html /// Returns a reference to the template string. pub fn get_ref(&self) -> &str { &self.src diff --git a/src/topology/builder.rs b/src/topology/builder.rs index 00252622a4742..847c0614ba697 100644 --- a/src/topology/builder.rs +++ b/src/topology/builder.rs @@ -61,8 +61,8 @@ static ENRICHMENT_TABLES: LazyLock = pub(crate) static SOURCE_SENDER_BUFFER_SIZE: LazyLock = LazyLock::new(|| *TRANSFORM_CONCURRENCY_LIMIT * CHUNK_SIZE); -const READY_ARRAY_CAPACITY: NonZeroUsize = unsafe { NonZeroUsize::new_unchecked(CHUNK_SIZE * 4) }; -pub(crate) const TOPOLOGY_BUFFER_SIZE: NonZeroUsize = unsafe { NonZeroUsize::new_unchecked(100) }; +const READY_ARRAY_CAPACITY: NonZeroUsize = NonZeroUsize::new(CHUNK_SIZE * 4).unwrap(); +pub(crate) const TOPOLOGY_BUFFER_SIZE: NonZeroUsize = NonZeroUsize::new(100).unwrap(); static TRANSFORM_CONCURRENCY_LIMIT: LazyLock = LazyLock::new(|| { crate::app::worker_threads() diff --git a/src/topology/task.rs b/src/topology/task.rs index afdbaf9da165b..2db88e1f48fbe 100644 --- a/src/topology/task.rs +++ b/src/topology/task.rs @@ -79,6 +79,7 @@ impl Task { self.key.id() } + #[allow(clippy::missing_const_for_fn)] // Adding `const` results in https://doc.rust-lang.org/error_codes/E0015.html pub fn typetag(&self) -> &str { &self.typetag } diff --git a/src/transforms/log_to_metric.rs b/src/transforms/log_to_metric.rs index 48cfbecfeb501..65eabd9300183 100644 --- a/src/transforms/log_to_metric.rs +++ b/src/transforms/log_to_metric.rs @@ -887,7 +887,7 @@ impl FunctionTransform for LogToMetric { kind: &kind.to_string(), }) } - TransformError::MetricDetailsNotFound {} => { + TransformError::MetricDetailsNotFound => { emit!(MetricMetadataMetricDetailsNotFoundError {}) } _ => {} diff --git a/src/transforms/remap.rs b/src/transforms/remap.rs index 9f92a9a61b26b..0f1d24c1aa007 100644 --- a/src/transforms/remap.rs +++ b/src/transforms/remap.rs @@ -493,7 +493,7 @@ where .notes() .iter() .filter(|note| matches!(note, Note::UserErrorMessage(_))) - .last() + .next_back() .map(|note| note.to_string()) .unwrap_or_else(|| error.to_string()); serde_json::json!({ From ab886d8e1b1f22ffae76ebc999c9494971ec0780 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 10 Jun 2025 11:56:11 -0400 Subject: [PATCH 044/305] fix(ci): update cargo-deny version to support 2024 edition (#23178) --- aqua/aqua.yaml | 2 +- scripts/environment/prepare.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/aqua/aqua.yaml b/aqua/aqua.yaml index e72564f079e00..7a1d785e753bc 100644 --- a/aqua/aqua.yaml +++ b/aqua/aqua.yaml @@ -9,6 +9,6 @@ packages: - name: crates.io/cargo-deb@2.9.1 - name: cross-rs/cross@v0.2.5 - name: nextest-rs/nextest/cargo-nextest@cargo-nextest-0.9.47 - - name: EmbarkStudios/cargo-deny@0.16.1 + - name: EmbarkStudios/cargo-deny@0.16.2 - name: foresterre/cargo-msrv@v0.15.1 - name: crates.io/dd-rust-license-tool@1.0.1 diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index a42daaf8c427d..fef332b218011 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -14,8 +14,8 @@ fi if [[ "$(cargo-nextest --version)" != "cargo-nextest 0.9.95" ]] ; then rustup run stable cargo install cargo-nextest --version 0.9.95 --force --locked fi -if [[ "$(cargo-deny --version)" != "cargo-deny 0.16.1" ]] ; then - rustup run stable cargo install cargo-deny --version 0.16.1 --force --locked +if [[ "$(cargo-deny --version)" != "cargo-deny 0.16.2" ]] ; then + rustup run stable cargo install cargo-deny --version 0.16.2 --force --locked fi if ! dd-rust-license-tool --help >& /dev/null ; then rustup run stable cargo install dd-rust-license-tool --version 1.0.2 --force --locked From 0191c3af087c1eb00094c7076452d27f19c92e50 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Jun 2025 18:44:51 +0000 Subject: [PATCH 045/305] chore(deps): Bump ring from 0.17.12 to 0.17.14 in the cargo group (#23020) * chore(deps): Bump ring from 0.17.12 to 0.17.14 in the cargo group Bumps the cargo group with 1 update: [ring](https://github.com/briansmith/ring). Updates `ring` from 0.17.12 to 0.17.14 - [Changelog](https://github.com/briansmith/ring/blob/main/RELEASES.md) - [Commits](https://github.com/briansmith/ring/commits) --- updated-dependencies: - dependency-name: ring dependency-version: 0.17.14 dependency-type: indirect dependency-group: cargo ... Signed-off-by: dependabot[bot] * update licenses --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Pavlos Rontidis --- Cargo.lock | 4 ++-- LICENSE-3rdparty.csv | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eb0faad3c3dff..4b59a190e6b44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8608,9 +8608,9 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.12" +version = "0.17.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed9b823fa29b721a59671b41d6b06e66b29e0628e207e8b1c3ceeda701ec928d" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index dbe0a174c9ec3..67538f6f1ded5 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -531,7 +531,7 @@ rend,https://github.com/djkoloski/rend,MIT,David Koloski reqwest,https://github.com/seanmonstar/reqwest,MIT OR Apache-2.0,Sean McArthur resolv-conf,http://github.com/tailhook/resolv-conf,MIT OR Apache-2.0,paul@colomiets.name rfc6979,https://github.com/RustCrypto/signatures/tree/master/rfc6979,Apache-2.0 OR MIT,RustCrypto Developers -ring,https://github.com/ctz/ring,Apache-2.0 AND ISC,The ring Authors +ring,https://github.com/briansmith/ring,Apache-2.0 AND ISC,The ring Authors rkyv,https://github.com/rkyv/rkyv,MIT,David Koloski rle-decode-fast,https://github.com/WanzenBug/rle-decode-helper,MIT OR Apache-2.0,Moritz Wanzenböck rmp,https://github.com/3Hren/msgpack-rust,MIT,Evgeny Safronov From f84dbbc915b525d4ba54257bbf40de209468b2f6 Mon Sep 17 00:00:00 2001 From: Burkov Egor Date: Tue, 10 Jun 2025 21:59:56 +0300 Subject: [PATCH 046/305] fix(parsing): Timestamp error handling in dnstap parser (#23072) * fix(parsing): Timestamp error handling in dnstap parser * Remove redundant check for time_sec * Change ok_or_else to ok_or * fix changelog name --------- Co-authored-by: Pavlos Rontidis --- changelog.d/dnstap_better_errors_timestamp.fix.md | 3 +++ lib/dnstap-parser/src/parser.rs | 11 ++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 changelog.d/dnstap_better_errors_timestamp.fix.md diff --git a/changelog.d/dnstap_better_errors_timestamp.fix.md b/changelog.d/dnstap_better_errors_timestamp.fix.md new file mode 100644 index 0000000000000..a970b91d6127c --- /dev/null +++ b/changelog.d/dnstap_better_errors_timestamp.fix.md @@ -0,0 +1,3 @@ +Fix problem with parsing timestamp that could lead into panic in `dnstap-parser::DnstapParser::parse_dnstap_message_time`. + +authors: wooffie diff --git a/lib/dnstap-parser/src/parser.rs b/lib/dnstap-parser/src/parser.rs index 14443ef581e76..80f6068525585 100644 --- a/lib/dnstap-parser/src/parser.rs +++ b/lib/dnstap-parser/src/parser.rs @@ -393,11 +393,10 @@ impl DnstapParser { if type_ids.contains(&dnstap_message_type_id) { DnstapParser::log_time(event, prefix.clone(), time_in_nanosec, "ns"); - let timestamp = Utc .timestamp_opt(time_sec.try_into().unwrap(), query_time_nsec) .single() - .expect("invalid timestamp"); + .ok_or("Invalid timestamp")?; if let Some(timestamp_key) = log_schema().timestamp_key() { DnstapParser::insert(event, prefix.clone(), timestamp_key, timestamp); } @@ -1358,9 +1357,9 @@ mod tests { fn test_one_timestamp_parse(time_sec: u64, time_nsec: Option) -> Result<()> { let mut event = LogEvent::default(); let root = owned_value_path!(); - let type_ids = HashSet::new(); + let type_ids = HashSet::from([1]); DnstapParser::parse_dnstap_message_time( - &mut event, &root, time_sec, time_nsec, 0, None, &type_ids, + &mut event, &root, time_sec, time_nsec, 1, None, &type_ids, ) } // okay case @@ -1374,7 +1373,9 @@ mod tests { // overflow in add assert!( test_one_timestamp_parse((i64::MAX / 1_000_000_000) as u64, Some(u32::MAX)).is_err() - ) + ); + // cannot be parsed by timestamp_opt + assert!(test_one_timestamp_parse(96, Some(1616928816)).is_err()); } #[test] From d1da30726228589ec8f84e38052feb60e67afdc7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Jun 2025 15:18:08 -0400 Subject: [PATCH 047/305] chore(deps): Bump the patches group across 1 directory with 27 updates (#23179) * chore(deps): Bump the patches group across 1 directory with 27 updates Bumps the patches group with 27 updates in the / directory: | Package | From | To | | --- | --- | --- | | [cfg-if](https://github.com/rust-lang/cfg-if) | `1.0.0` | `1.0.1` | | [clap](https://github.com/clap-rs/clap) | `4.5.37` | `4.5.40` | | [aws-runtime](https://github.com/smithy-lang/smithy-rs) | `1.5.6` | `1.5.7` | | [aws-sigv4](https://github.com/smithy-lang/smithy-rs) | `1.3.1` | `1.3.2` | | [opendal](https://github.com/apache/opendal) | `0.53.1` | `0.53.3` | | [async-graphql](https://github.com/async-graphql/async-graphql) | `7.0.16` | `7.0.17` | | [async-graphql-warp](https://github.com/async-graphql/async-graphql) | `7.0.16` | `7.0.17` | | [async-compression](https://github.com/Nullus157/async-compression) | `0.4.23` | `0.4.24` | | [flate2](https://github.com/rust-lang/flate2-rs) | `1.1.1` | `1.1.2` | | [h2](https://github.com/hyperium/h2) | `0.4.9` | `0.4.10` | | [hash_hasher](https://github.com/Fraser999/Hash-Hasher) | `2.0.3` | `2.0.4` | | [openssl](https://github.com/sfackler/rust-openssl) | `0.10.72` | `0.10.73` | | [smallvec](https://github.com/servo/rust-smallvec) | `1.15.0` | `1.15.1` | | [socket2](https://github.com/rust-lang/socket2) | `0.5.9` | `0.5.10` | | [sqlx](https://github.com/launchbadge/sqlx) | `0.8.5` | `0.8.6` | | [toml](https://github.com/toml-rs/toml) | `0.8.22` | `0.8.23` | | [hickory-proto](https://github.com/hickory-dns/hickory-dns) | `0.25.1` | `0.25.2` | | [mlua](https://github.com/mlua-rs/mlua) | `0.10.3` | `0.10.5` | | [temp-dir](https://gitlab.com/leonhard-llc/ops) | `0.1.14` | `0.1.16` | | [syn](https://github.com/dtolnay/syn) | `2.0.101` | `2.0.102` | | [enumflags2](https://github.com/meithecatte/enumflags2) | `0.7.11` | `0.7.12` | | [parking_lot](https://github.com/Amanieu/parking_lot) | `0.12.3` | `0.12.4` | | [quanta](https://github.com/metrics-rs/quanta) | `0.12.5` | `0.12.6` | | [twox-hash](https://github.com/shepmaster/twox-hash) | `2.1.0` | `2.1.1` | | [clap-verbosity-flag](https://github.com/clap-rs/clap-verbosity-flag) | `3.0.2` | `3.0.3` | | [clap_complete](https://github.com/clap-rs/clap) | `4.5.48` | `4.5.54` | | [owo-colors](https://github.com/owo-colors/owo-colors) | `4.2.0` | `4.2.1` | Updates `cfg-if` from 1.0.0 to 1.0.1 - [Release notes](https://github.com/rust-lang/cfg-if/releases) - [Changelog](https://github.com/rust-lang/cfg-if/blob/main/CHANGELOG.md) - [Commits](https://github.com/rust-lang/cfg-if/compare/1.0.0...v1.0.1) Updates `clap` from 4.5.37 to 4.5.40 - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/clap_complete-v4.5.37...clap_complete-v4.5.40) Updates `aws-runtime` from 1.5.6 to 1.5.7 - [Release notes](https://github.com/smithy-lang/smithy-rs/releases) - [Changelog](https://github.com/smithy-lang/smithy-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy-rs/commits) Updates `aws-sigv4` from 1.3.1 to 1.3.2 - [Release notes](https://github.com/smithy-lang/smithy-rs/releases) - [Changelog](https://github.com/smithy-lang/smithy-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy-rs/commits) Updates `opendal` from 0.53.1 to 0.53.3 - [Release notes](https://github.com/apache/opendal/releases) - [Changelog](https://github.com/apache/opendal/blob/main/CHANGELOG.md) - [Commits](https://github.com/apache/opendal/compare/v0.53.1...v0.53.3) Updates `async-graphql` from 7.0.16 to 7.0.17 - [Changelog](https://github.com/async-graphql/async-graphql/blob/master/CHANGELOG.md) - [Commits](https://github.com/async-graphql/async-graphql/commits) Updates `async-graphql-warp` from 7.0.16 to 7.0.17 - [Changelog](https://github.com/async-graphql/async-graphql/blob/master/CHANGELOG.md) - [Commits](https://github.com/async-graphql/async-graphql/commits) Updates `async-compression` from 0.4.23 to 0.4.24 - [Release notes](https://github.com/Nullus157/async-compression/releases) - [Changelog](https://github.com/Nullus157/async-compression/blob/main/CHANGELOG.md) - [Commits](https://github.com/Nullus157/async-compression/compare/v0.4.23...v0.4.24) Updates `flate2` from 1.1.1 to 1.1.2 - [Release notes](https://github.com/rust-lang/flate2-rs/releases) - [Commits](https://github.com/rust-lang/flate2-rs/compare/1.1.1...1.1.2) Updates `h2` from 0.4.9 to 0.4.10 - [Release notes](https://github.com/hyperium/h2/releases) - [Changelog](https://github.com/hyperium/h2/blob/master/CHANGELOG.md) - [Commits](https://github.com/hyperium/h2/compare/v0.4.9...v0.4.10) Updates `hash_hasher` from 2.0.3 to 2.0.4 - [Commits](https://github.com/Fraser999/Hash-Hasher/commits) Updates `openssl` from 0.10.72 to 0.10.73 - [Release notes](https://github.com/sfackler/rust-openssl/releases) - [Commits](https://github.com/sfackler/rust-openssl/compare/openssl-v0.10.72...openssl-v0.10.73) Updates `smallvec` from 1.15.0 to 1.15.1 - [Release notes](https://github.com/servo/rust-smallvec/releases) - [Commits](https://github.com/servo/rust-smallvec/compare/v1.15.0...v1.15.1) Updates `socket2` from 0.5.9 to 0.5.10 - [Release notes](https://github.com/rust-lang/socket2/releases) - [Changelog](https://github.com/rust-lang/socket2/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/socket2/commits) Updates `sqlx` from 0.8.5 to 0.8.6 - [Changelog](https://github.com/launchbadge/sqlx/blob/main/CHANGELOG.md) - [Commits](https://github.com/launchbadge/sqlx/compare/v0.8.5...v0.8.6) Updates `toml` from 0.8.22 to 0.8.23 - [Commits](https://github.com/toml-rs/toml/compare/toml-v0.8.22...toml-v0.8.23) Updates `hickory-proto` from 0.25.1 to 0.25.2 - [Release notes](https://github.com/hickory-dns/hickory-dns/releases) - [Changelog](https://github.com/hickory-dns/hickory-dns/blob/main/OLD-CHANGELOG.md) - [Commits](https://github.com/hickory-dns/hickory-dns/compare/v0.25.1...v0.25.2) Updates `mlua` from 0.10.3 to 0.10.5 - [Release notes](https://github.com/mlua-rs/mlua/releases) - [Changelog](https://github.com/mlua-rs/mlua/blob/v0.10.5/CHANGELOG.md) - [Commits](https://github.com/mlua-rs/mlua/compare/v0.10.3...v0.10.5) Updates `temp-dir` from 0.1.14 to 0.1.16 - [Commits](https://gitlab.com/leonhard-llc/ops/compare/temp-dir-v0.1.14...temp-dir-v0.1.16) Updates `syn` from 2.0.101 to 2.0.102 - [Release notes](https://github.com/dtolnay/syn/releases) - [Commits](https://github.com/dtolnay/syn/compare/2.0.101...2.0.102) Updates `enumflags2` from 0.7.11 to 0.7.12 - [Release notes](https://github.com/meithecatte/enumflags2/releases) - [Commits](https://github.com/meithecatte/enumflags2/compare/v0.7.11...v0.7.12) Updates `parking_lot` from 0.12.3 to 0.12.4 - [Release notes](https://github.com/Amanieu/parking_lot/releases) - [Changelog](https://github.com/Amanieu/parking_lot/blob/master/CHANGELOG.md) - [Commits](https://github.com/Amanieu/parking_lot/compare/0.12.3...parking_lot-v0.12.4) Updates `quanta` from 0.12.5 to 0.12.6 - [Changelog](https://github.com/metrics-rs/quanta/blob/main/CHANGELOG.md) - [Commits](https://github.com/metrics-rs/quanta/compare/v0.12.5...v0.12.6) Updates `twox-hash` from 2.1.0 to 2.1.1 - [Changelog](https://github.com/shepmaster/twox-hash/blob/main/CHANGELOG.md) - [Commits](https://github.com/shepmaster/twox-hash/compare/v2.1.0...v2.1.1) Updates `clap-verbosity-flag` from 3.0.2 to 3.0.3 - [Changelog](https://github.com/clap-rs/clap-verbosity-flag/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap-verbosity-flag/compare/v3.0.2...v3.0.3) Updates `clap_complete` from 4.5.48 to 4.5.54 - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/clap_complete-v4.5.48...clap_complete-v4.5.54) Updates `owo-colors` from 4.2.0 to 4.2.1 - [Release notes](https://github.com/owo-colors/owo-colors/releases) - [Changelog](https://github.com/owo-colors/owo-colors/blob/main/CHANGELOG.md) - [Commits](https://github.com/owo-colors/owo-colors/compare/v4.2.0...v4.2.1) --- updated-dependencies: - dependency-name: cfg-if dependency-version: 1.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: clap dependency-version: 4.5.40 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: aws-runtime dependency-version: 1.5.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: aws-sigv4 dependency-version: 1.3.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: opendal dependency-version: 0.53.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: async-graphql dependency-version: 7.0.17 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: async-graphql-warp dependency-version: 7.0.17 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: async-compression dependency-version: 0.4.24 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: flate2 dependency-version: 1.1.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: h2 dependency-version: 0.4.10 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: hash_hasher dependency-version: 2.0.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: openssl dependency-version: 0.10.73 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: smallvec dependency-version: 1.15.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: socket2 dependency-version: 0.5.10 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: sqlx dependency-version: 0.8.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: toml dependency-version: 0.8.23 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: hickory-proto dependency-version: 0.25.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: mlua dependency-version: 0.10.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: temp-dir dependency-version: 0.1.16 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: syn dependency-version: 2.0.102 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: enumflags2 dependency-version: 0.7.12 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: parking_lot dependency-version: 0.12.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: quanta dependency-version: 0.12.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: twox-hash dependency-version: 2.1.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: clap-verbosity-flag dependency-version: 3.0.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: clap_complete dependency-version: 4.5.54 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: owo-colors dependency-version: 4.2.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches ... Signed-off-by: dependabot[bot] * update licenses * update hickory proto code * bump msrv, required by async graph ql * clippy fix --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Pavlos Rontidis --- Cargo.lock | 380 ++++++++++---------- Cargo.toml | 34 +- LICENSE-3rdparty.csv | 12 +- lib/dnsmsg-parser/src/dns_message_parser.rs | 27 +- lib/file-source/Cargo.toml | 2 +- lib/vector-buffers/Cargo.toml | 2 +- lib/vector-core/Cargo.toml | 12 +- lib/vector-stream/Cargo.toml | 2 +- lib/vector-tap/Cargo.toml | 2 +- src/config/diff.rs | 4 +- vdev/Cargo.toml | 6 +- 11 files changed, 252 insertions(+), 231 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4b59a190e6b44..ff1cdfe628db0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -409,9 +409,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.23" +version = "0.4.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b37fc50485c4f3f736a4fb14199f6d5f5ba008d7f28fe710306c92780f004c07" +checksum = "d615619615a650c571269c00dca41db04b9210037fa76ed8239f70404ab56985" dependencies = [ "brotli", "flate2", @@ -477,9 +477,9 @@ dependencies = [ [[package]] name = "async-graphql" -version = "7.0.16" +version = "7.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3ee559e72d983e7e04001ba3bf32e6b71c1d670595780723727fd8a29d36e87" +checksum = "036618f842229ba0b89652ffe425f96c7c16a49f7e3cb23b56fca7f61fd74980" dependencies = [ "async-graphql-derive", "async-graphql-parser", @@ -508,9 +508,9 @@ dependencies = [ [[package]] name = "async-graphql-derive" -version = "7.0.16" +version = "7.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29db05b624fb6352fc11bfe30c54ab1b16a1fe937d7c05a783f4e88ef1292b3b" +checksum = "fd45deb3dbe5da5cdb8d6a670a7736d735ba65b455328440f236dfb113727a3d" dependencies = [ "Inflector", "async-graphql-parser", @@ -519,15 +519,15 @@ dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", "strum 0.26.3", - "syn 2.0.101", + "syn 2.0.102", "thiserror 1.0.68", ] [[package]] name = "async-graphql-parser" -version = "7.0.16" +version = "7.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4904895044116aab098ca82c6cec831ec43ed99efd04db9b70a390419bc88c5b" +checksum = "60b7607e59424a35dadbc085b0d513aa54ec28160ee640cf79ec3b634eba66d3" dependencies = [ "async-graphql-value", "pest", @@ -537,9 +537,9 @@ dependencies = [ [[package]] name = "async-graphql-value" -version = "7.0.16" +version = "7.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0cde74de18e3a00c5dd5cfa002ab6f532e1a06c2a79ee6671e2fc353b400b92" +checksum = "34ecdaff7c9cffa3614a9f9999bf9ee4c3078fe3ce4d6a6e161736b56febf2de" dependencies = [ "bytes 1.10.1", "indexmap 2.9.0", @@ -549,9 +549,9 @@ dependencies = [ [[package]] name = "async-graphql-warp" -version = "7.0.16" +version = "7.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e662bc1abaf791b2b21b54f679bb2cd34ba456c6c27b908c3f2b18ec5ce30a8b" +checksum = "ff6008a33c32d5a048aa72437821eb864dd56a80c0d80c8df48f11f12154db6c" dependencies = [ "async-graphql", "futures-util", @@ -700,7 +700,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -740,7 +740,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -757,7 +757,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -825,9 +825,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.6" +version = "1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aff45ffe35196e593ea3b9dd65b320e51e2dda95aff4390bc459e461d09c6ad" +checksum = "6c4063282c69991e57faab9e5cb21ae557e59f5b0fb285c196335243df8dc25c" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -842,7 +842,6 @@ dependencies = [ "fastrand 2.3.0", "http 0.2.9", "http-body 0.4.5", - "once_cell", "percent-encoding", "pin-project-lite", "tracing 0.1.41", @@ -1170,9 +1169,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.3.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3503af839bd8751d0bdc5a46b9cac93a003a353e635b0c12cf2376b5b53e41ea" +checksum = "3734aecf9ff79aa401a6ca099d076535ab465ff76b46440cf567c8e70b65dc13" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -1282,7 +1281,7 @@ dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "h2 0.4.9", + "h2 0.4.10", "http 0.2.9", "http-body 0.4.5", "hyper 0.14.28", @@ -1729,7 +1728,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6cbbb8f56245b5a479b30a62cdc86d26e2f35c2b9f594bc4671654b03851380" dependencies = [ "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -2048,9 +2047,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" [[package]] name = "cfg_aliases" @@ -2175,9 +2174,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.37" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eccb054f56cbd38340b380d4a8e69ef1f02f1af43db2f0cc817a4774d80ae071" +checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f" dependencies = [ "clap_builder", "clap_derive", @@ -2185,9 +2184,9 @@ dependencies = [ [[package]] name = "clap-verbosity-flag" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2678fade3b77aa3a8ff3aae87e9c008d3fb00473a41c71fbf74e91c8c7b37e84" +checksum = "eeab6a5cdfc795a05538422012f20a5496f050223c91be4e5420bfd13c641fb1" dependencies = [ "clap", "log", @@ -2195,9 +2194,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.37" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd9466fac8543255d3b1fcad4762c5e116ffe808c8a3043d4263cd4fd4862a2" +checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e" dependencies = [ "anstream", "anstyle", @@ -2208,23 +2207,23 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.5.48" +version = "4.5.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be8c97f3a6f02b9e24cadc12aaba75201d18754b53ea0a9d99642f806ccdb4c9" +checksum = "aad5b1b4de04fead402672b48897030eec1f3bfe1550776322f59f6d6e6a5677" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.5.32" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" +checksum = "d2c7947ae4cc3d851207c1adb5b5e260ff0cca11446b1d6d1423788e442257ce" dependencies = [ "heck 0.5.0", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -2815,7 +2814,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -2863,7 +2862,7 @@ dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", "strsim 0.11.1", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -2885,7 +2884,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -3022,7 +3021,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -3043,7 +3042,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -3053,7 +3052,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" dependencies = [ "derive_builder_core", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -3137,7 +3136,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -3148,7 +3147,7 @@ checksum = "e5766087c2235fec47fafa4cfecc81e494ee679d0fd4a59887ea0919bfb0e4fc" dependencies = [ "cfg-if", "libc", - "socket2 0.5.9", + "socket2 0.5.10", "windows-sys 0.48.0", ] @@ -3394,7 +3393,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -3406,27 +3405,27 @@ dependencies = [ "once_cell", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] name = "enumflags2" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba2f4b465f5318854c6f8dd686ede6c0a9dc67d4b1ac241cf0eb51521a309147" +checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef" dependencies = [ "enumflags2_derive", ] [[package]] name = "enumflags2_derive" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc4caf64a58d7a6d65ab00639b046ff54399a39f5f2554728895ace4b297cd79" +checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -3717,9 +3716,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" +checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" dependencies = [ "crc32fast", "libz-rs-sys", @@ -3920,7 +3919,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -4220,9 +4219,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75249d144030531f8dee69fe9cea04d3edf809a017ae445e2abdff6629e86633" +checksum = "a9421a676d1b147b16b82c9225157dc629087ef8ec4d5e2960f9437a90dac0a5" dependencies = [ "atomic-waker", "bytes 1.10.1", @@ -4249,9 +4248,9 @@ dependencies = [ [[package]] name = "hash_hasher" -version = "2.0.3" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74721d007512d0cb3338cd20f0654ac913920061a4c4d0d8708edb3f2a698c0c" +checksum = "1b4b9ebce26001bad2e6366295f64e381c1e9c479109202149b9e15e154973e9" [[package]] name = "hashbag" @@ -4501,15 +4500,13 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hickory-proto" -version = "0.25.1" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d844af74f7b799e41c78221be863bade11c430d46042c3b49ca8ae0c6d27287" +checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" dependencies = [ - "async-recursion", "async-trait", "bitflags 2.9.0", "cfg-if", - "critical-section", "data-encoding", "enum-as-inner 0.6.0", "futures-channel", @@ -4704,7 +4701,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.5.9", + "socket2 0.5.10", "tokio", "tower-service", "tracing 0.1.41", @@ -4720,7 +4717,7 @@ dependencies = [ "bytes 1.10.1", "futures-channel", "futures-util", - "h2 0.4.9", + "h2 0.4.10", "http 1.1.0", "http-body 1.0.0", "httparse", @@ -4905,7 +4902,7 @@ dependencies = [ "http-body 1.0.0", "hyper 1.4.1", "pin-project-lite", - "socket2 0.5.9", + "socket2 0.5.10", "tokio", "tower-service", "tracing 0.1.41", @@ -5064,7 +5061,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -5203,7 +5200,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b23a0c8dfe501baac4adf6ebbfa6eddf8f0c07f56b058cc1288017e32397846c" dependencies = [ "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -5250,7 +5247,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ - "socket2 0.5.9", + "socket2 0.5.10", "widestring 1.0.2", "windows-sys 0.48.0", "winreg", @@ -5723,9 +5720,9 @@ dependencies = [ [[package]] name = "libz-rs-sys" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a" +checksum = "172a788537a2221661b480fee8dc5f96c580eb34fa88764d3205dc356c7e4221" dependencies = [ "zlib-rs", ] @@ -5800,9 +5797,9 @@ checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" dependencies = [ "autocfg", "scopeguard", @@ -6129,9 +6126,9 @@ dependencies = [ [[package]] name = "mlua" -version = "0.10.3" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3f763c1041eff92ffb5d7169968a327e1ed2ebfe425dac0ee5a35f29082534b" +checksum = "c1f5f8fbebc7db5f671671134b9321c4b9aa9adeafccfd9a8c020ae45c6a35d0" dependencies = [ "bstr 1.12.0", "either", @@ -6140,13 +6137,14 @@ dependencies = [ "num-traits", "parking_lot", "rustc-hash", + "rustversion", ] [[package]] name = "mlua-sys" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63a11d485edf0f3f04a508615d36c7d50d299cf61a7ee6d3e2530651e0a31771" +checksum = "380c1f7e2099cafcf40e51d3a9f20a346977587aa4d012eae1f043149a728a93" dependencies = [ "cc", "cfg-if", @@ -6167,7 +6165,7 @@ dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", "regex", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -6709,7 +6707,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -6721,7 +6719,7 @@ dependencies = [ "proc-macro-crate 3.2.0", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -6852,9 +6850,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "opendal" -version = "0.53.1" +version = "0.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0f407ca2797ca6c010720aa3cedfa0187430e4f70b729c75c33142d44021f59" +checksum = "f947c4efbca344c1a125753366033c8107f552b2e3f8251815ed1908f116ca3e" dependencies = [ "anyhow", "async-trait", @@ -6911,9 +6909,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.72" +version = "0.10.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da" +checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" dependencies = [ "bitflags 2.9.0", "cfg-if", @@ -6932,7 +6930,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -6952,9 +6950,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.107" +version = "0.9.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8288979acd84749c744a9014b4382d42b8f7b2592847b5afb2ed29e5d16ede07" +checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" dependencies = [ "cc", "libc", @@ -7028,9 +7026,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "owo-colors" -version = "4.2.0" +version = "4.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1036865bb9422d3300cf723f657c2851d0e9ab12567854b1f4eba3d77decf564" +checksum = "26995317201fa17f3656c36716aed4a7c81743a9634ac4c99c0eeda495db0cec" dependencies = [ "supports-color 2.1.0", "supports-color 3.0.1", @@ -7077,9 +7075,9 @@ checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] name = "parking_lot" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" dependencies = [ "lock_api", "parking_lot_core", @@ -7087,15 +7085,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.4.1", + "redox_syscall 0.5.12", "smallvec", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -7196,7 +7194,7 @@ dependencies = [ "pest_meta", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -7284,7 +7282,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -7562,7 +7560,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2 1.0.95", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -7603,7 +7601,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" dependencies = [ - "toml_edit 0.22.26", + "toml_edit 0.22.27", ] [[package]] @@ -7625,7 +7623,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -7699,7 +7697,7 @@ checksum = "4ee1c9ac207483d5e7db4940700de86a9aae46ef90c48b57f99fe7edb8345e49" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -7771,7 +7769,7 @@ dependencies = [ "prost 0.12.6", "prost-types 0.12.6", "regex", - "syn 2.0.101", + "syn 2.0.102", "tempfile", ] @@ -7798,7 +7796,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -7811,7 +7809,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -7947,9 +7945,9 @@ checksum = "658fa1faf7a4cc5f057c9ee5ef560f717ad9d8dc66d975267f709624d6e1ab88" [[package]] name = "quanta" -version = "0.12.5" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bd1fe6824cea6538803de3ff1bc0cf3949024db3d43c9643024bfb33a807c0e" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" dependencies = [ "crossbeam-utils", "libc", @@ -8020,7 +8018,7 @@ checksum = "f71ee38b42f8459a88d3362be6f9b841ad2d5421844f61eb1c59c11bff3ac14a" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -8035,7 +8033,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls 0.23.23", - "socket2 0.5.9", + "socket2 0.5.10", "thiserror 2.0.3", "tokio", "tracing 0.1.41", @@ -8070,7 +8068,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.9", + "socket2 0.5.10", "tracing 0.1.41", "windows-sys 0.59.0", ] @@ -8394,6 +8392,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" +dependencies = [ + "bitflags 2.9.0", +] + [[package]] name = "redox_users" version = "0.4.3" @@ -8751,7 +8758,7 @@ dependencies = [ "regex", "relative-path", "rustc_version 0.4.1", - "syn 2.0.101", + "syn 2.0.102", "unicode-ident", ] @@ -9213,7 +9220,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -9224,7 +9231,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -9287,14 +9294,14 @@ checksum = "3081f5ffbb02284dda55132aa26daecedd7372a42417bbbab6f14ab7d6bb9145" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] name = "serde_spanned" -version = "0.6.8" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" dependencies = [ "serde", ] @@ -9360,7 +9367,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -9551,9 +9558,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" dependencies = [ "serde", ] @@ -9633,7 +9640,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -9654,9 +9661,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" dependencies = [ "libc", "windows-sys 0.52.0", @@ -9698,9 +9705,9 @@ dependencies = [ [[package]] name = "sqlx" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3a85280daca669cfd3bcb68a337882a8bc57ec882f72c5d13a430613a738e" +checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" dependencies = [ "sqlx-core", "sqlx-macros", @@ -9711,9 +9718,9 @@ dependencies = [ [[package]] name = "sqlx-core" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f743f2a3cea30a58cd479013f75550e879009e3a02f616f18ca699335aa248c3" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" dependencies = [ "base64 0.22.1", "bytes 1.10.1", @@ -9746,22 +9753,22 @@ dependencies = [ [[package]] name = "sqlx-macros" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4200e0fde19834956d4252347c12a083bdcb237d7a1a1446bffd8768417dce" +checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", "sqlx-core", "sqlx-macros-core", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] name = "sqlx-macros-core" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "882ceaa29cade31beca7129b6beeb05737f44f82dbe2a9806ecea5a7093d00b7" +checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" dependencies = [ "dotenvy", "either", @@ -9777,17 +9784,16 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.101", - "tempfile", + "syn 2.0.102", "tokio", "url", ] [[package]] name = "sqlx-mysql" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0afdd3aa7a629683c2d750c2df343025545087081ab5942593a5288855b1b7a7" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ "atoi", "base64 0.22.1", @@ -9828,9 +9834,9 @@ dependencies = [ [[package]] name = "sqlx-postgres" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0bedbe1bbb5e2615ef347a5e9d8cd7680fb63e77d9dafc0f29be15e53f1ebe6" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ "atoi", "base64 0.22.1", @@ -9866,9 +9872,9 @@ dependencies = [ [[package]] name = "sqlx-sqlite" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c26083e9a520e8eb87a06b12347679b142dc2ea29e6e409f805644a7a979a5bc" +checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" dependencies = [ "atoi", "chrono", @@ -9988,7 +9994,7 @@ dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", "rustversion", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -10001,7 +10007,7 @@ dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", "rustversion", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -10053,9 +10059,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.101" +version = "2.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" +checksum = "f6397daf94fa90f058bd0fd88429dd9e5738999cca8d701813c80723add80462" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", @@ -10085,7 +10091,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -10176,9 +10182,9 @@ dependencies = [ [[package]] name = "temp-dir" -version = "0.1.14" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc1ee6eef34f12f765cb94725905c6312b6610ab2b0940889cfe58dae7bc3c72" +checksum = "83176759e9416cf81ee66cb6508dbfe9c96f20b8b56265a39917551c23c70964" [[package]] name = "tempfile" @@ -10277,7 +10283,7 @@ checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -10288,7 +10294,7 @@ checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -10403,7 +10409,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.9", + "socket2 0.5.10", "tokio-macros", "tracing 0.1.41", "windows-sys 0.52.0", @@ -10438,7 +10444,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -10482,7 +10488,7 @@ dependencies = [ "postgres-protocol", "postgres-types", "rand 0.9.1", - "socket2 0.5.9", + "socket2 0.5.10", "tokio", "tokio-util", "whoami", @@ -10596,21 +10602,21 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.22" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ae329d1f08c4d17a59bed7ff5b5a769d062e64a62d34a3261b219e62cd5aae" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.26", + "toml_edit 0.22.27", ] [[package]] name = "toml_datetime" -version = "0.6.9" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" dependencies = [ "serde", ] @@ -10628,23 +10634,23 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.26" +version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ "indexmap 2.9.0", "serde", "serde_spanned", "toml_datetime", "toml_write", - "winnow 0.7.7", + "winnow 0.7.10", ] [[package]] name = "toml_write" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfb942dfe1d8e29a7ee7fcbde5bd2b9a25fb89aa70caea2eba3bee836ff41076" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" [[package]] name = "tonic" @@ -10690,7 +10696,7 @@ dependencies = [ "axum 0.7.5", "base64 0.22.1", "bytes 1.10.1", - "h2 0.4.9", + "h2 0.4.10", "http 1.1.0", "http-body 1.0.0", "http-body-util", @@ -10700,7 +10706,7 @@ dependencies = [ "percent-encoding", "pin-project", "prost 0.13.3", - "socket2 0.5.9", + "socket2 0.5.10", "tokio", "tokio-stream", "tower", @@ -10732,7 +10738,7 @@ dependencies = [ "proc-macro2 1.0.95", "prost-build 0.12.6", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -10852,7 +10858,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -10977,7 +10983,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04659ddb06c87d233c566112c1c9c5b9e98256d9af50ec3bc9c8327f873a7568" dependencies = [ "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -11098,9 +11104,9 @@ dependencies = [ [[package]] name = "twox-hash" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7b17f197b3050ba473acf9181f7b1d3b66d1cf7356c6cc57886662276e65908" +checksum = "8b907da542cbced5261bd3256de1b3a1bf340a3d37f93425a07362a1d687de56" [[package]] name = "typed-builder" @@ -11130,7 +11136,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -11160,7 +11166,7 @@ checksum = "35f5380909ffc31b4de4f4bdf96b877175a016aa2ca98cee39fcfd8c4d53d952" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -11487,7 +11493,7 @@ dependencies = [ "goauth", "governor", "greptimedb-ingester", - "h2 0.4.9", + "h2 0.4.10", "hash_hasher", "hashbrown 0.14.5", "headers", @@ -11574,7 +11580,7 @@ dependencies = [ "smpl_jwt", "snafu 0.7.5", "snap", - "socket2 0.5.9", + "socket2 0.5.10", "sqlx", "stream-cancel", "strip-ansi-escapes", @@ -11739,7 +11745,7 @@ dependencies = [ "quote 1.0.40", "serde", "serde_json", - "syn 2.0.101", + "syn 2.0.102", "tracing 0.1.41", ] @@ -11752,7 +11758,7 @@ dependencies = [ "quote 1.0.40", "serde", "serde_derive_internals", - "syn 2.0.101", + "syn 2.0.102", "vector-config", "vector-config-common", ] @@ -11815,7 +11821,7 @@ dependencies = [ "similar-asserts", "smallvec", "snafu 0.7.5", - "socket2 0.5.9", + "socket2 0.5.10", "tokio", "tokio-openssl", "tokio-stream", @@ -11883,7 +11889,7 @@ dependencies = [ "tokio-util", "tower", "tracing 0.1.41", - "twox-hash 2.1.0", + "twox-hash 2.1.1", "vector-common", "vector-core", ] @@ -12203,7 +12209,7 @@ dependencies = [ "log", "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", "wasm-bindgen-shared", ] @@ -12237,7 +12243,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -12444,7 +12450,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -12455,7 +12461,7 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -12739,9 +12745,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.7.7" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb8234a863ea0e8cd7284fcdd4f145233eb00fee02bbdd9861aec44e6477bc5" +checksum = "c06928c8748d81b05c9be96aad92e1b6ff01833332f281e8cfca3be4b35fc9ec" dependencies = [ "memchr", ] @@ -12846,7 +12852,7 @@ checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", "synstructure", ] @@ -12876,7 +12882,7 @@ checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -12887,7 +12893,7 @@ checksum = "5226bc9a9a9836e7428936cde76bb6b22feea1a8bfdbc0d241136e4d13417e25" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] @@ -12907,7 +12913,7 @@ checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", "synstructure", ] @@ -12936,14 +12942,14 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2 1.0.95", "quote 1.0.40", - "syn 2.0.101", + "syn 2.0.102", ] [[package]] name = "zlib-rs" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8" +checksum = "626bd9fa9734751fc50d6060752170984d7053f5a39061f524cda68023d4db8a" [[package]] name = "zstd" diff --git a/Cargo.toml b/Cargo.toml index 4424dbd79edfc..3864e4e9306f2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ default-run = "vector" autobenches = false # our benchmarks are not runnable on their own either way # Minimum supported rust version # See docs/DEVELOPING.md for policy -rust-version = "1.83" +rust-version = "1.86" [[bin]] name = "vector" @@ -135,15 +135,15 @@ members = [ [workspace.dependencies] anyhow = "1.0.98" -cfg-if = { version = "1.0.0", default-features = false } +cfg-if = { version = "1.0.1", default-features = false } chrono = { version = "0.4.41", default-features = false, features = ["clock", "serde"] } chrono-tz = { version = "0.10.3", default-features = false, features = ["serde"] } -clap = { version = "4.5.37", default-features = false, features = ["derive", "error-context", "env", "help", "std", "string", "usage", "wrap_help"] } +clap = { version = "4.5.40", default-features = false, features = ["derive", "error-context", "env", "help", "std", "string", "usage", "wrap_help"] } darling = { version = "0.20.11", default-features = false, features = ["suggestions"] } -flate2 = { version = "1.1.1", default-features = false, features = ["zlib-rs"] } +flate2 = { version = "1.1.2", default-features = false, features = ["zlib-rs"] } futures = { version = "0.3.31", default-features = false, features = ["compat", "io-compat", "std"], package = "futures" } glob = { version = "0.3.2", default-features = false } -hickory-proto = { version = "0.25.1", default-features = false, features = ["dnssec-ring"] } +hickory-proto = { version = "0.25.2", default-features = false, features = ["dnssec-ring"] } indexmap = { version = "2.9.0", default-features = false, features = ["serde", "std"] } inventory = { version = "0.3" } indoc = { version = "2.0.6" } @@ -166,7 +166,7 @@ serde = { version = "1.0.219", default-features = false, features = ["alloc", "d snafu = { version = "0.7.5", default-features = false, features = ["futures", "std"] } tempfile = "3.19.1" tokio = { version = "1.45.1", default-features = false, features = ["full"] } -toml = { version = "0.8.22", default-features = false, features = ["display", "parse"] } +toml = { version = "0.8.23", default-features = false, features = ["display", "parse"] } tonic = { version = "0.11", default-features = false, features = ["transport", "codegen", "prost", "tls", "tls-roots", "gzip"] } tonic-build = { version = "0.11", default-features = false, features = ["transport", "prost"] } uuid = { version = "1.17.0", features = ["v4", "v7", "serde"] } @@ -225,7 +225,7 @@ metrics.workspace = true metrics-tracing-context.workspace = true # AWS - Official SDK -aws-runtime = { version = "1.5.6", optional = true } +aws-runtime = { version = "1.5.7", optional = true } aws-config = { version = "1.6.1", default-features = false, features = ["behavior-version-latest", "credentials-process", "sso", "rt-tokio"], optional = true } aws-credential-types = { version = "1.2.3", default-features = false, features = ["hardcoded-credentials"], optional = true } aws-sdk-cloudwatch = { version = "1.70.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } @@ -247,7 +247,7 @@ aws-sdk-sts = { version = "1.65.0", default-features = false, features = ["behav # The `aws-sdk-sts` crate is needed despite not being referred to anywhere in the code because we need to set the # `behavior-version-latest` feature. Without this we get a runtime panic when `auth.assume_role` authentication is configured. -aws-sigv4 = { version = "1.3.1", default-features = false, features = ["sign-http"], optional = true } +aws-sigv4 = { version = "1.3.2", default-features = false, features = ["sign-http"], optional = true } aws-smithy-async = { version = "1.2.5", default-features = false, features = ["rt-tokio"], optional = true } aws-smithy-http = { version = "0.62", default-features = false, features = ["event-stream", "rt-tokio"], optional = true } @@ -292,8 +292,8 @@ smpl_jwt = { version = "0.8.0", default-features = false, optional = true } lapin = { version = "2.5.3", default-features = false, features = ["native-tls"], optional = true } # API -async-graphql = { version = "7.0.16", default-features = false, optional = true, features = ["chrono", "playground"] } -async-graphql-warp = { version = "7.0.16", default-features = false, optional = true } +async-graphql = { version = "7.0.17", default-features = false, optional = true, features = ["chrono", "playground"] } +async-graphql-warp = { version = "7.0.17", default-features = false, optional = true } # API client crossterm = { version = "0.29.0", default-features = false, features = ["event-stream", "windows"], optional = true } @@ -310,7 +310,7 @@ greptimedb-ingester = { git = "https://github.com/GreptimeTeam/greptimedb-ingest # External libs arc-swap = { version = "1.7", default-features = false, optional = true } -async-compression = { version = "0.4.23", default-features = false, features = ["tokio", "gzip", "zstd"], optional = true } +async-compression = { version = "0.4.24", default-features = false, features = ["tokio", "gzip", "zstd"], optional = true } apache-avro = { version = "0.16.0", default-features = false, optional = true } axum = { version = "0.6.20", default-features = false } base64 = { version = "0.22.1", default-features = false, optional = true } @@ -335,8 +335,8 @@ flate2.workspace = true futures-util = { version = "0.3.29", default-features = false } glob.workspace = true governor = { version = "0.7.0", default-features = false, features = ["dashmap", "jitter", "std"], optional = true } -h2 = { version = "0.4.9", default-features = false, optional = true } -hash_hasher = { version = "2.0.0", default-features = false } +h2 = { version = "0.4.10", default-features = false, optional = true } +hash_hasher = { version = "2.0.4", default-features = false } hashbrown = { version = "0.14.5", default-features = false, optional = true, features = ["ahash"] } headers = { version = "0.3.9", default-features = false } hostname = { version = "0.4.0", default-features = false } @@ -362,7 +362,7 @@ async-nats = { version = "0.33.0", default-features = false, optional = true } nkeys = { version = "0.4.4", default-features = false, optional = true } nom = { version = "7.1.3", default-features = false, optional = true } notify = { version = "8.0.0", default-features = false, features = ["macos_fsevent"] } -openssl = { version = "0.10.72", default-features = false, features = ["vendored"] } +openssl = { version = "0.10.73", default-features = false, features = ["vendored"] } openssl-probe = { version = "0.1.6", default-features = false } ordered-float = { version = "4.6.0", default-features = false } percent-encoding = { version = "2.3.1", default-features = false } @@ -379,8 +379,8 @@ rumqttc = { version = "0.24.0", default-features = false, features = ["use-rustl seahash = { version = "4.1.0", default-features = false } smallvec = { version = "1", default-features = false, features = ["union", "serde"] } snap = { version = "1.1.1", default-features = false } -socket2 = { version = "0.5.9", default-features = false } -sqlx = { version = "0.8.5", default-features = false, features = ["derive", "postgres", "chrono", "runtime-tokio"], optional=true } +socket2 = { version = "0.5.10", default-features = false } +sqlx = { version = "0.8.6", default-features = false, features = ["derive", "postgres", "chrono", "runtime-tokio"], optional=true } stream-cancel = { version = "0.8.2", default-features = false } strip-ansi-escapes = { version = "0.2.1", default-features = false } syslog = { version = "6.1.1", default-features = false, optional = true } @@ -402,7 +402,7 @@ arr_macro = { version = "0.2.1" } heim = { git = "https://github.com/vectordotdev/heim.git", branch = "update-nix", default-features = false, features = ["disk"] } # make sure to update the external docs when the Lua version changes -mlua = { version = "0.10.3", default-features = false, features = ["lua54", "send", "vendored", "macros"], optional = true } +mlua = { version = "0.10.5", default-features = false, features = ["lua54", "send", "vendored", "macros"], optional = true } sysinfo = "0.34.2" byteorder = "1.5.0" diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 67538f6f1ded5..e107729c35ddc 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -122,7 +122,7 @@ castaway,https://github.com/sagebind/castaway,MIT,Stephen M. Coakley cfb-mode,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers -cfg-if,https://github.com/alexcrichton/cfg-if,MIT OR Apache-2.0,Alex Crichton +cfg-if,https://github.com/rust-lang/cfg-if,MIT OR Apache-2.0,Alex Crichton chacha20,https://github.com/RustCrypto/stream-ciphers,Apache-2.0 OR MIT,RustCrypto Developers chacha20poly1305,https://github.com/RustCrypto/AEADs/tree/master/chacha20poly1305,Apache-2.0 OR MIT,RustCrypto Developers charset,https://github.com/hsivonen/charset,MIT OR Apache-2.0,Henri Sivonen @@ -271,7 +271,7 @@ grok,https://github.com/daschl/grok,Apache-2.0,Michael Nitschinger , Jack Grigg " h2,https://github.com/hyperium/h2,MIT,"Carl Lerche , Sean McArthur " half,https://github.com/starkat99/half-rs,MIT OR Apache-2.0,Kathryn Long -hash_hasher,https://github.com/Fraser999/Hash-Hasher,Apache-2.0 OR MIT,Fraser Hutchison +hash_hasher,https://github.com/Fraser999/Hash-Hasher,Apache-2.0 OR MIT,Fraser Hutchison hashbag,https://github.com/jonhoo/hashbag,MIT OR Apache-2.0,Jon Gjengset hashbrown,https://github.com/rust-lang/hashbrown,MIT OR Apache-2.0,Amanieu d'Antras hashlink,https://github.com/kyren/hashlink,MIT OR Apache-2.0,kyren @@ -389,8 +389,8 @@ minimal-lexical,https://github.com/Alexhuszagh/minimal-lexical,MIT OR Apache-2.0 miniz_oxide,https://github.com/Frommi/miniz_oxide/tree/master/miniz_oxide,MIT OR Zlib OR Apache-2.0,"Frommi , oyvindln " miniz_oxide,https://github.com/Frommi/miniz_oxide/tree/master/miniz_oxide,MIT OR Zlib OR Apache-2.0,"Frommi , oyvindln , Rich Geldreich richgel99@gmail.com" mio,https://github.com/tokio-rs/mio,MIT,"Carl Lerche , Thomas de Zeeuw , Tokio Contributors " -mlua,https://github.com/khvzak/mlua,MIT,"Aleksandr Orlenko , kyren " -mlua-sys,https://github.com/khvzak/mlua,MIT,Aleksandr Orlenko +mlua,https://github.com/mlua-rs/mlua,MIT,"Aleksandr Orlenko , kyren " +mlua-sys,https://github.com/mlua-rs/mlua,MIT,Aleksandr Orlenko mlua_derive,https://github.com/khvzak/mlua,MIT,Aleksandr Orlenko moka,https://github.com/moka-rs/moka,MIT OR Apache-2.0,The moka Authors mongodb,https://github.com/mongodb/mongo-rust-driver,Apache-2.0,"Saghm Rossi , Patrick Freed , Isabel Atkinson , Abraham Egnor , Kaitlin Mahar " @@ -655,8 +655,10 @@ tokio-postgres,https://github.com/sfackler/rust-postgres,MIT OR Apache-2.0,Steve tokio-retry,https://github.com/srijs/rust-tokio-retry,MIT,Sam Rijs tokio-rustls,https://github.com/rustls/tokio-rustls,MIT OR Apache-2.0,The tokio-rustls Authors tokio-tungstenite,https://github.com/snapview/tokio-tungstenite,MIT,"Daniel Abramov , Alexey Galakhov " -toml,https://github.com/toml-rs/toml,MIT OR Apache-2.0,Alex Crichton +toml,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml Authors +toml_datetime,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_datetime Authors toml_edit,https://github.com/toml-rs/toml,MIT OR Apache-2.0,"Andronik Ordian , Ed Page " +toml_edit,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_edit Authors toml_write,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_write Authors tonic,https://github.com/hyperium/tonic,MIT,Lucio Franco tower,https://github.com/tower-rs/tower,MIT,Tower Maintainers diff --git a/lib/dnsmsg-parser/src/dns_message_parser.rs b/lib/dnsmsg-parser/src/dns_message_parser.rs index e80c380324fe1..5da9fc5fcb78d 100644 --- a/lib/dnsmsg-parser/src/dns_message_parser.rs +++ b/lib/dnsmsg-parser/src/dns_message_parser.rs @@ -5,11 +5,11 @@ use std::str::Utf8Error; use data_encoding::{BASE32HEX_NOPAD, BASE64, HEXUPPER}; use hickory_proto::dnssec::rdata::{DNSSECRData, CDNSKEY, CDS, DNSKEY, DS}; use hickory_proto::dnssec::SupportedAlgorithms; +use hickory_proto::rr::rdata::caa::Property; use hickory_proto::{ op::{message::Message as TrustDnsMessage, Query}, rr::{ rdata::{ - caa::Value, opt::{EdnsCode, EdnsOption}, A, AAAA, NULL, OPT, SVCB, }, @@ -633,11 +633,19 @@ impl DnsMessageParser { "{} {} \"{}\"", caa.issuer_critical() as u8, caa.tag().as_str(), - match caa.value() { - Value::Url(url) => { + match caa.tag() { + Property::Iodef => { + let url = caa.value_as_iodef().map_err(|source| { + DnsMessageParserError::TrustDnsError { source } + })?; url.as_str().to_string() } - Value::Issuer(option_name, vec_keyvalue) => { + Property::Issue | Property::IssueWild => { + let (option_name, vec_keyvalue) = + caa.value_as_issue().map_err(|source| { + DnsMessageParserError::TrustDnsError { source } + })?; + let mut final_issuer = String::new(); if let Some(name) = option_name { final_issuer.push_str(&name.to_string_with_options(&self.options)); @@ -650,9 +658,14 @@ impl DnsMessageParser { } final_issuer.trim_end().to_string() } - Value::Unknown(unknown) => std::str::from_utf8(unknown) - .map_err(|source| DnsMessageParserError::Utf8ParsingError { source })? - .to_string(), + Property::Unknown(_) => { + let unknown = caa.raw_value(); + std::str::from_utf8(unknown) + .map_err(|source| DnsMessageParserError::Utf8ParsingError { + source, + })? + .to_string() + } } ); Ok((Some(caa_rdata), None)) diff --git a/lib/file-source/Cargo.toml b/lib/file-source/Cargo.toml index b61e0b036d3a4..c5789c43e17d7 100644 --- a/lib/file-source/Cargo.toml +++ b/lib/file-source/Cargo.toml @@ -43,7 +43,7 @@ default-features = false features = ["serde"] [dependencies.flate2] -version = "1.0" +version = "1.1" default-features = false features = ["rust_backend"] diff --git a/lib/vector-buffers/Cargo.toml b/lib/vector-buffers/Cargo.toml index 6d2f40cf91b3a..dfe21bae73628 100644 --- a/lib/vector-buffers/Cargo.toml +++ b/lib/vector-buffers/Cargo.toml @@ -44,7 +44,7 @@ proptest = "1.6" quickcheck = "1.0" rand.workspace = true serde_yaml = { version = "0.9", default-features = false } -temp-dir = "0.1.14" +temp-dir = "0.1.16" tokio-test = "0.4.4" tracing-fluent-assertions = { version = "0.3" } tracing-subscriber = { version = "0.3.19", default-features = false, features = ["env-filter", "fmt", "registry", "std", "ansi"] } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 2c805ae62ae3d..e7df732eb0bed 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -17,7 +17,7 @@ chrono-tz.workspace = true crossbeam-utils = { version = "0.8.21", default-features = false } derivative = { version = "2.2.0", default-features = false } dyn-clone = { version = "1.0.19", default-features = false } -enumflags2 = { version = "0.7.11", default-features = false } +enumflags2 = { version = "0.7.12", default-features = false } float_eq = { version = "1.0", default-features = false } futures.workspace = true futures-util = { version = "0.3.29", default-features = false, features = ["std"] } @@ -31,16 +31,16 @@ lookup = { package = "vector-lookup", path = "../vector-lookup" } metrics.workspace = true metrics-tracing-context.workspace = true metrics-util.workspace = true -mlua = { version = "0.10.3", default-features = false, features = ["lua54", "send", "vendored"], optional = true } +mlua = { version = "0.10.5", default-features = false, features = ["lua54", "send", "vendored"], optional = true } no-proxy = { version = "0.3.6", default-features = false, features = ["serialize"] } ordered-float = { version = "4.6.0", default-features = false } -openssl = { version = "0.10.72", default-features = false, features = ["vendored"] } -parking_lot = { version = "0.12.3", default-features = false } +openssl = { version = "0.10.73", default-features = false, features = ["vendored"] } +parking_lot = { version = "0.12.4", default-features = false } pin-project.workspace = true proptest = { version = "1.6", optional = true } prost-types.workspace = true prost .workspace = true -quanta = { version = "0.12.5", default-features = false } +quanta = { version = "0.12.6", default-features = false } regex = { version = "1.11.1", default-features = false, features = ["std", "perf"] } ryu = { version = "1", default-features = false } serde.workspace = true @@ -48,7 +48,7 @@ serde_json.workspace = true serde_with = { version = "3.12.0", default-features = false, features = ["std", "macros"] } smallvec = { version = "1", default-features = false, features = ["serde", "const_generics"] } snafu.workspace = true -socket2 = { version = "0.5.9", default-features = false } +socket2 = { version = "0.5.10", default-features = false } tokio = { version = "1.45.1", default-features = false, features = ["net"] } tokio-openssl = { version = "0.6.5", default-features = false } tokio-stream = { version = "0.1", default-features = false, features = ["time"], optional = true } diff --git a/lib/vector-stream/Cargo.toml b/lib/vector-stream/Cargo.toml index 87908f771ad13..d77ef065f1b73 100644 --- a/lib/vector-stream/Cargo.toml +++ b/lib/vector-stream/Cargo.toml @@ -14,7 +14,7 @@ tokio = { version = "1.45.1", default-features = false, features = ["net"] } tokio-util = { version = "0.7.0", default-features = false, features = ["time"] } tower = { version = "0.4", default-features = false, features = ["util"] } tracing = { version = "0.1.34", default-features = false } -twox-hash = { version = "2.1.0", default-features = false, features = ["xxhash64"] } +twox-hash = { version = "2.1.1", default-features = false, features = ["xxhash64"] } vector-common = { path = "../vector-common" } vector-core = { path = "../vector-core" } diff --git a/lib/vector-tap/Cargo.toml b/lib/vector-tap/Cargo.toml index 154e7a3f08e33..7b1640748f49e 100644 --- a/lib/vector-tap/Cargo.toml +++ b/lib/vector-tap/Cargo.toml @@ -10,7 +10,7 @@ license = "MPL-2.0" api = ["dep:async-graphql"] [dependencies] -async-graphql = { version = "7.0.16", default-features = false, features = ["playground"], optional = true} +async-graphql = { version = "7.0.17", default-features = false, features = ["playground"], optional = true} colored = { version = "3.0.0", default-features = false } futures.workspace = true glob.workspace = true diff --git a/src/config/diff.rs b/src/config/diff.rs index cce523cfb85ba..fa5f8cf09b8ff 100644 --- a/src/config/diff.rs +++ b/src/config/diff.rs @@ -35,7 +35,7 @@ impl ConfigDiff { } /// Swaps removed with added in Differences. - pub fn flip(mut self) -> Self { + pub const fn flip(mut self) -> Self { self.sources.flip(); self.transforms.flip(); self.sinks.flip(); @@ -189,7 +189,7 @@ impl Difference { self.to_remove.contains(key) } - fn flip(&mut self) { + const fn flip(&mut self) { std::mem::swap(&mut self.to_remove, &mut self.to_add); } diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index b35fe0fefce54..096ae69c0488b 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -11,8 +11,8 @@ publish = false anyhow.workspace = true chrono.workspace = true clap.workspace = true -clap-verbosity-flag = "3.0.2" -clap_complete = "4.5.48" +clap-verbosity-flag = "3.0.3" +clap_complete = "4.5.54" confy = "1.0.0" directories = "6.0.0" # remove this when stabilized https://doc.rust-lang.org/stable/std/path/fn.absolute.html @@ -24,7 +24,7 @@ indicatif = { version = "0.17.11", features = ["improved_unicode"] } itertools = "0.14.0" log = "0.4.27" # watch https://github.com/epage/anstyle for official interop with Clap -owo-colors = { version = "4.2.0", features = ["supports-colors"] } +owo-colors = { version = "4.2.1", features = ["supports-colors"] } paste.workspace = true regex = { version = "1.11.1", default-features = false, features = ["std", "perf"] } reqwest = { version = "0.11", features = ["json", "blocking"] } From 7082f2e5f0b809f8625ee5093247bd7ac5cadcec Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 10 Jun 2025 16:22:29 -0400 Subject: [PATCH 048/305] chore(internal docs): Fix spelling and add QA'd to allow.txt (#23182) Fix spelling and add QA'd to allow.txt --- .github/actions/spelling/allow.txt | 1 + website/content/en/docs/about/under-the-hood/guarantees.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index 341a2ed509dae..ac5a03921a55f 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -512,6 +512,7 @@ NRSC OPENPGP OPENSAFETY PROFIBUS +QA'd RFID riello ROHC diff --git a/website/content/en/docs/about/under-the-hood/guarantees.md b/website/content/en/docs/about/under-the-hood/guarantees.md index b5bc5e830b3b5..6a1c150ee074e 100644 --- a/website/content/en/docs/about/under-the-hood/guarantees.md +++ b/website/content/en/docs/about/under-the-hood/guarantees.md @@ -79,7 +79,7 @@ A **best-effort** delivery guarantee means that a Vector component makes a best each event but it can't _guarantee_ delivery. This is usually due to limitations of the underlying protocol, which is outside Vector's control. -Note that this is _not_ the same as at-most-once delivery, as it'is still possible for Vector to +Note that this is _not_ the same as at-most-once delivery, as it's still possible for Vector to introduce duplicates under extreme circumstances. ## Stability guarantees From 333cd765f0d8bd414413caac1b12b07960efece8 Mon Sep 17 00:00:00 2001 From: tot19 <31141271+tot19@users.noreply.github.com> Date: Wed, 11 Jun 2025 05:48:41 +0900 Subject: [PATCH 049/305] feat(splunk_hec source): Allow content-type header if it includes application/json (#23024) * #23022 Allow content-type header if it only includes application/json. Do not require exact match * Formatting * Ensuring response with invalid Content-Type header value will return same error response * Adding changelog * Adding newline * Fixed failing tests and added new unit test --- ...low_content_header_contains.enhancement.md | 3 + src/sources/splunk_hec/mod.rs | 104 ++++++++++++++++-- 2 files changed, 97 insertions(+), 10 deletions(-) create mode 100644 changelog.d/23022_hec_allow_content_header_contains.enhancement.md diff --git a/changelog.d/23022_hec_allow_content_header_contains.enhancement.md b/changelog.d/23022_hec_allow_content_header_contains.enhancement.md new file mode 100644 index 0000000000000..2cc0d2a827bfd --- /dev/null +++ b/changelog.d/23022_hec_allow_content_header_contains.enhancement.md @@ -0,0 +1,3 @@ +Updated the Splunk HEC source to accept requests that contain the header content-type with any value containing "application/json," not the exact value of "application/json." This matches the behavior of a true Splunk HEC. Allows sources from AWS to successfully send events to the Splunk HEC source without additional proxying to update headers. + +authors: Tot19 diff --git a/src/sources/splunk_hec/mod.rs b/src/sources/splunk_hec/mod.rs index 0be523bf8fe19..23b0f0e2f8aad 100644 --- a/src/sources/splunk_hec/mod.rs +++ b/src/sources/splunk_hec/mod.rs @@ -13,6 +13,7 @@ use flate2::read::MultiGzDecoder; use futures::FutureExt; use http::StatusCode; use hyper::{service::make_service_fn, Server}; +use serde::de::DeserializeOwned; use serde::Serialize; use serde_json::{ de::{Read as JsonRead, StrRead}, @@ -35,6 +36,7 @@ use vector_lib::{ use vector_lib::{configurable::configurable_component, tls::MaybeTlsIncomingStream}; use vrl::path::OwnedTargetPath; use vrl::value::{kind::Collection, Kind}; +use warp::http::header::{HeaderValue, CONTENT_TYPE}; use warp::{filters::BoxedFilter, path, reject::Rejection, reply::Response, Filter, Reply}; use self::{ @@ -529,26 +531,50 @@ impl SplunkSource { .boxed() } + fn lenient_json_content_type_check() -> impl Filter + Clone + where + T: Send + DeserializeOwned + 'static, + { + warp::header::optional::(CONTENT_TYPE.as_str()) + .and(warp::body::bytes()) + .and_then( + |ctype: Option, body: bytes::Bytes| async move { + let ok = ctype + .as_ref() + .and_then(|v| v.to_str().ok()) + .map(|h| h.to_ascii_lowercase().contains("application/json")) + .unwrap_or(true); + + if !ok { + return Err(warp::reject::custom(ApiError::UnsupportedContentType)); + } + + let value = serde_json::from_slice::(&body) + .map_err(|_| warp::reject::custom(ApiError::BadRequest))?; + + Ok(value) + }, + ) + } + fn ack_service(&self) -> BoxedFilter<(Response,)> { let idx_ack = self.idx_ack.clone(); + warp::post() - .and(path!("ack")) + .and(warp::path!("ack")) .and(self.authorization()) .and(SplunkSource::required_channel()) - .and(warp::body::json()) - .and_then(move |_, channel_id: String, body: HecAckStatusRequest| { + .and(Self::lenient_json_content_type_check::()) + .and_then(move |_, channel: String, req: HecAckStatusRequest| { let idx_ack = idx_ack.clone(); async move { if let Some(idx_ack) = idx_ack { - let ack_statuses = idx_ack - .get_acks_status_from_channel(channel_id, &body.acks) + let acks = idx_ack + .get_acks_status_from_channel(channel, &req.acks) .await?; - Ok( - warp::reply::json(&HecAckStatusResponse { acks: ack_statuses }) - .into_response(), - ) + Ok(warp::reply::json(&HecAckStatusResponse { acks }).into_response()) } else { - Err(Rejection::from(ApiError::AckIsDisabled)) + Err(warp::reject::custom(ApiError::AckIsDisabled)) } } }) @@ -1094,6 +1120,7 @@ pub(crate) enum ApiError { MissingAuthorization, InvalidAuthorization, UnsupportedEncoding, + UnsupportedContentType, MissingChannel, NoData, InvalidDataFormat { event: usize }, @@ -1188,6 +1215,14 @@ fn finish_ok(maybe_ack_id: Option) -> Response { response_json(StatusCode::OK, body) } +fn response_plain(code: StatusCode, msg: &'static str) -> Response { + warp::reply::with_status( + warp::reply::with_header(msg, http::header::CONTENT_TYPE, "text/plain; charset=utf-8"), + code, + ) + .into_response() +} + async fn finish_err(rejection: Rejection) -> Result<(Response,), Rejection> { if let Some(&error) = rejection.find::() { emit!(SplunkHecRequestError { error }); @@ -1200,6 +1235,10 @@ async fn finish_err(rejection: Rejection) -> Result<(Response,), Rejection> { splunk_response::INVALID_AUTHORIZATION, ), ApiError::UnsupportedEncoding => empty_response(StatusCode::UNSUPPORTED_MEDIA_TYPE), + ApiError::UnsupportedContentType => response_plain( + StatusCode::UNSUPPORTED_MEDIA_TYPE, + "The request's content-type is not supported", + ), ApiError::MissingChannel => { response_json(StatusCode::BAD_REQUEST, splunk_response::NO_CHANNEL) } @@ -2485,6 +2524,51 @@ mod tests { assert!(ack_res.acks.get(&event_res.ack_id).unwrap()); } + #[tokio::test] + async fn ack_service_accepts_parameterized_content_type() { + let ack_config = HecAcknowledgementsConfig { + enabled: Some(true), + ..Default::default() + }; + let (source, address) = source(Some(ack_config)).await; + let opts = SendWithOpts { + channel: Some(Channel::Header("guid")), + forwarded_for: None, + }; + + let event_res = send_with_response( + address, + "services/collector/event", + r#"{"event":"param-test"}"#, + TOKEN, + &opts, + ) + .await + .json::() + .await + .unwrap(); + let _ = collect_n(source, 1).await; + + let body = serde_json::to_string(&HecAckStatusRequest { + acks: vec![event_res.ack_id], + }) + .unwrap(); + + let res = reqwest::Client::new() + .post(format!("http://{}/services/collector/ack", address)) + .header("Authorization", format!("Splunk {}", TOKEN)) + .header("x-splunk-request-channel", "guid") + .header("Content-Type", "application/json; some-random-text; hello") + .body(body) + .send() + .await + .unwrap(); + + assert_eq!(200, res.status().as_u16()); + + let _parsed: HecAckStatusResponse = res.json().await.unwrap(); + } + #[tokio::test] async fn event_service_acknowledgements_enabled_channel_required() { let message = r#"{"event":"first", "color": "blue"}"#; From c03abd71ec406e4ec49b14d3461cc800f52f3286 Mon Sep 17 00:00:00 2001 From: Jorge Hermo Date: Wed, 11 Jun 2025 15:32:11 +0200 Subject: [PATCH 050/305] feat(sources): multicast udp socket support (#22099) * feat: multicast udp socket support * test: add tests for multicast udp * test: add tests for multicast udp * chore: add changelog * chore: remove dbg statement * chore: remove todo * chore: remove todo * chore: fix typo * docs: add docs for multicast_groups setting * docs: update website cue * fix: clippy lints * chore: fix typo * chore: fix typo * Update src/sources/socket/mod.rs Co-authored-by: Pavlos Rontidis * feat: change panic to unimplemented * chore: add todo about init_udp_with_config * chore: add todo about init_udp_with_config * feat: change "addr" to "to" in udp test helpers * feat: add new SocketMulticastGroupJoinError and refactor error test * feat: add variables to logs * feat: add variables to logs * feat: add INITIALIZING error stage * test: fix broken test on windows --------- Co-authored-by: Pavlos Rontidis --- ...32_multicast_udp_socket_sources.feature.md | 8 + .../src/internal_event/prelude.rs | 1 + lib/vector-config/src/stdlib.rs | 27 +++- src/internal_events/socket.rs | 44 +++++- src/sources/socket/mod.rs | 148 ++++++++++++++++-- src/sources/socket/udp.rs | 50 +++++- src/test_util/mod.rs | 4 + .../components/sources/base/socket.cue | 21 +++ 8 files changed, 289 insertions(+), 14 deletions(-) create mode 100644 changelog.d/5732_multicast_udp_socket_sources.feature.md diff --git a/changelog.d/5732_multicast_udp_socket_sources.feature.md b/changelog.d/5732_multicast_udp_socket_sources.feature.md new file mode 100644 index 0000000000000..f85c4451b53d0 --- /dev/null +++ b/changelog.d/5732_multicast_udp_socket_sources.feature.md @@ -0,0 +1,8 @@ +The `socket` source with `udp` mode now supports joining multicast groups via the `multicast_groups` option +of that source. This allows the source to receive multicast packets from the specified multicast groups. + +Note that in order to work properly, the `socket` address must be set to `0.0.0.0` and not +to `127.0.0.1` (localhost) or any other specific IP address. If other IP address is used, the host's interface +will filter out the multicast packets as the packet target IP (multicast) would not match the host's interface IP. + +authors: jorgehermo9 diff --git a/lib/vector-common/src/internal_event/prelude.rs b/lib/vector-common/src/internal_event/prelude.rs index e15a6b52b4503..92aa160fe509e 100644 --- a/lib/vector-common/src/internal_event/prelude.rs +++ b/lib/vector-common/src/internal_event/prelude.rs @@ -1,5 +1,6 @@ // Set of `stage` tags to use when emitting error events. pub mod error_stage { + pub const INITIALIZING: &str = "initializing"; pub const RECEIVING: &str = "receiving"; pub const PROCESSING: &str = "processing"; pub const SENDING: &str = "sending"; diff --git a/lib/vector-config/src/stdlib.rs b/lib/vector-config/src/stdlib.rs index ad3ae47aac32d..add05f87fd9ea 100644 --- a/lib/vector-config/src/stdlib.rs +++ b/lib/vector-config/src/stdlib.rs @@ -2,7 +2,7 @@ use std::{ cell::RefCell, collections::{BTreeMap, BTreeSet, HashMap, HashSet}, hash::Hash, - net::SocketAddr, + net::{Ipv4Addr, SocketAddr}, num::{ NonZeroI16, NonZeroI32, NonZeroI64, NonZeroI8, NonZeroU16, NonZeroU32, NonZeroU64, NonZeroU8, NonZeroUsize, @@ -402,6 +402,31 @@ impl ToValue for SocketAddr { } } +impl Configurable for Ipv4Addr { + fn referenceable_name() -> Option<&'static str> { + Some("stdlib::Ipv4Addr") + } + + fn metadata() -> Metadata { + let mut metadata = Metadata::default(); + metadata.set_description("An IPv4 address."); + metadata + } + + fn generate_schema(_: &RefCell) -> Result { + // TODO: We don't need anything other than a string schema to (de)serialize a `Ipv4Addr`, + // but we eventually should have validation since the format for the possible permutations + // is well-known and can be easily codified. + Ok(generate_string_schema()) + } +} + +impl ToValue for Ipv4Addr { + fn to_value(&self) -> Value { + Value::String(self.to_string()) + } +} + impl Configurable for PathBuf { fn referenceable_name() -> Option<&'static str> { Some("stdlib::PathBuf") diff --git a/src/internal_events/socket.rs b/src/internal_events/socket.rs index 97a648da2587b..5f2f7aac301e4 100644 --- a/src/internal_events/socket.rs +++ b/src/internal_events/socket.rs @@ -1,3 +1,5 @@ +use std::net::Ipv4Addr; + use metrics::{counter, histogram}; use vector_lib::internal_event::{ComponentEventsDropped, InternalEvent, UNINTENTIONAL}; use vector_lib::{ @@ -120,7 +122,7 @@ impl InternalEvent for SocketBindError { error = %self.error, error_code = "socket_bind", error_type = error_type::IO_FAILED, - stage = error_stage::RECEIVING, + stage = error_stage::INITIALIZING, %mode, internal_log_rate_limit = true, ); @@ -128,8 +130,46 @@ impl InternalEvent for SocketBindError { "component_errors_total", "error_code" => "socket_bind", "error_type" => error_type::IO_FAILED, - "stage" => error_stage::RECEIVING, + "stage" => error_stage::INITIALIZING, + "mode" => mode, + ) + .increment(1); + } +} + +#[derive(Debug)] +pub struct SocketMulticastGroupJoinError { + pub error: E, + pub group_addr: Ipv4Addr, + pub interface: Ipv4Addr, +} + +impl InternalEvent for SocketMulticastGroupJoinError { + fn emit(self) { + // Multicast groups are only used in UDP mode + let mode = SocketMode::Udp.as_str(); + let group_addr = self.group_addr.to_string(); + let interface = self.interface.to_string(); + + error!( + message = "Error joining multicast group.", + error = %self.error, + error_code = "socket_multicast_group_join", + error_type = error_type::IO_FAILED, + stage = error_stage::INITIALIZING, + %mode, + %group_addr, + %interface, + internal_log_rate_limit = true, + ); + counter!( + "component_errors_total", + "error_code" => "socket_multicast_group_join", + "error_type" => error_type::IO_FAILED, + "stage" => error_stage::INITIALIZING, "mode" => mode, + "group_addr" => group_addr, + "interface" => interface, ) .increment(1); } diff --git a/src/sources/socket/mod.rs b/src/sources/socket/mod.rs index df8eb08a2f915..13660f33693ce 100644 --- a/src/sources/socket/mod.rs +++ b/src/sources/socket/mod.rs @@ -319,7 +319,7 @@ mod test { use approx::assert_relative_eq; use std::{ collections::HashMap, - net::{SocketAddr, UdpSocket}, + net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}, sync::{ atomic::{AtomicBool, Ordering}, Arc, @@ -373,8 +373,11 @@ mod test { sources::util::net::SocketListenAddr, test_util::{ collect_n, collect_n_limited, - components::{assert_source_compliance, SOCKET_PUSH_SOURCE_TAGS}, - next_addr, random_string, send_lines, send_lines_tls, wait_for_tcp, + components::{ + assert_source_compliance, assert_source_error, COMPONENT_ERROR_TAGS, + SOCKET_PUSH_SOURCE_TAGS, + }, + next_addr, next_addr_any, random_string, send_lines, send_lines_tls, wait_for_tcp, }, tls::{self, TlsConfig, TlsEnableableConfig, TlsSourceConfig}, SourceSender, @@ -898,13 +901,28 @@ mod test { } //////// UDP TESTS //////// - fn send_lines_udp(addr: SocketAddr, lines: impl IntoIterator) -> SocketAddr { - send_packets_udp(addr, lines.into_iter().map(|line| line.into())) + fn send_lines_udp(to: SocketAddr, lines: impl IntoIterator) -> SocketAddr { + send_lines_udp_from(next_addr(), to, lines) + } + + fn send_lines_udp_from( + from: SocketAddr, + to: SocketAddr, + lines: impl IntoIterator, + ) -> SocketAddr { + send_packets_udp_from(from, to, lines.into_iter().map(|line| line.into())) } - fn send_packets_udp(addr: SocketAddr, packets: impl IntoIterator) -> SocketAddr { - let bind = next_addr(); - let socket = UdpSocket::bind(bind) + fn send_packets_udp(to: SocketAddr, packets: impl IntoIterator) -> SocketAddr { + send_packets_udp_from(next_addr(), to, packets) + } + + fn send_packets_udp_from( + from: SocketAddr, + to: SocketAddr, + packets: impl IntoIterator, + ) -> SocketAddr { + let socket = UdpSocket::bind(from) .map_err(|error| panic!("{:}", error)) .ok() .unwrap(); @@ -912,7 +930,7 @@ mod test { for packet in packets { assert_eq!( socket - .send_to(&packet, addr) + .send_to(&packet, to) .map_err(|error| panic!("{:}", error)) .ok() .unwrap(), @@ -926,7 +944,7 @@ mod test { thread::sleep(Duration::from_millis(10)); // Done - bind + from } async fn init_udp_with_shutdown( @@ -1304,6 +1322,116 @@ mod test { .await; } + #[tokio::test] + async fn multicast_udp_message() { + assert_source_compliance(&SOCKET_PUSH_SOURCE_TAGS, async { + let (tx, mut rx) = SourceSender::new_test(); + // The socket address must be `IPADDR_ANY` (0.0.0.0) in order to receive multicast packets + let socket_address = next_addr_any(); + let multicast_ip_address: Ipv4Addr = "224.0.0.2".parse().unwrap(); + let multicast_socket_address = + SocketAddr::new(IpAddr::V4(multicast_ip_address), socket_address.port()); + let mut config = UdpConfig::from_address(socket_address.into()); + config.multicast_groups = vec![multicast_ip_address]; + init_udp_with_config(tx, config).await; + + // We must send packets to the same interface the `socket_address` is bound to + // in order to receive the multicast packets the `from` socket sends. + // To do so, we use the `IPADDR_ANY` address + let from = next_addr_any(); + send_lines_udp_from(from, multicast_socket_address, ["test".to_string()]); + + let event = rx.next().await.expect("must receive an event"); + assert_eq!( + event.as_log()[log_schema().message_key().unwrap().to_string()], + "test".into() + ); + }) + .await; + } + + #[tokio::test] + async fn multiple_multicast_addresses_udp_message() { + assert_source_compliance(&SOCKET_PUSH_SOURCE_TAGS, async { + let (tx, mut rx) = SourceSender::new_test(); + let socket_address = next_addr_any(); + let multicast_ip_addresses = (2..12) + .map(|i| format!("224.0.0.{i}").parse().unwrap()) + .collect::>(); + let multicast_ip_socket_addresses = multicast_ip_addresses + .iter() + .map(|ip_address| SocketAddr::new(IpAddr::V4(*ip_address), socket_address.port())) + .collect::>(); + let mut config = UdpConfig::from_address(socket_address.into()); + config.multicast_groups = multicast_ip_addresses; + init_udp_with_config(tx, config).await; + + let from = next_addr_any(); + for multicast_ip_socket_address in multicast_ip_socket_addresses { + send_lines_udp_from( + from, + multicast_ip_socket_address, + [multicast_ip_socket_address.to_string()], + ); + + let event = rx.next().await.expect("must receive an event"); + assert_eq!( + event.as_log()[log_schema().message_key().unwrap().to_string()], + multicast_ip_socket_address.to_string().into() + ); + } + }) + .await; + } + + #[tokio::test] + async fn multicast_and_unicast_udp_message() { + assert_source_compliance(&SOCKET_PUSH_SOURCE_TAGS, async { + let (tx, mut rx) = SourceSender::new_test(); + let socket_address = next_addr_any(); + let multicast_ip_address: Ipv4Addr = "224.0.0.2".parse().unwrap(); + let multicast_socket_address = + SocketAddr::new(IpAddr::V4(multicast_ip_address), socket_address.port()); + let mut config = UdpConfig::from_address(socket_address.into()); + config.multicast_groups = vec![multicast_ip_address]; + init_udp_with_config(tx, config).await; + + let from = next_addr_any(); + // Send packet to multicast address + send_lines_udp_from(from, multicast_socket_address, ["test".to_string()]); + let event = rx.next().await.expect("must receive an event"); + assert_eq!( + event.as_log()[log_schema().message_key().unwrap().to_string()], + "test".into() + ); + + // Windows does not support connecting to `0.0.0.0`, + // therefore we connect to `127.0.0.1` instead (the socket is listening at `0.0.0.0`) + let to = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), socket_address.port()); + // Send packet to unicast address + send_lines_udp_from(from, to, ["test".to_string()]); + let event = rx.next().await.expect("must receive an event"); + assert_eq!( + event.as_log()[log_schema().message_key().unwrap().to_string()], + "test".into() + ); + }) + .await; + } + + #[tokio::test] + async fn udp_invalid_multicast_group() { + assert_source_error(&COMPONENT_ERROR_TAGS, async { + let (tx, _rx) = SourceSender::new_test(); + let socket_address = next_addr_any(); + let invalid_multicast_ip_address: Ipv4Addr = "192.168.0.3".parse().unwrap(); + let mut config = UdpConfig::from_address(socket_address.into()); + config.multicast_groups = vec![invalid_multicast_ip_address]; + init_udp_with_config(tx, config).await; + }) + .await; + } + ////////////// UNIX TEST LIBS ////////////// #[cfg(unix)] diff --git a/src/sources/socket/udp.rs b/src/sources/socket/udp.rs index a5d57207ea0a0..4cfa052c85d7e 100644 --- a/src/sources/socket/udp.rs +++ b/src/sources/socket/udp.rs @@ -1,3 +1,5 @@ +use std::net::{Ipv4Addr, SocketAddr}; + use super::default_host_key; use bytes::BytesMut; use chrono::Utc; @@ -20,7 +22,8 @@ use crate::{ codecs::Decoder, event::Event, internal_events::{ - SocketBindError, SocketEventsReceived, SocketMode, SocketReceiveError, StreamClosedError, + SocketBindError, SocketEventsReceived, SocketMode, SocketMulticastGroupJoinError, + SocketReceiveError, StreamClosedError, }, net, serde::default_decoding, @@ -41,6 +44,21 @@ pub struct UdpConfig { #[configurable(derived)] address: SocketListenAddr, + /// List of IPv4 multicast groups to join on socket's binding process. + /// + /// In order to read multicast packets, this source's listening address should be set to `0.0.0.0`. + /// If any other address is used (such as `127.0.0.1` or an specific interface address), the + /// listening interface will filter out all multicast packets received, + /// as their target IP would be the one of the multicast group + /// and it will not match the socket's bound IP. + /// + /// Note that this setting will only work if the source's address + /// is an IPv4 address (IPv6 and systemd file descriptor as source's address are not supported + /// with multicast groups). + #[serde(default)] + #[configurable(metadata(docs::examples = "['224.0.0.2', '224.0.0.4']"))] + pub(super) multicast_groups: Vec, + /// The maximum buffer size of incoming messages. /// /// Messages larger than this are truncated. @@ -118,6 +136,7 @@ impl UdpConfig { pub fn from_address(address: SocketListenAddr) -> Self { Self { address, + multicast_groups: Vec::new(), max_length: default_max_length(), host_key: None, port_key: default_port_key(), @@ -152,6 +171,35 @@ pub(super) fn udp( }) })?; + if !config.multicast_groups.is_empty() { + socket.set_multicast_loop_v4(true).unwrap(); + let listen_addr = match config.address() { + SocketListenAddr::SocketAddr(SocketAddr::V4(addr)) => addr, + SocketListenAddr::SocketAddr(SocketAddr::V6(_)) => { + // We could support Ipv6 multicast with the + // https://doc.rust-lang.org/std/net/struct.UdpSocket.html#method.join_multicast_v6 method + // and specifying the interface index as `0`, in order to bind all interfaces. + unimplemented!("IPv6 multicast is not supported") + } + SocketListenAddr::SystemdFd(_) => { + unimplemented!("Multicast for systemd fd sockets is not supported") + } + }; + for group_addr in config.multicast_groups { + let interface = *listen_addr.ip(); + socket + .join_multicast_v4(group_addr, interface) + .map_err(|error| { + emit!(SocketMulticastGroupJoinError { + error, + group_addr, + interface, + }) + })?; + info!(message = "Joined multicast group.", group = %group_addr); + } + } + if let Some(receive_buffer_bytes) = config.receive_buffer_bytes { if let Err(error) = net::set_receive_buffer_size(&socket, receive_buffer_bytes) { warn!(message = "Failed configuring receive buffer size on UDP socket.", %error); diff --git a/src/test_util/mod.rs b/src/test_util/mod.rs index e446b5455d9d4..93162c2e23bc4 100644 --- a/src/test_util/mod.rs +++ b/src/test_util/mod.rs @@ -124,6 +124,10 @@ pub fn next_addr() -> SocketAddr { next_addr_for_ip(IpAddr::V4(Ipv4Addr::LOCALHOST)) } +pub fn next_addr_any() -> SocketAddr { + next_addr_for_ip(IpAddr::V4(Ipv4Addr::UNSPECIFIED)) +} + pub fn next_addr_v6() -> SocketAddr { next_addr_for_ip(IpAddr::V6(Ipv6Addr::LOCALHOST)) } diff --git a/website/cue/reference/components/sources/base/socket.cue b/website/cue/reference/components/sources/base/socket.cue index f6df0070b2dd1..39fec3ee13018 100644 --- a/website/cue/reference/components/sources/base/socket.cue +++ b/website/cue/reference/components/sources/base/socket.cue @@ -498,6 +498,27 @@ base: components: sources: socket: configuration: { unix_stream: "Listen on a Unix domain socket (UDS), in stream mode." } } + multicast_groups: { + description: """ + List of IPv4 multicast groups to join on socket's binding process. + + In order to read multicast packets, this source's listening address should be set to `0.0.0.0`. + If any other address is used (such as `127.0.0.1` or an specific interface address), the + listening interface will filter out all multicast packets received, + as their target IP would be the one of the multicast group + and it will not match the socket's bound IP. + + Note that this setting will only work if the source's address + is an IPv4 address (IPv6 and systemd file descriptor as source's address are not supported + with multicast groups). + """ + relevant_when: "mode = \"udp\"" + required: false + type: array: { + default: [] + items: type: string: examples: ["['224.0.0.2', '224.0.0.4']"] + } + } path: { description: """ The Unix socket path. From 25296d0d01c8e2e9eac87096b2bff5ea07094afe Mon Sep 17 00:00:00 2001 From: Burkov Egor Date: Wed, 11 Jun 2025 16:52:11 +0300 Subject: [PATCH 051/305] fix(parsing): Prometheus timestamp parse int overflow handle (#23077) * fix(parsing): Prometheus timestamp parse int overflow handle * fix typo * Change changelog * Update changelog.d/fix_prometheus_timestamp_parse.fix.md --------- Co-authored-by: Pavlos Rontidis --- .../fix_prometheus_timestamp_parse.fix.md | 3 ++ lib/prometheus-parser/src/line.rs | 33 ++++++++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 changelog.d/fix_prometheus_timestamp_parse.fix.md diff --git a/changelog.d/fix_prometheus_timestamp_parse.fix.md b/changelog.d/fix_prometheus_timestamp_parse.fix.md new file mode 100644 index 0000000000000..0ed27abdee620 --- /dev/null +++ b/changelog.d/fix_prometheus_timestamp_parse.fix.md @@ -0,0 +1,3 @@ +Fix bug allowing invalid Prometheus timestamps; now properly rejected during parsing. + +authors: wooffie diff --git a/lib/prometheus-parser/src/line.rs b/lib/prometheus-parser/src/line.rs index aeac1e81105c6..4a41db94ce6b5 100644 --- a/lib/prometheus-parser/src/line.rs +++ b/lib/prometheus-parser/src/line.rs @@ -6,7 +6,7 @@ use nom::{ branch::alt, bytes::complete::{is_not, tag, take_while, take_while1}, character::complete::{char, digit1}, - combinator::{map, opt, recognize, value}, + combinator::{map, map_res, opt, recognize, value}, error::ParseError, multi::fold_many0, number::complete::double, @@ -151,9 +151,16 @@ impl Metric { fn parse_timestamp(input: &str) -> IResult> { let input = trim_space(input); - opt(map(recognize(pair(opt(char('-')), digit1)), |s: &str| { - s.parse().unwrap() - }))(input) + opt(map_res( + recognize(pair(opt(char('-')), digit1)), + |s: &str| s.parse(), + ))(input) + .map_err(|_: NomError| { + ErrorKind::ParseTimestampError { + input: input.to_owned(), + } + .into() + }) } fn parse_name_value(input: &str) -> IResult<(String, String)> { @@ -667,6 +674,24 @@ mod test { assert_eq!(Metric::parse_timestamp(""), Ok(("", None))); assert_eq!(Metric::parse_timestamp("123"), Ok(("", Some(123)))); assert_eq!(Metric::parse_timestamp(" -23"), Ok(("", Some(-23)))); + // Edge cases + assert_eq!( + Metric::parse_timestamp("9223372036854775807"), + Ok(("", Some(9223372036854775807i64))) + ); + assert_eq!( + Metric::parse_timestamp("-9223372036854775808"), + Ok(("", Some(-9223372036854775808i64))) + ); + // overflow + assert_eq!( + Metric::parse_timestamp("9223372036854775809"), + Ok(("9223372036854775809", None)) + ); + assert_eq!( + Metric::parse_timestamp("-9223372036854775809"), + Ok(("-9223372036854775809", None)) + ); } #[test] From 60fdc8a1cf96963bc99d2b0148befb11a6d75f34 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 11 Jun 2025 11:19:31 -0400 Subject: [PATCH 052/305] fix(core): Add deny_unknown_fields to TlsEnableableConfig (#23187) * Add deny_unknown_fields to TlsEnableableConfig * Add changelog * Update changelog.d/23187_tls_enableable_config_deny_unknown_fields.fix.md Co-authored-by: Pavlos Rontidis * Update changelog.d/23187_tls_enableable_config_deny_unknown_fields.fix.md --------- Co-authored-by: Pavlos Rontidis --- .../23187_tls_enableable_config_deny_unknown_fields.fix.md | 3 +++ lib/vector-core/src/tls/settings.rs | 1 + 2 files changed, 4 insertions(+) create mode 100644 changelog.d/23187_tls_enableable_config_deny_unknown_fields.fix.md diff --git a/changelog.d/23187_tls_enableable_config_deny_unknown_fields.fix.md b/changelog.d/23187_tls_enableable_config_deny_unknown_fields.fix.md new file mode 100644 index 0000000000000..61b9c7bcae91c --- /dev/null +++ b/changelog.d/23187_tls_enableable_config_deny_unknown_fields.fix.md @@ -0,0 +1,3 @@ +Unknown fields in the `tls` config are now rejected so these fields now need to be removed for Vector to start successfully. + +authors: thomasqueirozb diff --git a/lib/vector-core/src/tls/settings.rs b/lib/vector-core/src/tls/settings.rs index 0fece816894a9..fbedb57fac35c 100644 --- a/lib/vector-core/src/tls/settings.rs +++ b/lib/vector-core/src/tls/settings.rs @@ -41,6 +41,7 @@ pub const TEST_PEM_CLIENT_KEY_PATH: &str = #[configurable_component] #[configurable(metadata(docs::advanced))] #[derive(Clone, Debug, Default)] +#[serde(deny_unknown_fields)] pub struct TlsEnableableConfig { /// Whether or not to require TLS for incoming or outgoing connections. /// From ce5e6d445254b5f284076dd70c1b2ced27825c90 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 11 Jun 2025 15:38:20 -0400 Subject: [PATCH 053/305] docs(internal docs): remove vector-test-harness from REVIEWING.md (#23192) chore(docs): remove vector-test-harness from REVIEWING.md --- docs/REVIEWING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/REVIEWING.md b/docs/REVIEWING.md index 9075b74556a89..d40d311c63a91 100644 --- a/docs/REVIEWING.md +++ b/docs/REVIEWING.md @@ -26,7 +26,7 @@ should be used for all pull requests: - [ ] Is backward compatibility broken? If so, can it be avoided or deprecated? (see [Backward compatibility](#backward-compatibility)) - [ ] Have dependencies changed? (see [Dependencies](#dependencies)) - [ ] Has the code been explicitly reviewed for security issues? Dependencies included. (see [Security](#security)) -- [ ] Is there a risk of performance regressions? If so, have run the [Vector test harness](https://github.com/vectordotdev/vector-test-harness)? (see [Performance Testing](#performance-testing)) +- [ ] Is there a risk of performance regressions? (see [Performance Testing](#performance-testing)) - [ ] Should documentation be adjusted to reflect any of these changes? (see [Documentation](#documentation)) For component changes, especially pull requests introducing new components, the From 5b8afb4d20c5678ff43e9eb91eaf7dbbb425c231 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Jun 2025 20:53:22 +0000 Subject: [PATCH 054/305] chore(deps): Bump tempfile from 3.19.1 to 3.20.0 (#23137) * chore(deps): Bump tempfile from 3.19.1 to 3.20.0 Bumps [tempfile](https://github.com/Stebalien/tempfile) from 3.19.1 to 3.20.0. - [Changelog](https://github.com/Stebalien/tempfile/blob/master/CHANGELOG.md) - [Commits](https://github.com/Stebalien/tempfile/compare/v3.19.1...v3.20.0) --- updated-dependencies: - dependency-name: tempfile dependency-version: 3.20.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Use keep instead of into_path * Use keep instead of into_path in multiple files --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Pavlos Rontidis Co-authored-by: Thomas Schneider --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- lib/k8s-test-framework/src/temp_file.rs | 2 +- src/convert_config.rs | 2 +- src/sinks/socket.rs | 2 +- src/sinks/util/unix.rs | 2 +- src/sources/file.rs | 2 +- src/sources/socket/mod.rs | 8 ++++---- src/sources/statsd/mod.rs | 2 +- src/sources/syslog.rs | 2 +- src/sources/util/framestream.rs | 2 +- 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ff1cdfe628db0..3f09488ccebad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10188,9 +10188,9 @@ checksum = "83176759e9416cf81ee66cb6508dbfe9c96f20b8b56265a39917551c23c70964" [[package]] name = "tempfile" -version = "3.19.1" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" dependencies = [ "fastrand 2.3.0", "getrandom 0.3.1", diff --git a/Cargo.toml b/Cargo.toml index 3864e4e9306f2..370158465977f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -164,7 +164,7 @@ semver = { version = "1.0.26", default-features = false, features = ["serde", "s serde_json = { version = "1.0.140", default-features = false, features = ["raw_value", "std"] } serde = { version = "1.0.219", default-features = false, features = ["alloc", "derive", "rc"] } snafu = { version = "0.7.5", default-features = false, features = ["futures", "std"] } -tempfile = "3.19.1" +tempfile = "3.20.0" tokio = { version = "1.45.1", default-features = false, features = ["full"] } toml = { version = "0.8.23", default-features = false, features = ["display", "parse"] } tonic = { version = "0.11", default-features = false, features = ["transport", "codegen", "prost", "tls", "tls-roots", "gzip"] } diff --git a/lib/k8s-test-framework/src/temp_file.rs b/lib/k8s-test-framework/src/temp_file.rs index d1535b6b5a658..567de01031037 100644 --- a/lib/k8s-test-framework/src/temp_file.rs +++ b/lib/k8s-test-framework/src/temp_file.rs @@ -9,7 +9,7 @@ pub struct TempFile { impl TempFile { pub fn new(file_name: &str, data: &str) -> std::io::Result { - let dir = tempdir()?.into_path(); + let dir = tempdir()?.keep(); let path = dir.join(file_name); std::fs::write(&path, data)?; Ok(Self { path }) diff --git a/src/convert_config.rs b/src/convert_config.rs index ec80424c12fd1..526838b06cd43 100644 --- a/src/convert_config.rs +++ b/src/convert_config.rs @@ -271,7 +271,7 @@ mod tests { let input_path = test_data_dir(); let output_dir = tempdir() .expect("Unable to create tempdir for config") - .into_path(); + .keep(); walk_dir_and_convert(&input_path, &output_dir, Format::Yaml).unwrap(); let mut count: usize = 0; diff --git a/src/sinks/socket.rs b/src/sinks/socket.rs index 4768eede706e6..805fa6354355c 100644 --- a/src/sinks/socket.rs +++ b/src/sinks/socket.rs @@ -634,6 +634,6 @@ mod test { #[cfg(unix)] fn temp_uds_path(name: &str) -> PathBuf { - tempfile::tempdir().unwrap().into_path().join(name) + tempfile::tempdir().unwrap().keep().join(name) } } diff --git a/src/sinks/util/unix.rs b/src/sinks/util/unix.rs index efe31ad9dd9e7..9dd6e6f671e49 100644 --- a/src/sinks/util/unix.rs +++ b/src/sinks/util/unix.rs @@ -315,7 +315,7 @@ mod tests { }; fn temp_uds_path(name: &str) -> PathBuf { - tempfile::tempdir().unwrap().into_path().join(name) + tempfile::tempdir().unwrap().keep().join(name) } #[tokio::test] diff --git a/src/sources/file.rs b/src/sources/file.rs index c5611caddaee6..b435d6afc8cbc 100644 --- a/src/sources/file.rs +++ b/src/sources/file.rs @@ -978,7 +978,7 @@ mod tests { let local_dir = tempdir().unwrap(); let mut config = Config::default(); - config.global.data_dir = global_dir.into_path().into(); + config.global.data_dir = global_dir.keep().into(); // local path given -- local should win let res = config diff --git a/src/sources/socket/mod.rs b/src/sources/socket/mod.rs index 13660f33693ce..af27117cfd1c1 100644 --- a/src/sources/socket/mod.rs +++ b/src/sources/socket/mod.rs @@ -1457,7 +1457,7 @@ mod test { config: Option, ) -> PathBuf { let mut config = config.unwrap_or_else(|| { - UnixConfig::new(tempfile::tempdir().unwrap().into_path().join("unix_test")) + UnixConfig::new(tempfile::tempdir().unwrap().keep().join("unix_test")) }); let in_path = config.path.clone(); @@ -1648,7 +1648,7 @@ mod test { async fn unix_datagram_chunked_gelf_messages() { assert_source_compliance(&SOCKET_PUSH_SOURCE_TAGS, async { let (tx, rx) = SourceSender::new_test(); - let in_path = tempfile::tempdir().unwrap().into_path().join("unix_test"); + let in_path = tempfile::tempdir().unwrap().keep().join("unix_test"); let mut config = UnixConfig::new(in_path.clone()); config.decoding = GelfDeserializerConfig::default().into(); let path = init_unix_with_config(tx, false, false, config).await; @@ -1749,7 +1749,7 @@ mod test { #[cfg(unix)] #[tokio::test] async fn unix_datagram_permissions() { - let in_path = tempfile::tempdir().unwrap().into_path().join("unix_test"); + let in_path = tempfile::tempdir().unwrap().keep().join("unix_test"); let (tx, _) = SourceSender::new_test(); let mut config = UnixConfig::new(in_path.clone()); @@ -1894,7 +1894,7 @@ mod test { #[cfg(unix)] #[tokio::test] async fn unix_stream_permissions() { - let in_path = tempfile::tempdir().unwrap().into_path().join("unix_test"); + let in_path = tempfile::tempdir().unwrap().keep().join("unix_test"); let (tx, _) = SourceSender::new_test(); let mut config = UnixConfig::new(in_path.clone()); diff --git a/src/sources/statsd/mod.rs b/src/sources/statsd/mod.rs index 2d64af174e851..b77ecaeca7a13 100644 --- a/src/sources/statsd/mod.rs +++ b/src/sources/statsd/mod.rs @@ -498,7 +498,7 @@ mod test { #[tokio::test] async fn test_statsd_unix() { assert_source_compliance(&SOCKET_PUSH_SOURCE_TAGS, async move { - let in_path = tempfile::tempdir().unwrap().into_path().join("unix_test"); + let in_path = tempfile::tempdir().unwrap().keep().join("unix_test"); let config = StatsdConfig::Unix(UnixConfig { path: in_path.clone(), sanitize: true, diff --git a/src/sources/syslog.rs b/src/sources/syslog.rs index 2892285fda65d..c2a9364173c4c 100644 --- a/src/sources/syslog.rs +++ b/src/sources/syslog.rs @@ -1195,7 +1195,7 @@ mod test { assert_source_compliance(&SOCKET_PUSH_SOURCE_TAGS, async { let num_messages: usize = 1; - let in_path = tempfile::tempdir().unwrap().into_path().join("stream_test"); + let in_path = tempfile::tempdir().unwrap().keep().join("stream_test"); // Create and spawn the source. let config = SyslogConfig::from_mode(Mode::Unix { diff --git a/src/sources/util/framestream.rs b/src/sources/util/framestream.rs index 404e56c306306..7a36196732ceb 100644 --- a/src/sources/util/framestream.rs +++ b/src/sources/util/framestream.rs @@ -948,7 +948,7 @@ mod test { pub fn new(content_type: String, multithreaded: bool, extra_routine: F) -> Self { Self { frame_handler: MockFrameHandler::new(content_type, multithreaded, extra_routine), - socket_path: tempfile::tempdir().unwrap().into_path().join("unix_test"), + socket_path: tempfile::tempdir().unwrap().keep().join("unix_test"), socket_file_mode: None, socket_receive_buffer_size: None, socket_send_buffer_size: None, From bf9f3444833aa8334125632bc0ba92dcaebb47d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Jun 2025 09:34:33 -0400 Subject: [PATCH 055/305] chore(deps): Bump snafu from 0.7.5 to 0.8.0 (#19481) Bumps [snafu](https://github.com/shepmaster/snafu) from 0.7.5 to 0.8.0. - [Changelog](https://github.com/shepmaster/snafu/blob/main/CHANGELOG.md) - [Commits](https://github.com/shepmaster/snafu/compare/0.7.5...0.8.0) --- updated-dependencies: - dependency-name: snafu dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 22 +++++++++++----------- Cargo.toml | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f09488ccebad..9698a0f4e00fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2287,7 +2287,7 @@ dependencies = [ "serde_with 3.12.0", "similar-asserts", "smallvec", - "snafu 0.7.5", + "snafu 0.8.0", "syslog_loose", "tokio", "tokio-util", @@ -3158,7 +3158,7 @@ dependencies = [ "criterion", "data-encoding", "hickory-proto", - "snafu 0.7.5", + "snafu 0.8.0", ] [[package]] @@ -3175,7 +3175,7 @@ dependencies = [ "paste", "prost 0.12.6", "prost-build 0.12.6", - "snafu 0.7.5", + "snafu 0.8.0", "tracing 0.1.41", "vector-lib", "vrl", @@ -3194,7 +3194,7 @@ dependencies = [ "anyhow", "serde", "serde_json", - "snafu 0.7.5", + "snafu 0.8.0", "tracing 0.1.41", "vector-config", "vector-config-common", @@ -7665,7 +7665,7 @@ dependencies = [ "prost 0.12.6", "prost-build 0.12.6", "prost-types 0.12.6", - "snafu 0.7.5", + "snafu 0.8.0", "vector-common", ] @@ -9605,8 +9605,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" dependencies = [ "doc-comment", - "futures-core", - "pin-project", "snafu-derive 0.7.5", ] @@ -9616,6 +9614,8 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d342c51730e54029130d7dc9fd735d28c4cd360f1368c01981d4f03ff207f096" dependencies = [ + "futures-core", + "pin-project", "snafu-derive 0.8.0", ] @@ -11578,7 +11578,7 @@ dependencies = [ "similar-asserts", "smallvec", "smpl_jwt", - "snafu 0.7.5", + "snafu 0.8.0", "snap", "socket2 0.5.10", "sqlx", @@ -11674,7 +11674,7 @@ dependencies = [ "rkyv", "serde", "serde_yaml", - "snafu 0.7.5", + "snafu 0.8.0", "temp-dir", "tokio", "tokio-test", @@ -11726,7 +11726,7 @@ dependencies = [ "serde", "serde_json", "serde_with 3.12.0", - "snafu 0.7.5", + "snafu 0.8.0", "toml", "tracing 0.1.41", "url", @@ -11820,7 +11820,7 @@ dependencies = [ "serde_yaml", "similar-asserts", "smallvec", - "snafu 0.7.5", + "snafu 0.8.0", "socket2 0.5.10", "tokio", "tokio-openssl", diff --git a/Cargo.toml b/Cargo.toml index 370158465977f..e705a9f37d613 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -163,7 +163,7 @@ rand_distr = { version = "0.5.1", default-features = false } semver = { version = "1.0.26", default-features = false, features = ["serde", "std"] } serde_json = { version = "1.0.140", default-features = false, features = ["raw_value", "std"] } serde = { version = "1.0.219", default-features = false, features = ["alloc", "derive", "rc"] } -snafu = { version = "0.7.5", default-features = false, features = ["futures", "std"] } +snafu = { version = "0.8.0", default-features = false, features = ["futures", "std"] } tempfile = "3.20.0" tokio = { version = "1.45.1", default-features = false, features = ["full"] } toml = { version = "0.8.23", default-features = false, features = ["display", "parse"] } From 064efc13e37ae69500c2ae8c4cd5bb168605dc0f Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 12 Jun 2025 10:20:02 -0400 Subject: [PATCH 056/305] chore(internal docs): Add enableable to allow.txt (#23197) Add enableable to allow.txt --- .github/actions/spelling/allow.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index ac5a03921a55f..71ad2a86332d6 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -291,6 +291,7 @@ downsides downwardapi ede emoji +enableable esbuild etld fakeintake From dffcb5a02c881dd94a84470722ca4e6040a9cc43 Mon Sep 17 00:00:00 2001 From: Shivanth MP Date: Thu, 12 Jun 2025 18:29:08 +0200 Subject: [PATCH 057/305] feat(dev): Enable `internal_log_rate_limit` by default (#22899) * Set ratelimitng as default * Remove explicit internal_log_rate_limit flag * Add tests * Review comments * add internal ratelimit only if the field is present q --------- Co-authored-by: Pavlos Rontidis --- .../src/decoding/framing/chunked_gelf.rs | 2 - lib/dnstap-parser/src/internal_events.rs | 2 +- lib/tracing-limit/benches/limit.rs | 1 - lib/tracing-limit/examples/basic.rs | 6 +-- lib/tracing-limit/examples/by_span.rs | 1 - lib/tracing-limit/src/lib.rs | 45 +++++++++++++------ lib/vector-buffers/src/internal_events.rs | 2 +- .../component_events_dropped.rs | 2 - .../src/internal_event/service.rs | 3 +- src/app.rs | 5 +-- src/cli.rs | 1 + src/internal_events/amqp.rs | 6 +-- src/internal_events/apache_metrics.rs | 2 +- src/internal_events/aws_cloudwatch_logs.rs | 1 - src/internal_events/aws_ec2_metadata.rs | 2 +- src/internal_events/aws_ecs_metrics.rs | 4 +- src/internal_events/aws_kinesis.rs | 2 +- src/internal_events/aws_kinesis_firehose.rs | 3 +- src/internal_events/aws_sqs.rs | 10 ++--- src/internal_events/batch.rs | 2 +- src/internal_events/codecs.rs | 10 ++--- src/internal_events/common.rs | 5 +-- src/internal_events/conditions.rs | 1 - src/internal_events/datadog_agent.rs | 2 +- src/internal_events/datadog_metrics.rs | 1 - src/internal_events/datadog_traces.rs | 4 +- src/internal_events/dnstap.rs | 2 +- src/internal_events/docker_logs.rs | 2 +- src/internal_events/encoding_transcode.rs | 1 - src/internal_events/eventstoredb_metrics.rs | 4 +- src/internal_events/exec.rs | 9 ++-- src/internal_events/file.rs | 9 ++-- src/internal_events/fluent.rs | 2 +- src/internal_events/gcp_pubsub.rs | 6 +-- src/internal_events/host_metrics.rs | 5 +-- src/internal_events/http.rs | 2 +- src/internal_events/http_client.rs | 2 +- src/internal_events/http_client_source.rs | 4 +- src/internal_events/influxdb.rs | 1 - src/internal_events/journald.rs | 9 ++-- src/internal_events/kafka.rs | 5 +-- src/internal_events/kubernetes_logs.rs | 10 ++--- src/internal_events/logplex.rs | 2 +- src/internal_events/loki.rs | 4 -- src/internal_events/lua.rs | 3 +- src/internal_events/metric_to_log.rs | 1 - src/internal_events/mongodb_metrics.rs | 4 +- src/internal_events/mqtt.rs | 2 +- src/internal_events/nginx_metrics.rs | 4 +- src/internal_events/parser.rs | 1 - src/internal_events/postgresql_metrics.rs | 2 +- src/internal_events/process.rs | 3 -- src/internal_events/prometheus.rs | 5 +-- src/internal_events/pulsar.rs | 10 ++--- src/internal_events/redis.rs | 2 +- src/internal_events/reduce.rs | 2 +- src/internal_events/remap.rs | 2 +- src/internal_events/sematext_metrics.rs | 4 +- src/internal_events/socket.rs | 6 +-- src/internal_events/splunk_hec.rs | 10 ++--- src/internal_events/statsd_sink.rs | 2 +- src/internal_events/tag_cardinality_limit.rs | 2 - src/internal_events/tcp.rs | 4 -- src/internal_events/template.rs | 4 +- src/internal_events/udp.rs | 1 - src/internal_events/unix.rs | 7 ++- src/internal_events/websocket.rs | 4 +- src/internal_events/websocket_server.rs | 4 +- src/internal_events/windows.rs | 1 - src/sinks/datadog/logs/sink.rs | 1 - src/sinks/datadog/traces/apm_stats/flusher.rs | 1 - src/sinks/gcp/stackdriver/logs/encoder.rs | 1 - .../util/adaptive_concurrency/controller.rs | 1 - src/sinks/util/retries.rs | 4 +- src/sources/datadog_agent/metrics.rs | 5 +-- src/sources/heroku_logs.rs | 1 - src/sources/socket/udp.rs | 2 - src/transforms/lua/v1/mod.rs | 1 - 78 files changed, 135 insertions(+), 176 deletions(-) diff --git a/lib/codecs/src/decoding/framing/chunked_gelf.rs b/lib/codecs/src/decoding/framing/chunked_gelf.rs index b5a67b7aec5ba..286645cfa5698 100644 --- a/lib/codecs/src/decoding/framing/chunked_gelf.rs +++ b/lib/codecs/src/decoding/framing/chunked_gelf.rs @@ -387,7 +387,6 @@ impl ChunkedGelfDecoder { warn!( message_id = message_id, timeout_secs = timeout.as_secs_f64(), - internal_log_rate_limit = true, "Message was not fully received within the timeout window. Discarding it." ); } @@ -409,7 +408,6 @@ impl ChunkedGelfDecoder { debug!( message_id = message_id, sequence_number = sequence_number, - internal_log_rate_limit = true, "Received a duplicate chunk. Ignoring it." ); return Ok(None); diff --git a/lib/dnstap-parser/src/internal_events.rs b/lib/dnstap-parser/src/internal_events.rs index ab1635c9b3e03..2ce75acb19709 100644 --- a/lib/dnstap-parser/src/internal_events.rs +++ b/lib/dnstap-parser/src/internal_events.rs @@ -14,7 +14,7 @@ impl InternalEvent for DnstapParseWarning { error = %self.error, stage = error_stage::PROCESSING, error_type = error_type::PARSER_FAILED, - internal_log_rate_limit = true, + ); } } diff --git a/lib/tracing-limit/benches/limit.rs b/lib/tracing-limit/benches/limit.rs index f7214525e9819..86ffbae5bf206 100644 --- a/lib/tracing-limit/benches/limit.rs +++ b/lib/tracing-limit/benches/limit.rs @@ -59,7 +59,6 @@ fn bench(c: &mut Criterion) { bar = "bar", baz = 3, quuux = ?0.99, - internal_log_rate_limit = true ) } }) diff --git a/lib/tracing-limit/examples/basic.rs b/lib/tracing-limit/examples/basic.rs index 3b4428d540997..86c973739b13a 100644 --- a/lib/tracing-limit/examples/basic.rs +++ b/lib/tracing-limit/examples/basic.rs @@ -14,11 +14,7 @@ fn main() { tracing::dispatcher::with_default(&dispatch, || { for i in 0..40usize { trace!("This field is not rate limited!"); - info!( - message = "This message is rate limited", - count = &i, - internal_log_rate_limit = true, - ); + info!(message = "This message is rate limited", count = &i,); std::thread::sleep(std::time::Duration::from_millis(1000)); } }) diff --git a/lib/tracing-limit/examples/by_span.rs b/lib/tracing-limit/examples/by_span.rs index 63b8775d9905d..0c7e81666d279 100644 --- a/lib/tracing-limit/examples/by_span.rs +++ b/lib/tracing-limit/examples/by_span.rs @@ -28,7 +28,6 @@ fn main() { message = "This message is rate limited by its component and vrl_line_number", count = &i, - internal_log_rate_limit = true, ); } } diff --git a/lib/tracing-limit/src/lib.rs b/lib/tracing-limit/src/lib.rs index bd5952bf39760..dc131b3e31b39 100644 --- a/lib/tracing-limit/src/lib.rs +++ b/lib/tracing-limit/src/lib.rs @@ -129,7 +129,7 @@ where let mut limit_visitor = LimitVisitor::default(); event.record(&mut limit_visitor); - let limit_exists = limit_visitor.limit.unwrap_or(false); + let limit_exists = limit_visitor.limit.unwrap_or(true); if !limit_exists { return self.inner.on_event(event, ctx); } @@ -264,11 +264,8 @@ where let valueset = fields.value_set(&values); let event = Event::new(metadata, &valueset); self.inner.on_event(&event, ctx.clone()); - } else { - let values = [( - &fields.field(RATE_LIMIT_FIELD).unwrap(), - Some(&rate_limit as &dyn Value), - )]; + } else if let Some(ratelimit_field) = fields.field(RATE_LIMIT_FIELD) { + let values = [(&ratelimit_field, Some(&rate_limit as &dyn Value))]; let valueset = fields.value_set(&values); let event = Event::new(metadata, &valueset); @@ -525,6 +522,34 @@ mod test { ); } + #[test] + fn rate_limits_default() { + let events: Arc>> = Default::default(); + + let recorder = RecordingLayer::new(Arc::clone(&events)); + let sub = tracing_subscriber::registry::Registry::default() + .with(RateLimitedLayer::new(recorder).with_default_limit(10)); + tracing::subscriber::with_default(sub, || { + for _ in 0..21 { + info!(message = "Hello world!"); + MockClock::advance(Duration::from_millis(100)); + } + }); + + let events = events.lock().unwrap(); + + assert_eq!( + *events, + vec![ + "Hello world!", + "Internal log [Hello world!] is being suppressed to avoid flooding.", + ] + .into_iter() + .map(std::borrow::ToOwned::to_owned) + .collect::>() + ); + } + #[test] fn override_rate_limit_at_callsite() { let events: Arc>> = Default::default(); @@ -534,11 +559,7 @@ mod test { .with(RateLimitedLayer::new(recorder).with_default_limit(100)); tracing::subscriber::with_default(sub, || { for _ in 0..21 { - info!( - message = "Hello world!", - internal_log_rate_limit = true, - internal_log_rate_secs = 1 - ); + info!(message = "Hello world!", internal_log_rate_secs = 1); MockClock::advance(Duration::from_millis(100)); } }); @@ -579,7 +600,6 @@ mod test { info!( message = format!("Hello {} on line_number {}!", key, line_number).as_str(), - internal_log_rate_limit = true ); } } @@ -641,7 +661,6 @@ mod test { info!( message = format!("Hello {} on line_number {}!", key, line_number).as_str(), - internal_log_rate_limit = true, component_id = &key, vrl_position = &line_number ); diff --git a/lib/vector-buffers/src/internal_events.rs b/lib/vector-buffers/src/internal_events.rs index 0cda5beec46f9..055f09ebb6e62 100644 --- a/lib/vector-buffers/src/internal_events.rs +++ b/lib/vector-buffers/src/internal_events.rs @@ -115,7 +115,7 @@ impl InternalEvent for BufferReadError { error_code = self.error_code, error_type = error_type::READER_FAILED, stage = "processing", - internal_log_rate_limit = true, + ); counter!( "buffer_errors_total", "error_code" => self.error_code, diff --git a/lib/vector-common/src/internal_event/component_events_dropped.rs b/lib/vector-common/src/internal_event/component_events_dropped.rs index c7d2bd9eedd32..f8838911ff8a0 100644 --- a/lib/vector-common/src/internal_event/component_events_dropped.rs +++ b/lib/vector-common/src/internal_event/component_events_dropped.rs @@ -58,7 +58,6 @@ impl InternalEventHandle for DroppedHandle<'_, INTENDED> { intentional = INTENDED, count = data.0, reason = self.reason, - internal_log_rate_limit = true, ); } else { error!( @@ -66,7 +65,6 @@ impl InternalEventHandle for DroppedHandle<'_, INTENDED> { intentional = INTENDED, count = data.0, reason = self.reason, - internal_log_rate_limit = true, ); } self.discarded_events.increment(data.0 as u64); diff --git a/lib/vector-common/src/internal_event/service.rs b/lib/vector-common/src/internal_event/service.rs index 20132cc99fc61..d6c912f9d3fbc 100644 --- a/lib/vector-common/src/internal_event/service.rs +++ b/lib/vector-common/src/internal_event/service.rs @@ -14,7 +14,6 @@ impl InternalEvent for PollReadyError { error = ?self.error, error_type = error_type::REQUEST_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, ); counter!( "component_errors_total", @@ -45,7 +44,7 @@ impl InternalEvent for CallError { request_id = self.request_id, error_type = error_type::REQUEST_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/app.rs b/src/app.rs index 04023ebba4bab..2095ce57fbbb6 100644 --- a/src/app.rs +++ b/src/app.rs @@ -590,10 +590,7 @@ pub fn init_logging(color: bool, format: LogFormat, log_level: &str, rate: u64) }; trace::init(color, json, &level, rate); - debug!( - message = "Internal log rate limit configured.", - internal_log_rate_secs = rate, - ); + debug!(message = "Internal log rate limit configured.",); info!(message = "Log level is enabled.", level = ?level); } diff --git a/src/cli.rs b/src/cli.rs index 279560154585c..93e1dfea4d6bd 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -178,6 +178,7 @@ pub struct RootOpts { pub watch_config_poll_interval_seconds: NonZeroU64, /// Set the internal log rate limit + /// Note that traces are throttled by default unless tagged with `internal_log_rate_limit = false`. #[arg( short, long, diff --git a/src/internal_events/amqp.rs b/src/internal_events/amqp.rs index c66a3d9087016..677ec605ea234 100644 --- a/src/internal_events/amqp.rs +++ b/src/internal_events/amqp.rs @@ -36,7 +36,7 @@ pub mod source { error = ?self.error, error_type = error_type::REQUEST_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -58,7 +58,7 @@ pub mod source { error = ?self.error, error_type = error_type::ACKNOWLEDGMENT_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -80,7 +80,7 @@ pub mod source { error = ?self.error, error_type = error_type::COMMAND_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/apache_metrics.rs b/src/internal_events/apache_metrics.rs index 79ae7dced79b4..849741e44dd41 100644 --- a/src/internal_events/apache_metrics.rs +++ b/src/internal_events/apache_metrics.rs @@ -46,7 +46,7 @@ impl InternalEvent for ApacheMetricsParseError<'_> { stage = error_stage::PROCESSING, error_type = error_type::PARSER_FAILED, endpoint = %self.endpoint, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/aws_cloudwatch_logs.rs b/src/internal_events/aws_cloudwatch_logs.rs index d4bfe0328a53b..d8c4838dab7e1 100644 --- a/src/internal_events/aws_cloudwatch_logs.rs +++ b/src/internal_events/aws_cloudwatch_logs.rs @@ -18,7 +18,6 @@ impl InternalEvent for AwsCloudwatchLogsMessageSizeError { error_code = "message_too_long", error_type = error_type::ENCODER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, ); counter!( "component_errors_total", diff --git a/src/internal_events/aws_ec2_metadata.rs b/src/internal_events/aws_ec2_metadata.rs index 3e0996a3c9f60..3eafadf20222d 100644 --- a/src/internal_events/aws_ec2_metadata.rs +++ b/src/internal_events/aws_ec2_metadata.rs @@ -24,7 +24,7 @@ impl InternalEvent for AwsEc2MetadataRefreshError { error = %self.error, error_type = error_type::REQUEST_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/aws_ecs_metrics.rs b/src/internal_events/aws_ecs_metrics.rs index 1a2f7133094ce..969c5c33aa2e6 100644 --- a/src/internal_events/aws_ecs_metrics.rs +++ b/src/internal_events/aws_ecs_metrics.rs @@ -50,12 +50,12 @@ impl InternalEvent for AwsEcsMetricsParseError<'_> { error = ?self.error, stage = error_stage::PROCESSING, error_type = error_type::PARSER_FAILED, - internal_log_rate_limit = true, + ); debug!( message = %format!("Failed to parse response:\\n\\n{}\\n\\n", self.body.escape_debug()), endpoint = %self.endpoint, - internal_log_rate_limit = true, + ); counter!("parse_errors_total").increment(1); counter!( diff --git a/src/internal_events/aws_kinesis.rs b/src/internal_events/aws_kinesis.rs index 332f55d08b25e..fd5f73bb7e11f 100644 --- a/src/internal_events/aws_kinesis.rs +++ b/src/internal_events/aws_kinesis.rs @@ -17,7 +17,7 @@ impl InternalEvent for AwsKinesisStreamNoPartitionKeyError<'_> { partition_key_field = %self.partition_key_field, error_type = error_type::PARSER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( diff --git a/src/internal_events/aws_kinesis_firehose.rs b/src/internal_events/aws_kinesis_firehose.rs index d98d5ed5fa1c4..317435696b799 100644 --- a/src/internal_events/aws_kinesis_firehose.rs +++ b/src/internal_events/aws_kinesis_firehose.rs @@ -48,7 +48,6 @@ impl InternalEvent for AwsKinesisFirehoseRequestError<'_> { error_type = error_type::REQUEST_FAILED, error_code = %self.error_code, request_id = %self.request_id.unwrap_or(""), - internal_log_rate_limit = true, ); counter!( "component_errors_total", @@ -75,7 +74,7 @@ impl InternalEvent for AwsKinesisFirehoseAutomaticRecordDecodeError { error_type = error_type::PARSER_FAILED, error_code = %io_error_code(&self.error), compression = %self.compression, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/aws_sqs.rs b/src/internal_events/aws_sqs.rs index 490a65f004190..37f4ea98d25ec 100644 --- a/src/internal_events/aws_sqs.rs +++ b/src/internal_events/aws_sqs.rs @@ -30,7 +30,7 @@ mod s3 { error_code = "failed_processing_sqs_message", error_type = error_type::PARSER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -74,7 +74,7 @@ mod s3 { error_code = "failed_deleting_some_sqs_messages", error_type = error_type::ACKNOWLEDGMENT_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -104,7 +104,7 @@ mod s3 { error_code = "failed_deleting_all_sqs_messages", error_type = error_type::ACKNOWLEDGMENT_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -204,7 +204,7 @@ impl InternalEvent for SqsMessageReceiveError<'_, E> { error_code = "failed_fetching_sqs_events", error_type = error_type::REQUEST_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -257,7 +257,7 @@ impl InternalEvent for SqsMessageDeleteError<'_, E> { error = %self.error, error_type = error_type::WRITER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/batch.rs b/src/internal_events/batch.rs index 816afa4398d5f..5afc99d47f1bf 100644 --- a/src/internal_events/batch.rs +++ b/src/internal_events/batch.rs @@ -17,7 +17,7 @@ impl InternalEvent for LargeEventDroppedError { length = %self.length, error_type = error_type::CONDITION_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/codecs.rs b/src/internal_events/codecs.rs index 50251bf089cd3..e876db1dab5ab 100644 --- a/src/internal_events/codecs.rs +++ b/src/internal_events/codecs.rs @@ -15,7 +15,7 @@ impl InternalEvent for DecoderFramingError { error_code = "decoder_frame", error_type = error_type::PARSER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -40,7 +40,7 @@ impl InternalEvent for DecoderDeserializeError<'_> { error_code = "decoder_deserialize", error_type = error_type::PARSER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -66,7 +66,7 @@ impl InternalEvent for EncoderFramingError<'_> { error_code = "encoder_frame", error_type = error_type::ENCODER_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -93,7 +93,7 @@ impl InternalEvent for EncoderSerializeError<'_> { error_code = "encoder_serialize", error_type = error_type::ENCODER_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -120,7 +120,7 @@ impl InternalEvent for EncoderWriteError<'_, E> { error = %self.error, error_type = error_type::IO_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/common.rs b/src/internal_events/common.rs index 78971840d7dba..f342710daedb2 100644 --- a/src/internal_events/common.rs +++ b/src/internal_events/common.rs @@ -66,7 +66,7 @@ impl InternalEvent for SocketOutgoingConnectionError { error_code = "failed_connecting", error_type = error_type::CONNECTION_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -92,7 +92,6 @@ impl InternalEvent for StreamClosedError { error_code = STREAM_CLOSED, error_type = error_type::WRITER_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, ); counter!( "component_errors_total", @@ -137,7 +136,7 @@ impl InternalEvent for SinkRequestBuildError { error = %self.error, error_type = error_type::ENCODER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/conditions.rs b/src/internal_events/conditions.rs index bf825213c6719..0f7cb70962b08 100644 --- a/src/internal_events/conditions.rs +++ b/src/internal_events/conditions.rs @@ -14,7 +14,6 @@ impl InternalEvent for VrlConditionExecutionError<'_> { error = %self.error, error_type = error_type::SCRIPT_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, ); counter!( "component_errors_total", diff --git a/src/internal_events/datadog_agent.rs b/src/internal_events/datadog_agent.rs index 02f017bc076bb..686e3e0e3179c 100644 --- a/src/internal_events/datadog_agent.rs +++ b/src/internal_events/datadog_agent.rs @@ -15,7 +15,7 @@ impl InternalEvent for DatadogAgentJsonParseError<'_> { error = ?self.error, error_type = error_type::PARSER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/datadog_metrics.rs b/src/internal_events/datadog_metrics.rs index 8875b29281ac7..03812c2c81def 100644 --- a/src/internal_events/datadog_metrics.rs +++ b/src/internal_events/datadog_metrics.rs @@ -17,7 +17,6 @@ impl InternalEvent for DatadogMetricsEncodingError<'_> { error_type = error_type::ENCODER_FAILED, intentional = "false", stage = error_stage::PROCESSING, - internal_log_rate_limit = true, ); counter!( "component_errors_total", diff --git a/src/internal_events/datadog_traces.rs b/src/internal_events/datadog_traces.rs index dd1df213cdf06..aee946528391b 100644 --- a/src/internal_events/datadog_traces.rs +++ b/src/internal_events/datadog_traces.rs @@ -18,7 +18,7 @@ impl InternalEvent for DatadogTracesEncodingError { error_reason = %self.error_reason, error_type = error_type::ENCODER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -48,7 +48,7 @@ impl InternalEvent for DatadogTracesAPMStatsError { error = %self.error, error_type = error_type::WRITER_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/dnstap.rs b/src/internal_events/dnstap.rs index 231a6527830d7..adf9ea8688d6c 100644 --- a/src/internal_events/dnstap.rs +++ b/src/internal_events/dnstap.rs @@ -14,7 +14,7 @@ impl InternalEvent for DnstapParseError { error = %self.error, stage = error_stage::PROCESSING, error_type = error_type::PARSER_FAILED, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/docker_logs.rs b/src/internal_events/docker_logs.rs index 81950b1fa947a..f1daaa6c2d4f4 100644 --- a/src/internal_events/docker_logs.rs +++ b/src/internal_events/docker_logs.rs @@ -170,7 +170,7 @@ impl InternalEvent for DockerLogsLoggingDriverUnsupportedError<'_> { error_type = error_type::CONFIGURATION_FAILED, stage = error_stage::RECEIVING, container_id = ?self.container_id, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/encoding_transcode.rs b/src/internal_events/encoding_transcode.rs index 0a377f52ca1ee..869423fb98241 100644 --- a/src/internal_events/encoding_transcode.rs +++ b/src/internal_events/encoding_transcode.rs @@ -27,7 +27,6 @@ impl InternalEvent for DecoderMalformedReplacement { warn!( message = "Replaced malformed sequences with replacement character while decoding to utf8.", from_encoding = %self.from_encoding, - internal_log_rate_limit = true ); // NOT the actual number of replacements in the output: there's no easy // way to get that from the lib we use here (encoding_rs) diff --git a/src/internal_events/eventstoredb_metrics.rs b/src/internal_events/eventstoredb_metrics.rs index b5d7fadd088dc..7ee7f36cf9f50 100644 --- a/src/internal_events/eventstoredb_metrics.rs +++ b/src/internal_events/eventstoredb_metrics.rs @@ -14,7 +14,7 @@ impl InternalEvent for EventStoreDbMetricsHttpError { error = ?self.error, stage = error_stage::RECEIVING, error_type = error_type::REQUEST_FAILED, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -37,7 +37,7 @@ impl InternalEvent for EventStoreDbStatsParsingError { error = ?self.error, stage = error_stage::PROCESSING, error_type = error_type::PARSER_FAILED, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/exec.rs b/src/internal_events/exec.rs index 539471719d0e9..3a5ffe7c1c9ef 100644 --- a/src/internal_events/exec.rs +++ b/src/internal_events/exec.rs @@ -53,7 +53,7 @@ impl InternalEvent for ExecFailedError<'_> { error_type = error_type::COMMAND_FAILED, error_code = %io_error_code(&self.error), stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -82,7 +82,7 @@ impl InternalEvent for ExecTimeoutError<'_> { error = %self.error, error_type = error_type::TIMED_OUT, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -118,7 +118,7 @@ impl InternalEvent for ExecCommandExecuted<'_> { command = %self.command, exit_status = %exit_status, elapsed_millis = %self.exec_duration.as_millis(), - internal_log_rate_limit = true, + ); counter!( "command_executed_total", @@ -194,7 +194,7 @@ impl InternalEvent for ExecFailedToSignalChildError<'_> { error_code = %self.error.to_error_code(), error_type = error_type::COMMAND_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -216,7 +216,6 @@ impl InternalEvent for ExecChannelClosedError { message = exec_reason, error_type = error_type::COMMAND_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, ); counter!( "component_errors_total", diff --git a/src/internal_events/file.rs b/src/internal_events/file.rs index fba86d6ad1b72..84cf828c89d7a 100644 --- a/src/internal_events/file.rs +++ b/src/internal_events/file.rs @@ -83,7 +83,6 @@ impl InternalEvent for FileIoError<'_, P> { error_code = %self.code, error_type = error_type::IO_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, ); counter!( "component_errors_total", @@ -222,7 +221,7 @@ mod source { error_code = "reading_fingerprint", error_type = error_type::READER_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); if self.include_file_metric_tag { counter!( @@ -262,7 +261,6 @@ mod source { error_code = DELETION_FAILED, error_type = error_type::COMMAND_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, ); if self.include_file_metric_tag { counter!( @@ -355,7 +353,7 @@ mod source { error_type = error_type::COMMAND_FAILED, stage = error_stage::RECEIVING, file = %self.file.display(), - internal_log_rate_limit = true, + ); if self.include_file_metric_tag { counter!( @@ -457,7 +455,7 @@ mod source { error_code = "writing_checkpoints", error_type = error_type::WRITER_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -484,7 +482,6 @@ mod source { error_type = error_type::READER_FAILED, stage = error_stage::RECEIVING, path = %self.path.display(), - internal_log_rate_limit = true, ); counter!( "component_errors_total", diff --git a/src/internal_events/fluent.rs b/src/internal_events/fluent.rs index 3493db9cd762d..aa0d26610b72a 100644 --- a/src/internal_events/fluent.rs +++ b/src/internal_events/fluent.rs @@ -30,7 +30,7 @@ impl InternalEvent for FluentMessageDecodeError<'_> { base64_encoded_message = %self.base64_encoded_message, error_type = error_type::PARSER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/gcp_pubsub.rs b/src/internal_events/gcp_pubsub.rs index 1f6a7b5253514..be3f274e8d225 100644 --- a/src/internal_events/gcp_pubsub.rs +++ b/src/internal_events/gcp_pubsub.rs @@ -14,7 +14,7 @@ impl InternalEvent for GcpPubsubConnectError { error_code = "failed_connecting", error_type = error_type::CONNECTION_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); counter!( @@ -39,7 +39,7 @@ impl InternalEvent for GcpPubsubStreamingPullError { error_code = "failed_streaming_pull", error_type = error_type::REQUEST_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); counter!( @@ -64,7 +64,7 @@ impl InternalEvent for GcpPubsubReceiveError { error_code = "failed_fetching_events", error_type = error_type::REQUEST_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); counter!( diff --git a/src/internal_events/host_metrics.rs b/src/internal_events/host_metrics.rs index 3352e529edf0a..56d5649159dfb 100644 --- a/src/internal_events/host_metrics.rs +++ b/src/internal_events/host_metrics.rs @@ -13,7 +13,6 @@ impl InternalEvent for HostMetricsScrapeError { message = self.message, error_type = error_type::READER_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, ); counter!( @@ -38,7 +37,7 @@ impl InternalEvent for HostMetricsScrapeDetailError { error = %self.error, error_type = error_type::READER_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); counter!( @@ -65,7 +64,7 @@ impl InternalEvent for HostMetricsScrapeFilesystemError { error = %self.error, error_type = error_type::READER_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); counter!( diff --git a/src/internal_events/http.rs b/src/internal_events/http.rs index b2b3bfec4ad95..5a1e818dcc484 100644 --- a/src/internal_events/http.rs +++ b/src/internal_events/http.rs @@ -127,7 +127,7 @@ impl InternalEvent for HttpBadRequest<'_> { error_type = error_type::REQUEST_FAILED, error_stage = error_stage::RECEIVING, http_code = %self.code, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/http_client.rs b/src/internal_events/http_client.rs index a6fde2f1a6dd1..155e9d8d67739 100644 --- a/src/internal_events/http_client.rs +++ b/src/internal_events/http_client.rs @@ -86,7 +86,7 @@ impl InternalEvent for GotHttpWarning<'_> { error = %self.error, error_type = error_type::REQUEST_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!("http_client_errors_total", "error_kind" => self.error.to_string()).increment(1); histogram!("http_client_rtt_seconds").record(self.roundtrip); diff --git a/src/internal_events/http_client_source.rs b/src/internal_events/http_client_source.rs index 376adf35eda56..ad1b989d3969a 100644 --- a/src/internal_events/http_client_source.rs +++ b/src/internal_events/http_client_source.rs @@ -49,7 +49,7 @@ impl InternalEvent for HttpClientHttpResponseError { stage = error_stage::RECEIVING, error_type = error_type::REQUEST_FAILED, error_code = %http_error_code(self.code.as_u16()), - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -76,7 +76,7 @@ impl InternalEvent for HttpClientHttpError { error = ?self.error, error_type = error_type::REQUEST_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/influxdb.rs b/src/internal_events/influxdb.rs index 007f5565f1a8f..d487353b8dc68 100644 --- a/src/internal_events/influxdb.rs +++ b/src/internal_events/influxdb.rs @@ -16,7 +16,6 @@ impl InternalEvent for InfluxdbEncodingError { error = %self.error_message, error_type = error_type::ENCODER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, ); counter!( "component_errors_total", diff --git a/src/internal_events/journald.rs b/src/internal_events/journald.rs index f467137186baa..9f7e9a694f9ca 100644 --- a/src/internal_events/journald.rs +++ b/src/internal_events/journald.rs @@ -17,7 +17,7 @@ impl InternalEvent for JournaldInvalidRecordError { text = %self.text, error_type = error_type::PARSER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -40,7 +40,7 @@ impl InternalEvent for JournaldStartJournalctlError { error = %self.error, error_type = error_type::COMMAND_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -63,7 +63,7 @@ impl InternalEvent for JournaldReadError { error = %self.error, error_type = error_type::READER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -88,7 +88,6 @@ impl InternalEvent for JournaldCheckpointSetError { error = %self.error, error_type = error_type::IO_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, ); counter!( "component_errors_total", @@ -113,7 +112,7 @@ impl InternalEvent for JournaldCheckpointFileOpenError { error = %self.error, error_type = error_type::IO_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/kafka.rs b/src/internal_events/kafka.rs index ea1b198be7053..05e06ea5161ff 100644 --- a/src/internal_events/kafka.rs +++ b/src/internal_events/kafka.rs @@ -78,7 +78,7 @@ impl InternalEvent for KafkaOffsetUpdateError { error_code = "kafka_offset_update", error_type = error_type::READER_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -103,7 +103,7 @@ impl InternalEvent for KafkaReadError { error_code = "reading_message", error_type = error_type::READER_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -163,7 +163,6 @@ impl InternalEvent for KafkaHeaderExtractionError<'_> { error_type = error_type::PARSER_FAILED, stage = error_stage::RECEIVING, header_field = self.header_field.to_string(), - internal_log_rate_limit = true, ); counter!( "component_errors_total", diff --git a/src/internal_events/kubernetes_logs.rs b/src/internal_events/kubernetes_logs.rs index 77dac30b1945b..e70f3f47cc4a7 100644 --- a/src/internal_events/kubernetes_logs.rs +++ b/src/internal_events/kubernetes_logs.rs @@ -70,7 +70,7 @@ impl InternalEvent for KubernetesLogsEventAnnotationError<'_> { error_code = ANNOTATION_FAILED, error_type = error_type::READER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -95,7 +95,7 @@ impl InternalEvent for KubernetesLogsEventNamespaceAnnotationError<'_> { error_code = ANNOTATION_FAILED, error_type = error_type::READER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -121,7 +121,7 @@ impl InternalEvent for KubernetesLogsEventNodeAnnotationError<'_> { error_code = ANNOTATION_FAILED, error_type = error_type::READER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -161,7 +161,7 @@ impl InternalEvent for KubernetesLogsDockerFormatParseError<'_> { error = %self.error, error_type = error_type::PARSER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -190,7 +190,7 @@ impl InternalEvent for KubernetesLifecycleError { error_code = KUBERNETES_LIFECYCLE, error_type = error_type::READER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/logplex.rs b/src/internal_events/logplex.rs index 160947fb0d147..b08df7c624d4f 100644 --- a/src/internal_events/logplex.rs +++ b/src/internal_events/logplex.rs @@ -36,7 +36,7 @@ impl InternalEvent for HerokuLogplexRequestReadError { error_type = error_type::READER_FAILED, error_code = io_error_code(&self.error), stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/loki.rs b/src/internal_events/loki.rs index 2c95d524c655f..6a5e7567b65b2 100644 --- a/src/internal_events/loki.rs +++ b/src/internal_events/loki.rs @@ -12,7 +12,6 @@ impl InternalEvent for LokiEventUnlabeledError { error_code = "unlabeled_event", error_type = error_type::CONDITION_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, ); counter!( @@ -39,7 +38,6 @@ impl InternalEvent for LokiOutOfOrderEventDroppedError { error_code = "out_of_order", error_type = error_type::CONDITION_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, ); emit!(ComponentEventsDropped:: { @@ -68,7 +66,6 @@ impl InternalEvent for LokiOutOfOrderEventRewritten { message = "Timestamps rewritten.", count = self.count, reason = "out_of_order", - internal_log_rate_limit = true, ); counter!("rewritten_timestamp_events_total").increment(self.count as u64); } @@ -86,7 +83,6 @@ impl InternalEvent for LokiTimestampNonParsableEventsDropped { error_code = "non-parsable_timestamp", error_type = error_type::CONDITION_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, ); emit!(ComponentEventsDropped:: { count: 1, reason }); diff --git a/src/internal_events/lua.rs b/src/internal_events/lua.rs index 2805b5b57917d..392f1b8b26199 100644 --- a/src/internal_events/lua.rs +++ b/src/internal_events/lua.rs @@ -28,7 +28,6 @@ impl InternalEvent for LuaScriptError { error_code = mlua_error_code(&self.error), error_type = error_type::COMMAND_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, ); counter!( "component_errors_total", @@ -58,7 +57,7 @@ impl InternalEvent for LuaBuildError { error_type = error_type::SCRIPT_FAILED, error_code = lua_build_error_code(&self.error), stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/metric_to_log.rs b/src/internal_events/metric_to_log.rs index 79a9d117054a8..5bbceeb5017c6 100644 --- a/src/internal_events/metric_to_log.rs +++ b/src/internal_events/metric_to_log.rs @@ -16,7 +16,6 @@ impl InternalEvent for MetricToLogSerializeError { error = ?self.error, error_type = error_type::ENCODER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true ); counter!( "component_errors_total", diff --git a/src/internal_events/mongodb_metrics.rs b/src/internal_events/mongodb_metrics.rs index 8d292342d3f0d..cf57b7af94e6a 100644 --- a/src/internal_events/mongodb_metrics.rs +++ b/src/internal_events/mongodb_metrics.rs @@ -48,7 +48,7 @@ impl InternalEvent for MongoDbMetricsRequestError<'_> { error = ?self.error, error_type = error_type::REQUEST_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -72,7 +72,7 @@ impl InternalEvent for MongoDbMetricsBsonParseError<'_> { error = ?self.error, error_type = error_type::PARSER_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/mqtt.rs b/src/internal_events/mqtt.rs index d20ccad0bad98..507f4aea2464c 100644 --- a/src/internal_events/mqtt.rs +++ b/src/internal_events/mqtt.rs @@ -18,7 +18,7 @@ impl InternalEvent for MqttConnectionError { error_code = "mqtt_connection_error", error_type = error_type::WRITER_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/nginx_metrics.rs b/src/internal_events/nginx_metrics.rs index 02f3309aec3a1..d6561ccf21096 100644 --- a/src/internal_events/nginx_metrics.rs +++ b/src/internal_events/nginx_metrics.rs @@ -48,7 +48,7 @@ impl InternalEvent for NginxMetricsRequestError<'_> { error = %self.error, error_type = error_type::REQUEST_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -73,7 +73,7 @@ impl InternalEvent for NginxMetricsStubStatusParseError<'_> { error = %self.error, error_type = error_type::PARSER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/parser.rs b/src/internal_events/parser.rs index 84cd0556b4821..3638ab4189e7b 100644 --- a/src/internal_events/parser.rs +++ b/src/internal_events/parser.rs @@ -30,7 +30,6 @@ impl InternalEvent for ParserMatchError<'_> { error_type = error_type::CONDITION_FAILED, stage = error_stage::PROCESSING, field = &truncate_string_at(&String::from_utf8_lossy(self.value), 60)[..], - internal_log_rate_limit = true ); counter!( "component_errors_total", diff --git a/src/internal_events/postgresql_metrics.rs b/src/internal_events/postgresql_metrics.rs index 8843140cb54e7..d7979f8dd1185 100644 --- a/src/internal_events/postgresql_metrics.rs +++ b/src/internal_events/postgresql_metrics.rs @@ -16,7 +16,7 @@ impl InternalEvent for PostgresqlMetricsCollectError<'_> { error_type = error_type::REQUEST_FAILED, stage = error_stage::RECEIVING, endpoint = %self.endpoint, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/process.rs b/src/internal_events/process.rs index 5a54154eca849..ef30c1ea9e7e2 100644 --- a/src/internal_events/process.rs +++ b/src/internal_events/process.rs @@ -73,7 +73,6 @@ impl InternalEvent for VectorReloadError { error_code = "reload", error_type = error_type::CONFIGURATION_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, ); counter!( "component_errors_total", @@ -95,7 +94,6 @@ impl InternalEvent for VectorConfigLoadError { error_code = "config_load", error_type = error_type::CONFIGURATION_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, ); counter!( "component_errors_total", @@ -117,7 +115,6 @@ impl InternalEvent for VectorRecoveryError { error_code = "recovery", error_type = error_type::CONFIGURATION_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, ); counter!( "component_errors_total", diff --git a/src/internal_events/prometheus.rs b/src/internal_events/prometheus.rs index 752e8e238fac4..2285ee1323773 100644 --- a/src/internal_events/prometheus.rs +++ b/src/internal_events/prometheus.rs @@ -24,7 +24,7 @@ impl InternalEvent for PrometheusParseError<'_> { error = ?self.error, error_type = error_type::PARSER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); debug!( message = %format!("Failed to parse response:\n\n{}\n\n", self.body), @@ -53,7 +53,7 @@ impl InternalEvent for PrometheusRemoteWriteParseError { error = ?self.error, error_type = error_type::PARSER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -74,7 +74,6 @@ impl InternalEvent for PrometheusNormalizationError { message = normalization_reason, error_type = error_type::CONVERSION_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, ); counter!( "component_errors_total", diff --git a/src/internal_events/pulsar.rs b/src/internal_events/pulsar.rs index c03dea3870086..416768fcff6a5 100644 --- a/src/internal_events/pulsar.rs +++ b/src/internal_events/pulsar.rs @@ -19,7 +19,7 @@ impl InternalEvent for PulsarSendingError { error = %self.error, error_type = error_type::REQUEST_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -46,7 +46,7 @@ impl InternalEvent for PulsarPropertyExtractionError { error_type = error_type::PARSER_FAILED, stage = error_stage::PROCESSING, property_field = %self.property_field, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -105,7 +105,7 @@ registered_event!( error_code = "reading_message", error_type = error_type::READER_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); self.read_errors.increment(1_u64); @@ -117,7 +117,7 @@ registered_event!( error_code = "acknowledge_message", error_type = error_type::ACKNOWLEDGMENT_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); self.ack_errors.increment(1_u64); @@ -129,7 +129,7 @@ registered_event!( error_code = "negative_acknowledge_message", error_type = error_type::ACKNOWLEDGMENT_FAILED, stage = error_stage::RECEIVING, - internal_log_rate_limit = true, + ); self.nack_errors.increment(1_u64); diff --git a/src/internal_events/redis.rs b/src/internal_events/redis.rs index 0ac9e9e59d417..32af514f4215d 100644 --- a/src/internal_events/redis.rs +++ b/src/internal_events/redis.rs @@ -23,7 +23,7 @@ impl InternalEvent for RedisReceiveEventError { error_code = %self.error_code, error_type = error_type::READER_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/reduce.rs b/src/internal_events/reduce.rs index b9887aaf54bb7..c62c1ed3639fa 100644 --- a/src/internal_events/reduce.rs +++ b/src/internal_events/reduce.rs @@ -26,7 +26,7 @@ impl InternalEvent for ReduceAddEventError { error = ?self.error, error_type = error_type::CONDITION_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/remap.rs b/src/internal_events/remap.rs index d05e6dfb9c874..93193941b23fe 100644 --- a/src/internal_events/remap.rs +++ b/src/internal_events/remap.rs @@ -19,7 +19,7 @@ impl InternalEvent for RemapMappingError { error = ?self.error, error_type = error_type::CONVERSION_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/sematext_metrics.rs b/src/internal_events/sematext_metrics.rs index b4b5be737fb2e..67839e8c90e79 100644 --- a/src/internal_events/sematext_metrics.rs +++ b/src/internal_events/sematext_metrics.rs @@ -19,7 +19,7 @@ impl InternalEvent for SematextMetricsInvalidMetricError<'_> { stage = error_stage::PROCESSING, value = ?self.metric.value(), kind = ?self.metric.kind(), - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -46,7 +46,7 @@ impl InternalEvent for SematextMetricsEncodeEventError error = %self.error, error_type = error_type::ENCODER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/socket.rs b/src/internal_events/socket.rs index 5f2f7aac301e4..389adb0f478af 100644 --- a/src/internal_events/socket.rs +++ b/src/internal_events/socket.rs @@ -124,7 +124,7 @@ impl InternalEvent for SocketBindError { error_type = error_type::IO_FAILED, stage = error_stage::INITIALIZING, %mode, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -191,7 +191,7 @@ impl InternalEvent for SocketReceiveError { error_type = error_type::READER_FAILED, stage = error_stage::RECEIVING, %mode, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -221,7 +221,7 @@ impl InternalEvent for SocketSendError { error_type = error_type::WRITER_FAILED, stage = error_stage::SENDING, %mode, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/splunk_hec.rs b/src/internal_events/splunk_hec.rs index 4ff5838cdc0b7..e4853c1a8b40e 100644 --- a/src/internal_events/splunk_hec.rs +++ b/src/internal_events/splunk_hec.rs @@ -33,7 +33,7 @@ mod sink { error_code = "serializing_json", error_type = error_type::ENCODER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -62,7 +62,7 @@ mod sink { stage = error_stage::PROCESSING, value = ?self.value, kind = ?self.kind, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -92,7 +92,7 @@ mod sink { error_code = "invalid_response", error_type = error_type::PARSER_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -118,7 +118,7 @@ mod sink { error_code = "indexer_ack_failed", error_type = error_type::ACKNOWLEDGMENT_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -143,7 +143,7 @@ mod sink { error_code = "indexer_ack_unavailable", error_type = error_type::ACKNOWLEDGMENT_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/statsd_sink.rs b/src/internal_events/statsd_sink.rs index 165ef885eabbd..b7ee84cfbc457 100644 --- a/src/internal_events/statsd_sink.rs +++ b/src/internal_events/statsd_sink.rs @@ -20,7 +20,7 @@ impl InternalEvent for StatsdInvalidMetricError<'_> { stage = error_stage::PROCESSING, value = ?self.value, kind = ?self.kind, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/tag_cardinality_limit.rs b/src/internal_events/tag_cardinality_limit.rs index 4c0c315eb11c1..e0a79e583bbcc 100644 --- a/src/internal_events/tag_cardinality_limit.rs +++ b/src/internal_events/tag_cardinality_limit.rs @@ -14,7 +14,6 @@ impl InternalEvent for TagCardinalityLimitRejectingEvent<'_> { metric_name = self.metric_name, tag_key = self.tag_key, tag_value = self.tag_value, - internal_log_rate_limit = true, ); counter!("tag_value_limit_exceeded_total").increment(1); @@ -38,7 +37,6 @@ impl InternalEvent for TagCardinalityLimitRejectingTag<'_> { metric_name = self.metric_name, tag_key = self.tag_key, tag_value = self.tag_value, - internal_log_rate_limit = true, ); counter!("tag_value_limit_exceeded_total").increment(1); } diff --git a/src/internal_events/tcp.rs b/src/internal_events/tcp.rs index 23ec1fc88e830..ae050b99be309 100644 --- a/src/internal_events/tcp.rs +++ b/src/internal_events/tcp.rs @@ -60,7 +60,6 @@ impl InternalEvent for TcpSocketError<'_, E> { peer_addr = ?self.peer_addr, error_type = error_type::CONNECTION_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, ); counter!( "component_errors_total", @@ -89,7 +88,6 @@ impl InternalEvent for TcpSocketTlsConnectionError { debug!( message = "Connection error, probably a healthcheck.", error = %self.error, - internal_log_rate_limit = true, ); } _ => { @@ -99,7 +97,6 @@ impl InternalEvent for TcpSocketTlsConnectionError { error_code = "connection_failed", error_type = error_type::WRITER_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, ); counter!( "component_errors_total", @@ -127,7 +124,6 @@ impl InternalEvent for TcpSendAckError { error_code = "ack_failed", error_type = error_type::WRITER_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, ); counter!( "component_errors_total", diff --git a/src/internal_events/template.rs b/src/internal_events/template.rs index 82df77cefdb8d..c7b42b1b10363 100644 --- a/src/internal_events/template.rs +++ b/src/internal_events/template.rs @@ -23,7 +23,7 @@ impl InternalEvent for TemplateRenderingError<'_> { error = %self.error, error_type = error_type::TEMPLATE_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( @@ -43,7 +43,7 @@ impl InternalEvent for TemplateRenderingError<'_> { error = %self.error, error_type = error_type::TEMPLATE_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); } } diff --git a/src/internal_events/udp.rs b/src/internal_events/udp.rs index 15e9df4c2b07b..b983543e6cae1 100644 --- a/src/internal_events/udp.rs +++ b/src/internal_events/udp.rs @@ -47,7 +47,6 @@ impl InternalEvent for UdpSendIncompleteError { dropped = self.data_size - self.sent, error_type = error_type::WRITER_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, ); counter!( "component_errors_total", diff --git a/src/internal_events/unix.rs b/src/internal_events/unix.rs index a45777ab8f2e1..f66327011da98 100644 --- a/src/internal_events/unix.rs +++ b/src/internal_events/unix.rs @@ -54,7 +54,7 @@ impl InternalEvent for UnixSocketError<'_, E> { path = ?self.path, error_type = error_type::CONNECTION_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -80,7 +80,7 @@ impl InternalEvent for UnixSocketSendError<'_, E> { path = ?self.path, error_type = error_type::WRITER_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -109,7 +109,6 @@ impl InternalEvent for UnixSendIncompleteError { dropped = self.data_size - self.sent, error_type = error_type::WRITER_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, ); counter!( "component_errors_total", @@ -137,7 +136,7 @@ impl InternalEvent for UnixSocketFileDeleteError<'_> { error_code = "delete_socket_file", error_type = error_type::WRITER_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/websocket.rs b/src/internal_events/websocket.rs index 59d64574940f4..bd3bbb78a6ede 100644 --- a/src/internal_events/websocket.rs +++ b/src/internal_events/websocket.rs @@ -33,7 +33,7 @@ impl InternalEvent for WsConnectionFailedError { error_code = "ws_connection_error", error_type = error_type::CONNECTION_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", @@ -76,7 +76,7 @@ impl InternalEvent for WsConnectionError { error_code = "ws_connection_error", error_type = error_type::WRITER_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/websocket_server.rs b/src/internal_events/websocket_server.rs index 2fdb79394998e..881cba02a0b9d 100644 --- a/src/internal_events/websocket_server.rs +++ b/src/internal_events/websocket_server.rs @@ -43,7 +43,7 @@ impl InternalEvent for WsListenerConnectionFailedError { error_code = "ws_connection_error", error_type = error_type::CONNECTION_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, + ); let mut all_tags = self.extra_tags.clone(); all_tags.extend([ @@ -100,7 +100,7 @@ impl InternalEvent for WsListenerSendError { error_code = "ws_server_connection_error", error_type = error_type::WRITER_FAILED, stage = error_stage::SENDING, - internal_log_rate_limit = true, + ); counter!( "component_errors_total", diff --git a/src/internal_events/windows.rs b/src/internal_events/windows.rs index 994b89345e3c0..f44c0a26bc8f3 100644 --- a/src/internal_events/windows.rs +++ b/src/internal_events/windows.rs @@ -102,7 +102,6 @@ impl InternalEvent for WindowsServiceDoesNotExistError<'_> { error_code = "service_missing", error_type = error_type::CONDITION_FAILED, stage = error_stage::PROCESSING, - internal_log_rate_limit = true, ); counter!( "component_errors_total", diff --git a/src/sinks/datadog/logs/sink.rs b/src/sinks/datadog/logs/sink.rs index 8544bf8f61947..f901f9f9bab7e 100644 --- a/src/sinks/datadog/logs/sink.rs +++ b/src/sinks/datadog/logs/sink.rs @@ -200,7 +200,6 @@ pub fn position_reserved_attr_event_root( message = "Semantic meaning is defined, but the event path already exists. Renaming to not overwrite.", meaning = meaning, renamed = &rename_attr, - internal_log_rate_limit = true, ); log.rename_key(desired_path, rename_path); } diff --git a/src/sinks/datadog/traces/apm_stats/flusher.rs b/src/sinks/datadog/traces/apm_stats/flusher.rs index d1ee1e73e949b..e529f49869dbd 100644 --- a/src/sinks/datadog/traces/apm_stats/flusher.rs +++ b/src/sinks/datadog/traces/apm_stats/flusher.rs @@ -70,7 +70,6 @@ pub async fn flush_apm_stats_thread( } Err(_) => { error!( - internal_log_rate_limit = true, message = "Tokio Sender unexpectedly dropped." ); break; diff --git a/src/sinks/gcp/stackdriver/logs/encoder.rs b/src/sinks/gcp/stackdriver/logs/encoder.rs index 5b4faaac8740e..f8dbdee3bec4d 100644 --- a/src/sinks/gcp/stackdriver/logs/encoder.rs +++ b/src/sinks/gcp/stackdriver/logs/encoder.rs @@ -173,7 +173,6 @@ pub(super) fn remap_severity(severity: Value) -> Value { warn!( message = "Unknown severity value string, using DEFAULT.", value = %s, - internal_log_rate_limit = true ); 0 } diff --git a/src/sinks/util/adaptive_concurrency/controller.rs b/src/sinks/util/adaptive_concurrency/controller.rs index 4457e71cad4a7..227ee29c7122a 100644 --- a/src/sinks/util/adaptive_concurrency/controller.rs +++ b/src/sinks/util/adaptive_concurrency/controller.rs @@ -291,7 +291,6 @@ where warn!( message = "Unhandled error response.", %error, - internal_log_rate_limit = true ); false } diff --git a/src/sinks/util/retries.rs b/src/sinks/util/retries.rs index 32ac8fd251f2e..6e4549fb8cbac 100644 --- a/src/sinks/util/retries.rs +++ b/src/sinks/util/retries.rs @@ -152,7 +152,6 @@ where error!( message = "OK/retry response but retries exhausted; dropping the request.", reason = ?reason, - internal_log_rate_limit = true, ); return None; } @@ -182,7 +181,7 @@ where error!( message = "Non-retriable error; dropping the request.", %error, - internal_log_rate_limit = true, + ); None } @@ -196,7 +195,6 @@ where error!( message = "Unexpected error type; dropping the request.", %error, - internal_log_rate_limit = true ); None } diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index a27400e995ef1..b093df4a31a5a 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -182,10 +182,7 @@ fn decode_datadog_sketches( ) -> Result, ErrorMessage> { if body.is_empty() { // The datadog agent may send an empty payload as a keep alive - debug!( - message = "Empty payload ignored.", - internal_log_rate_limit = true - ); + debug!(message = "Empty payload ignored.",); return Ok(Vec::new()); } diff --git a/src/sources/heroku_logs.rs b/src/sources/heroku_logs.rs index c4bde0c2898e1..c86dec895eb7a 100644 --- a/src/sources/heroku_logs.rs +++ b/src/sources/heroku_logs.rs @@ -407,7 +407,6 @@ fn line_to_events( warn!( message = "Line didn't match expected logplex format, so raw message is forwarded.", fields = parts.len(), - internal_log_rate_limit = true ); events.push(LogEvent::from_str_legacy(line).into()) diff --git a/src/sources/socket/udp.rs b/src/sources/socket/udp.rs index 4cfa052c85d7e..d1031fdd8f531 100644 --- a/src/sources/socket/udp.rs +++ b/src/sources/socket/udp.rs @@ -231,7 +231,6 @@ pub(super) fn udp( warn!( message = "Discarding frame larger than max_length.", max_length = max_length, - internal_log_rate_limit = true ); continue; } @@ -259,7 +258,6 @@ pub(super) fn udp( warn!( message = "Discarding frame larger than max_length.", max_length = max_length, - internal_log_rate_limit = true ); } diff --git a/src/transforms/lua/v1/mod.rs b/src/transforms/lua/v1/mod.rs index 1255365f22116..97cef1684ea8e 100644 --- a/src/transforms/lua/v1/mod.rs +++ b/src/transforms/lua/v1/mod.rs @@ -255,7 +255,6 @@ impl mlua::UserData for LuaEvent { message = "Could not set field to Lua value of invalid type, dropping field.", field = key.as_str(), - internal_log_rate_limit = true ); this.inner.as_mut_log().remove(&key_path); } From fbb2dea4c1fedf514d7e6fa0d764c45e489ee3a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ensar=20Saraj=C4=8Di=C4=87?= Date: Thu, 12 Jun 2025 20:59:59 +0000 Subject: [PATCH 058/305] feat(kafka sink): add rate limit configuration for `kafka` sink (#23196) * feature(kafka sink): add rate limit configuration for `kafka` sink Related: #21879 * Add changelog entry --------- Co-authored-by: Pavlos Rontidis --- .../23196_kafka_sink_rate_limit.feature.md | 3 +++ src/sinks/kafka/config.rs | 22 +++++++++++++++++++ src/sinks/kafka/sink.rs | 12 ++++++++-- src/sinks/kafka/tests.rs | 8 +++++++ .../reference/components/sinks/base/kafka.cue | 16 ++++++++++++++ 5 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 changelog.d/23196_kafka_sink_rate_limit.feature.md diff --git a/changelog.d/23196_kafka_sink_rate_limit.feature.md b/changelog.d/23196_kafka_sink_rate_limit.feature.md new file mode 100644 index 0000000000000..a8562b04ab962 --- /dev/null +++ b/changelog.d/23196_kafka_sink_rate_limit.feature.md @@ -0,0 +1,3 @@ +Added `rate_limit_num` and `rate_limit_duration_secs` options to `kafka` sink, to enable rate limiting this sink. + +authors: esensar Quad9DNS diff --git a/src/sinks/kafka/config.rs b/src/sinks/kafka/config.rs index e348cf68532b6..69f3f6b6ec306 100644 --- a/src/sinks/kafka/config.rs +++ b/src/sinks/kafka/config.rs @@ -96,6 +96,18 @@ pub struct KafkaSinkConfig { #[configurable(metadata(docs::advanced))] pub message_timeout_ms: Duration, + /// The time window used for the `rate_limit_num` option. + #[configurable(metadata(docs::type_unit = "seconds"))] + #[configurable(metadata(docs::human_name = "Rate Limit Duration"))] + #[serde(default = "default_rate_limit_duration_secs")] + pub rate_limit_duration_secs: u64, + + /// The maximum number of requests allowed within the `rate_limit_duration_secs` time window. + #[configurable(metadata(docs::type_unit = "requests"))] + #[configurable(metadata(docs::human_name = "Rate Limit Number"))] + #[serde(default = "default_rate_limit_num")] + pub rate_limit_num: u64, + /// A map of advanced options to pass directly to the underlying `librdkafka` client. /// /// For more information on configuration options, see [Configuration properties][config_props_docs]. @@ -134,6 +146,14 @@ const fn default_message_timeout_ms() -> Duration { Duration::from_millis(300000) // default in librdkafka } +const fn default_rate_limit_duration_secs() -> u64 { + 1 +} + +const fn default_rate_limit_num() -> u64 { + i64::MAX as u64 // i64 avoids TOML deserialize issue +} + fn example_librdkafka_options() -> HashMap { HashMap::<_, _>::from_iter([ ("client.id".to_string(), "${ENV_VAR}".to_string()), @@ -245,6 +265,8 @@ impl GenerateConfig for KafkaSinkConfig { auth: Default::default(), socket_timeout_ms: default_socket_timeout_ms(), message_timeout_ms: default_message_timeout_ms(), + rate_limit_duration_secs: default_rate_limit_duration_secs(), + rate_limit_num: default_rate_limit_num(), librdkafka_options: Default::default(), headers_key: None, acknowledgements: Default::default(), diff --git a/src/sinks/kafka/sink.rs b/src/sinks/kafka/sink.rs index 6770572a7b08c..e62fe3450a2c2 100644 --- a/src/sinks/kafka/sink.rs +++ b/src/sinks/kafka/sink.rs @@ -1,9 +1,12 @@ +use std::time::Duration; + use rdkafka::{ error::KafkaError, producer::{BaseProducer, FutureProducer, Producer}, ClientConfig, }; use snafu::{ResultExt, Snafu}; +use tower::limit::RateLimit; use tracing::Span; use vrl::path::OwnedTargetPath; @@ -27,7 +30,7 @@ pub(super) enum BuildError { pub struct KafkaSink { transformer: Transformer, encoder: Encoder<()>, - service: KafkaService, + service: RateLimit, topic: Template, key_field: Option, headers_key: Option, @@ -57,7 +60,12 @@ impl KafkaSink { headers_key: config.headers_key.map(|key| key.0), transformer, encoder, - service: KafkaService::new(producer), + service: ServiceBuilder::new() + .rate_limit( + config.rate_limit_num, + Duration::from_secs(config.rate_limit_duration_secs), + ) + .service(KafkaService::new(producer)), topic: config.topic, key_field: config.key_field.map(|key| key.0), }) diff --git a/src/sinks/kafka/tests.rs b/src/sinks/kafka/tests.rs index 8c2de36ff6867..d6a5a9c1cf0a3 100644 --- a/src/sinks/kafka/tests.rs +++ b/src/sinks/kafka/tests.rs @@ -59,6 +59,8 @@ mod integration_test { auth: KafkaAuthConfig::default(), socket_timeout_ms: Duration::from_millis(60000), message_timeout_ms: Duration::from_millis(300000), + rate_limit_duration_secs: 1, + rate_limit_num: i64::MAX as u64, librdkafka_options: HashMap::new(), headers_key: None, acknowledgements: Default::default(), @@ -85,6 +87,8 @@ mod integration_test { auth: KafkaAuthConfig::default(), socket_timeout_ms: Duration::from_millis(60000), message_timeout_ms: Duration::from_millis(300000), + rate_limit_duration_secs: 1, + rate_limit_num: i64::MAX as u64, librdkafka_options: HashMap::new(), headers_key: None, acknowledgements: Default::default(), @@ -185,6 +189,8 @@ mod integration_test { }, socket_timeout_ms: Duration::from_millis(60000), message_timeout_ms: Duration::from_millis(300000), + rate_limit_duration_secs: 1, + rate_limit_num: i64::MAX as u64, batch, librdkafka_options, headers_key: None, @@ -334,6 +340,8 @@ mod integration_test { auth: kafka_auth.clone(), socket_timeout_ms: Duration::from_millis(60000), message_timeout_ms: Duration::from_millis(300000), + rate_limit_duration_secs: 1, + rate_limit_num: i64::MAX as u64, librdkafka_options: HashMap::new(), headers_key: Some(headers_key.clone()), acknowledgements: Default::default(), diff --git a/website/cue/reference/components/sinks/base/kafka.cue b/website/cue/reference/components/sinks/base/kafka.cue index 1f88e96d23741..f4ec89ef30d8c 100644 --- a/website/cue/reference/components/sinks/base/kafka.cue +++ b/website/cue/reference/components/sinks/base/kafka.cue @@ -489,6 +489,22 @@ base: components: sinks: kafka: configuration: { unit: "milliseconds" } } + rate_limit_duration_secs: { + description: "The time window used for the `rate_limit_num` option." + required: false + type: uint: { + default: 1 + unit: "seconds" + } + } + rate_limit_num: { + description: "The maximum number of requests allowed within the `rate_limit_duration_secs` time window." + required: false + type: uint: { + default: 9223372036854775807 + unit: "requests" + } + } sasl: { description: "Configuration for SASL authentication when interacting with Kafka." required: false From 865c8aca2007e4e280dffae27babc33fee079216 Mon Sep 17 00:00:00 2001 From: Anil Gupta <90670783+anil-db@users.noreply.github.com> Date: Thu, 12 Jun 2025 14:02:13 -0700 Subject: [PATCH 059/305] enhancement(sinks): allow to provide aws session token (#22964) * allow to provide aws session token * doc generation * Update changelog.d/aws-auth-session-token.enhancement.md Co-authored-by: Pavlos Rontidis * incorporate comments * rerun make generate-component-docs * fix bare url in doc --------- Co-authored-by: Pavlos Rontidis --- changelog.d/aws-auth-session-token.enhancement.md | 3 +++ src/aws/auth.rs | 9 ++++++++- website/cue/reference/base/configuration.cue | 8 ++++++++ .../components/sinks/base/aws_cloudwatch_logs.cue | 8 ++++++++ .../components/sinks/base/aws_cloudwatch_metrics.cue | 8 ++++++++ .../components/sinks/base/aws_kinesis_firehose.cue | 8 ++++++++ .../components/sinks/base/aws_kinesis_streams.cue | 8 ++++++++ website/cue/reference/components/sinks/base/aws_s3.cue | 8 ++++++++ website/cue/reference/components/sinks/base/aws_sns.cue | 8 ++++++++ website/cue/reference/components/sinks/base/aws_sqs.cue | 8 ++++++++ .../cue/reference/components/sinks/base/clickhouse.cue | 8 ++++++++ website/cue/reference/components/sinks/base/databend.cue | 8 ++++++++ .../reference/components/sinks/base/elasticsearch.cue | 9 +++++++++ website/cue/reference/components/sinks/base/http.cue | 8 ++++++++ website/cue/reference/components/sinks/base/loki.cue | 8 ++++++++ .../reference/components/sinks/base/opentelemetry.cue | 8 ++++++++ .../components/sinks/base/prometheus_exporter.cue | 8 ++++++++ .../components/sinks/base/prometheus_remote_write.cue | 9 +++++++++ .../cue/reference/components/sinks/base/websocket.cue | 8 ++++++++ website/cue/reference/components/sources/base/aws_s3.cue | 8 ++++++++ .../cue/reference/components/sources/base/aws_sqs.cue | 8 ++++++++ .../reference/components/sources/base/http_client.cue | 8 ++++++++ .../reference/components/sources/base/nginx_metrics.cue | 8 ++++++++ .../components/sources/base/prometheus_scrape.cue | 8 ++++++++ 24 files changed, 189 insertions(+), 1 deletion(-) create mode 100644 changelog.d/aws-auth-session-token.enhancement.md diff --git a/changelog.d/aws-auth-session-token.enhancement.md b/changelog.d/aws-auth-session-token.enhancement.md new file mode 100644 index 0000000000000..5d49320932445 --- /dev/null +++ b/changelog.d/aws-auth-session-token.enhancement.md @@ -0,0 +1,3 @@ +Adds support for session tokens in AWS authentication options. When using temporary credentials (access key, secret key, and session token), the session token is required. Temporary credentials can be provided by an external system and updated using the `SECRET` backend. + +authors: anil-db diff --git a/src/aws/auth.rs b/src/aws/auth.rs index 1b8aa111eec33..4dd00f5cdbc24 100644 --- a/src/aws/auth.rs +++ b/src/aws/auth.rs @@ -71,6 +71,11 @@ pub enum AwsAuthentication { #[configurable(metadata(docs::examples = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"))] secret_access_key: SensitiveString, + /// The AWS session token. + /// See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + #[configurable(metadata(docs::examples = "AQoDYXdz...AQoDYXdz..."))] + session_token: Option, + /// The ARN of an [IAM role][iam_role] to assume. /// /// [iam_role]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html @@ -275,11 +280,12 @@ impl AwsAuthentication { external_id, region, session_name, + session_token, } => { let provider = SharedCredentialsProvider::new(Credentials::from_keys( access_key_id.inner(), secret_access_key.inner(), - None, + session_token.clone().map(|v| v.inner().into()), )); if let Some(assume_role) = assume_role { let auth_region = region.clone().map(Region::new).unwrap_or(service_region); @@ -372,6 +378,7 @@ impl AwsAuthentication { external_id: None, region: None, session_name: None, + session_token: None, } } } diff --git a/website/cue/reference/base/configuration.cue b/website/cue/reference/base/configuration.cue index 41650b84adf79..facbc3d1caada 100644 --- a/website/cue/reference/base/configuration.cue +++ b/website/cue/reference/base/configuration.cue @@ -364,6 +364,14 @@ base: configuration: configuration: { """ required: false } + session_token: { + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + required: false + } credentials_file: { type: string: examples: ["/my/aws/credentials"] description: "Path to the credentials file." diff --git a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue index e0f57c6ce0730..7eda53f14b84c 100644 --- a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue +++ b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue @@ -139,6 +139,14 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { required: false type: string: examples: ["vector-indexer-role"] } + session_token: { + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + required: false + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + } } } batch: { diff --git a/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue b/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue index 2f587a9ab0ae9..bc03d510b10c6 100644 --- a/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue +++ b/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue @@ -139,6 +139,14 @@ base: components: sinks: aws_cloudwatch_metrics: configuration: { required: false type: string: examples: ["vector-indexer-role"] } + session_token: { + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + required: false + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + } } } batch: { diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue index 66fc0c718bf69..0f2d80d4e8390 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue @@ -139,6 +139,14 @@ base: components: sinks: aws_kinesis_firehose: configuration: { required: false type: string: examples: ["vector-indexer-role"] } + session_token: { + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + required: false + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + } } } batch: { diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue index fe0c944ef400d..6028c01ba4241 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue @@ -139,6 +139,14 @@ base: components: sinks: aws_kinesis_streams: configuration: { required: false type: string: examples: ["vector-indexer-role"] } + session_token: { + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + required: false + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + } } } batch: { diff --git a/website/cue/reference/components/sinks/base/aws_s3.cue b/website/cue/reference/components/sinks/base/aws_s3.cue index 625daa2730ac5..5654d6ac48727 100644 --- a/website/cue/reference/components/sinks/base/aws_s3.cue +++ b/website/cue/reference/components/sinks/base/aws_s3.cue @@ -214,6 +214,14 @@ base: components: sinks: aws_s3: configuration: { required: false type: string: examples: ["vector-indexer-role"] } + session_token: { + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + required: false + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + } } } batch: { diff --git a/website/cue/reference/components/sinks/base/aws_sns.cue b/website/cue/reference/components/sinks/base/aws_sns.cue index 8953e1893f24d..8d183c33d9b04 100644 --- a/website/cue/reference/components/sinks/base/aws_sns.cue +++ b/website/cue/reference/components/sinks/base/aws_sns.cue @@ -139,6 +139,14 @@ base: components: sinks: aws_sns: configuration: { required: false type: string: examples: ["vector-indexer-role"] } + session_token: { + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + required: false + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + } } } encoding: { diff --git a/website/cue/reference/components/sinks/base/aws_sqs.cue b/website/cue/reference/components/sinks/base/aws_sqs.cue index 0fcfe80f4f0b0..0493d2ede171f 100644 --- a/website/cue/reference/components/sinks/base/aws_sqs.cue +++ b/website/cue/reference/components/sinks/base/aws_sqs.cue @@ -139,6 +139,14 @@ base: components: sinks: aws_sqs: configuration: { required: false type: string: examples: ["vector-indexer-role"] } + session_token: { + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + required: false + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + } } } encoding: { diff --git a/website/cue/reference/components/sinks/base/clickhouse.cue b/website/cue/reference/components/sinks/base/clickhouse.cue index c472c8736c682..c4b97366ddd18 100644 --- a/website/cue/reference/components/sinks/base/clickhouse.cue +++ b/website/cue/reference/components/sinks/base/clickhouse.cue @@ -149,6 +149,14 @@ base: components: sinks: clickhouse: configuration: { required: false type: string: examples: ["vector-indexer-role"] } + session_token: { + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + required: false + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + } } } password: { diff --git a/website/cue/reference/components/sinks/base/databend.cue b/website/cue/reference/components/sinks/base/databend.cue index 0fa20d0dd1d62..4fc3a763ab059 100644 --- a/website/cue/reference/components/sinks/base/databend.cue +++ b/website/cue/reference/components/sinks/base/databend.cue @@ -144,6 +144,14 @@ base: components: sinks: databend: configuration: { required: false type: string: examples: ["vector-indexer-role"] } + session_token: { + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + required: false + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + } } } password: { diff --git a/website/cue/reference/components/sinks/base/elasticsearch.cue b/website/cue/reference/components/sinks/base/elasticsearch.cue index aa29493a59583..59284c4260e4a 100644 --- a/website/cue/reference/components/sinks/base/elasticsearch.cue +++ b/website/cue/reference/components/sinks/base/elasticsearch.cue @@ -182,6 +182,15 @@ base: components: sinks: elasticsearch: configuration: { required: false type: string: examples: ["vector-indexer-role"] } + session_token: { + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + relevant_when: "strategy = \"aws\"" + required: false + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + } strategy: { description: """ The authentication strategy to use. diff --git a/website/cue/reference/components/sinks/base/http.cue b/website/cue/reference/components/sinks/base/http.cue index 5cf6181e34e96..286cf7dd842fb 100644 --- a/website/cue/reference/components/sinks/base/http.cue +++ b/website/cue/reference/components/sinks/base/http.cue @@ -149,6 +149,14 @@ base: components: sinks: http: configuration: { required: false type: string: examples: ["vector-indexer-role"] } + session_token: { + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + required: false + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + } } } password: { diff --git a/website/cue/reference/components/sinks/base/loki.cue b/website/cue/reference/components/sinks/base/loki.cue index 0641330c99865..218aed6a861fb 100644 --- a/website/cue/reference/components/sinks/base/loki.cue +++ b/website/cue/reference/components/sinks/base/loki.cue @@ -149,6 +149,14 @@ base: components: sinks: loki: configuration: { required: false type: string: examples: ["vector-indexer-role"] } + session_token: { + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + required: false + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + } } } password: { diff --git a/website/cue/reference/components/sinks/base/opentelemetry.cue b/website/cue/reference/components/sinks/base/opentelemetry.cue index 4d7a2d4b5228d..0f9d2a62585c1 100644 --- a/website/cue/reference/components/sinks/base/opentelemetry.cue +++ b/website/cue/reference/components/sinks/base/opentelemetry.cue @@ -152,6 +152,14 @@ base: components: sinks: opentelemetry: configuration: protocol: { required: false type: string: examples: ["vector-indexer-role"] } + session_token: { + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + required: false + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + } } } password: { diff --git a/website/cue/reference/components/sinks/base/prometheus_exporter.cue b/website/cue/reference/components/sinks/base/prometheus_exporter.cue index 23b073699ba7b..50f16c511e1a9 100644 --- a/website/cue/reference/components/sinks/base/prometheus_exporter.cue +++ b/website/cue/reference/components/sinks/base/prometheus_exporter.cue @@ -161,6 +161,14 @@ base: components: sinks: prometheus_exporter: configuration: { required: false type: string: examples: ["vector-indexer-role"] } + session_token: { + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + required: false + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + } } } password: { diff --git a/website/cue/reference/components/sinks/base/prometheus_remote_write.cue b/website/cue/reference/components/sinks/base/prometheus_remote_write.cue index 62b12a901aa80..cbdd3ac14a7da 100644 --- a/website/cue/reference/components/sinks/base/prometheus_remote_write.cue +++ b/website/cue/reference/components/sinks/base/prometheus_remote_write.cue @@ -155,6 +155,15 @@ base: components: sinks: prometheus_remote_write: configuration: { required: false type: string: examples: ["vector-indexer-role"] } + session_token: { + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + relevant_when: "strategy = \"aws\"" + required: false + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + } strategy: { description: "The authentication strategy to use." required: true diff --git a/website/cue/reference/components/sinks/base/websocket.cue b/website/cue/reference/components/sinks/base/websocket.cue index 301315bbf9ea1..d6cadf59f1fbc 100644 --- a/website/cue/reference/components/sinks/base/websocket.cue +++ b/website/cue/reference/components/sinks/base/websocket.cue @@ -149,6 +149,14 @@ base: components: sinks: websocket: configuration: { required: false type: string: examples: ["vector-indexer-role"] } + session_token: { + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + required: false + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + } } } password: { diff --git a/website/cue/reference/components/sources/base/aws_s3.cue b/website/cue/reference/components/sources/base/aws_s3.cue index 8b3da586e11e8..e49d52b08c818 100644 --- a/website/cue/reference/components/sources/base/aws_s3.cue +++ b/website/cue/reference/components/sources/base/aws_s3.cue @@ -134,6 +134,14 @@ base: components: sources: aws_s3: configuration: { required: false type: string: examples: ["vector-indexer-role"] } + session_token: { + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + required: false + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + } } } compression: { diff --git a/website/cue/reference/components/sources/base/aws_sqs.cue b/website/cue/reference/components/sources/base/aws_sqs.cue index 9c713b33b2bd4..bc717121c5f09 100644 --- a/website/cue/reference/components/sources/base/aws_sqs.cue +++ b/website/cue/reference/components/sources/base/aws_sqs.cue @@ -134,6 +134,14 @@ base: components: sources: aws_sqs: configuration: { required: false type: string: examples: ["vector-indexer-role"] } + session_token: { + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + required: false + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + } } } client_concurrency: { diff --git a/website/cue/reference/components/sources/base/http_client.cue b/website/cue/reference/components/sources/base/http_client.cue index 5718041692c20..290740a94ac1a 100644 --- a/website/cue/reference/components/sources/base/http_client.cue +++ b/website/cue/reference/components/sources/base/http_client.cue @@ -118,6 +118,14 @@ base: components: sources: http_client: configuration: { required: false type: string: examples: ["vector-indexer-role"] } + session_token: { + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + required: false + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + } } } password: { diff --git a/website/cue/reference/components/sources/base/nginx_metrics.cue b/website/cue/reference/components/sources/base/nginx_metrics.cue index 57fb2200f7234..12cb35c733faf 100644 --- a/website/cue/reference/components/sources/base/nginx_metrics.cue +++ b/website/cue/reference/components/sources/base/nginx_metrics.cue @@ -123,6 +123,14 @@ base: components: sources: nginx_metrics: configuration: { required: false type: string: examples: ["vector-indexer-role"] } + session_token: { + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + required: false + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + } } } password: { diff --git a/website/cue/reference/components/sources/base/prometheus_scrape.cue b/website/cue/reference/components/sources/base/prometheus_scrape.cue index 396aa751a7ee9..ecaff2a727185 100644 --- a/website/cue/reference/components/sources/base/prometheus_scrape.cue +++ b/website/cue/reference/components/sources/base/prometheus_scrape.cue @@ -123,6 +123,14 @@ base: components: sources: prometheus_scrape: configuration: { required: false type: string: examples: ["vector-indexer-role"] } + session_token: { + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + required: false + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + } } } password: { From f31839be8b0b7425fdc3c3fc44e6557dece627ee Mon Sep 17 00:00:00 2001 From: Orri Ganel Date: Thu, 12 Jun 2025 17:11:44 -0400 Subject: [PATCH 060/305] enhancement(kubernetes_logs source): Allow specification of a maximum line size to be applied after merging instead of just before (#22582) * Add config for maximum allowed line size after merging * Add warns when we drop partial logs for being too big; shift some comments around * Add changelog * Format * Increment component_discarded_events_total on violation of max_line_size and max_merged_line_size * Update changelog.d/22581_max_merged_line_bytes.feature.md Co-authored-by: Pavlos Rontidis * Don't emit expired events that are too big nor ones that don't appear to be partial; fix test * Fix another test * Update src/sources/kubernetes_logs/mod.rs Co-authored-by: Pavlos Rontidis * Update src/sources/kubernetes_logs/mod.rs Co-authored-by: Pavlos Rontidis * Remove inadvertently added file * Include Value rather than Event in error struct * Rename field in bucket struct * Move max_merged_line_bytes from being a param to being a field on the state struct * Make new config field optional, defaulting to old behavior * Format * Appease check-events * docs regen * Tweak wording of doc; emit only first 1k bytes of dropped lines in error * Rename fields for clarity * Per PR feedback: copy just the initial 1000 bytes rather than cloning the whole buffer and then truncating, use more idiomatic Rust for handling both configured and unconfigured cases of max_merged_line_bytes * Allow spelling of already-merged changelog filename * Don't try to include more characters than there actually are in the slice * Don't just get enough capacity, make sure length matches too * Formatting --------- Co-authored-by: Orri Ganel Co-authored-by: Pavlos Rontidis --- .github/actions/spelling/expect.txt | 1 + .../22581_max_merged_line_bytes.feature.md | 3 + lib/file-source/src/buffer.rs | 55 +++++--- lib/file-source/src/file_server.rs | 16 ++- lib/file-source/src/file_watcher/mod.rs | 54 ++++++-- .../src/file_watcher/tests/experiment.rs | 9 +- .../tests/experiment_no_truncations.rs | 14 +- lib/file-source/src/fingerprinter.rs | 3 + lib/file-source/src/internal_events.rs | 9 ++ src/internal_events/file.rs | 47 +++++++ src/internal_events/kubernetes_logs.rs | 35 ++++- src/sources/kubernetes_logs/mod.rs | 30 ++++- .../kubernetes_logs/partial_events_merger.rs | 123 ++++++++++++++++-- .../sources/base/kubernetes_logs.cue | 13 ++ 14 files changed, 358 insertions(+), 54 deletions(-) create mode 100644 changelog.d/22581_max_merged_line_bytes.feature.md diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index ee745e5c30d5e..82fea89d4fb57 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -175,6 +175,7 @@ efgh Elhage emerg Enableable +enableable endianess endler eni diff --git a/changelog.d/22581_max_merged_line_bytes.feature.md b/changelog.d/22581_max_merged_line_bytes.feature.md new file mode 100644 index 0000000000000..b5d36e53bde52 --- /dev/null +++ b/changelog.d/22581_max_merged_line_bytes.feature.md @@ -0,0 +1,3 @@ +The `kubernetes_logs` source now includes a new `max_merged_line_bytes` configuration option. This setting enables users to cap the size of log lines after they’ve been combined using `auto_partial_merge`. Previously, the `max_line_bytes` field only restricted line sizes *before* merging, leaving no practical way to limit the length of merged lines—unless you set a size so tiny that it prevented merging altogether by stopping short of the continuation character. This new option gives you better control over merged line sizes. + +authors: ganelo diff --git a/lib/file-source/src/buffer.rs b/lib/file-source/src/buffer.rs index c8dbe1f400905..55dd481334e1d 100644 --- a/lib/file-source/src/buffer.rs +++ b/lib/file-source/src/buffer.rs @@ -1,11 +1,18 @@ -use std::io::{self, BufRead}; +use std::{ + cmp::min, + io::{self, BufRead}, +}; use bstr::Finder; use bytes::BytesMut; -use tracing::warn; use crate::FilePosition; +pub struct ReadResult { + pub successfully_read: Option, + pub discarded_for_size_and_truncated: Vec, +} + /// Read up to `max_size` bytes from `reader`, splitting by `delim` /// /// The function reads up to `max_size` bytes from `reader`, splitting the input @@ -29,17 +36,18 @@ use crate::FilePosition; /// Benchmarks indicate that this function processes in the high single-digit /// GiB/s range for buffers of length 1KiB. For buffers any smaller than this /// the overhead of setup dominates our benchmarks. -pub fn read_until_with_max_size( - reader: &mut R, - position: &mut FilePosition, - delim: &[u8], - buf: &mut BytesMut, +pub fn read_until_with_max_size<'a, R: BufRead + ?Sized>( + reader: &'a mut R, + position: &'a mut FilePosition, + delim: &'a [u8], + buf: &'a mut BytesMut, max_size: usize, -) -> io::Result> { +) -> io::Result { let mut total_read = 0; let mut discarding = false; let delim_finder = Finder::new(delim); let delim_len = delim.len(); + let mut discarded_for_size_and_truncated = Vec::new(); loop { let available: &[u8] = match reader.fill_buf() { Ok(n) => n, @@ -68,16 +76,20 @@ pub fn read_until_with_max_size( total_read += used; if !discarding && buf.len() > max_size { - warn!( - message = "Found line that exceeds max_line_bytes; discarding.", - internal_log_rate_limit = true - ); + // keep only the first <1k bytes to make sure we can actually emit a usable error + let length_to_keep = min(1000, max_size); + let mut truncated: BytesMut = BytesMut::zeroed(length_to_keep); + truncated.copy_from_slice(&buf[0..length_to_keep]); + discarded_for_size_and_truncated.push(truncated); discarding = true; } if done { if !discarding { - return Ok(Some(total_read)); + return Ok(ReadResult { + successfully_read: Some(total_read), + discarded_for_size_and_truncated, + }); } else { discarding = false; buf.clear(); @@ -87,7 +99,10 @@ pub fn read_until_with_max_size( // us to observe an incomplete write. We return None here and let the loop continue // next time the method is called. This is safe because the buffer is specific to this // FileWatcher. - return Ok(None); + return Ok(ReadResult { + successfully_read: None, + discarded_for_size_and_truncated, + }); } } } @@ -99,6 +114,8 @@ mod test { use bytes::{BufMut, BytesMut}; use quickcheck::{QuickCheck, TestResult}; + use crate::buffer::ReadResult; + use super::read_until_with_max_size; fn qc_inner(chunks: Vec>, delim: u8, max_size: NonZeroU8) -> TestResult { @@ -181,7 +198,10 @@ mod test { ) .unwrap() { - None => { + ReadResult { + successfully_read: None, + discarded_for_size_and_truncated: _, + } => { // Subject only returns None if this is the last chunk _and_ // the chunk did not contain a delimiter _or_ the delimiter // was outside the max_size range _or_ the current chunk is empty. @@ -190,7 +210,10 @@ mod test { .any(|details| ((details.chunk_index == idx) && details.within_max_size)); assert!(chunk.is_empty() || !has_valid_delimiter) } - Some(total_read) => { + ReadResult { + successfully_read: Some(total_read), + discarded_for_size_and_truncated: _, + } => { // Now that the function has returned we confirm that the // returned details match our `first_delim` and also that // the `buffer` is populated correctly. diff --git a/lib/file-source/src/file_server.rs b/lib/file-source/src/file_server.rs index 1dfc6ebd08004..ebb1867339242 100644 --- a/lib/file-source/src/file_server.rs +++ b/lib/file-source/src/file_server.rs @@ -19,7 +19,7 @@ use tracing::{debug, error, info, trace}; use crate::{ checkpointer::{Checkpointer, CheckpointsView}, - file_watcher::FileWatcher, + file_watcher::{FileWatcher, RawLineResult}, fingerprinter::{FileFingerprint, Fingerprinter}, paths_provider::PathsProvider, FileSourceInternalEvents, ReadFrom, @@ -263,7 +263,19 @@ where let start = time::Instant::now(); let mut bytes_read: usize = 0; - while let Ok(Some(line)) = watcher.read_line() { + while let Ok(RawLineResult { + raw_line: Some(line), + discarded_for_size_and_truncated, + }) = watcher.read_line() + { + discarded_for_size_and_truncated.iter().for_each(|buf| { + self.emitter.emit_file_line_too_long( + &buf.clone(), + self.max_line_bytes, + buf.len(), + ) + }); + let sz = line.bytes.len(); trace!( message = "Read bytes.", diff --git a/lib/file-source/src/file_watcher/mod.rs b/lib/file-source/src/file_watcher/mod.rs index 80db2b9bc876c..a8d4fd0f81641 100644 --- a/lib/file-source/src/file_watcher/mod.rs +++ b/lib/file-source/src/file_watcher/mod.rs @@ -12,7 +12,9 @@ use tracing::debug; use vector_common::constants::GZIP_MAGIC; use crate::{ - buffer::read_until_with_max_size, metadata_ext::PortableFileExt, FilePosition, ReadFrom, + buffer::{read_until_with_max_size, ReadResult}, + metadata_ext::PortableFileExt, + FilePosition, ReadFrom, }; #[cfg(test)] mod tests; @@ -28,6 +30,12 @@ pub(super) struct RawLine { pub bytes: Bytes, } +#[derive(Debug)] +pub struct RawLineResult { + pub raw_line: Option, + pub discarded_for_size_and_truncated: Vec, +} + /// The `FileWatcher` struct defines the polling based state machine which reads /// from a file path, transparently updating the underlying file descriptor when /// the file has been rolled over, as is common for logs. @@ -207,7 +215,7 @@ impl FileWatcher { /// This function will attempt to read a new line from its file, blocking, /// up to some maximum but unspecified amount of time. `read_line` will open /// a new file handler as needed, transparently to the caller. - pub(super) fn read_line(&mut self) -> io::Result> { + pub(super) fn read_line(&mut self) -> io::Result { self.track_read_attempt(); let reader = &mut self.reader; @@ -220,14 +228,23 @@ impl FileWatcher { &mut self.buf, self.max_line_bytes, ) { - Ok(Some(_)) => { + Ok(ReadResult { + successfully_read: Some(_), + discarded_for_size_and_truncated, + }) => { self.track_read_success(); - Ok(Some(RawLine { - offset: initial_position, - bytes: self.buf.split().freeze(), - })) + Ok(RawLineResult { + raw_line: Some(RawLine { + offset: initial_position, + bytes: self.buf.split().freeze(), + }), + discarded_for_size_and_truncated, + }) } - Ok(None) => { + Ok(ReadResult { + successfully_read: None, + discarded_for_size_and_truncated, + }) => { if !self.file_findable() { self.set_dead(); // File has been deleted, so return what we have in the buffer, even though it @@ -237,16 +254,25 @@ impl FileWatcher { if buf.is_empty() { // EOF self.reached_eof = true; - Ok(None) + Ok(RawLineResult { + raw_line: None, + discarded_for_size_and_truncated, + }) } else { - Ok(Some(RawLine { - offset: initial_position, - bytes: buf, - })) + Ok(RawLineResult { + raw_line: Some(RawLine { + offset: initial_position, + bytes: buf, + }), + discarded_for_size_and_truncated, + }) } } else { self.reached_eof = true; - Ok(None) + Ok(RawLineResult { + raw_line: None, + discarded_for_size_and_truncated, + }) } } Err(e) => { diff --git a/lib/file-source/src/file_watcher/tests/experiment.rs b/lib/file-source/src/file_watcher/tests/experiment.rs index decdbdab98240..cbbfabe0a67b7 100644 --- a/lib/file-source/src/file_watcher/tests/experiment.rs +++ b/lib/file-source/src/file_watcher/tests/experiment.rs @@ -8,7 +8,7 @@ use bytes::Bytes; use quickcheck::{QuickCheck, TestResult}; use crate::{ - file_watcher::{tests::*, FileWatcher}, + file_watcher::{tests::*, FileWatcher, RawLineResult}, ReadFrom, }; @@ -96,11 +96,14 @@ fn experiment(actions: Vec) { Err(_) => { unreachable!(); } - Ok(Some(line)) if line.bytes.is_empty() => { + Ok(RawLineResult { + raw_line: Some(line), + .. + }) if line.bytes.is_empty() => { attempts -= 1; continue; } - Ok(None) => { + Ok(RawLineResult { raw_line: None, .. }) => { attempts -= 1; continue; } diff --git a/lib/file-source/src/file_watcher/tests/experiment_no_truncations.rs b/lib/file-source/src/file_watcher/tests/experiment_no_truncations.rs index ee8a24a9f95bf..78aa7536b9a38 100644 --- a/lib/file-source/src/file_watcher/tests/experiment_no_truncations.rs +++ b/lib/file-source/src/file_watcher/tests/experiment_no_truncations.rs @@ -4,7 +4,7 @@ use bytes::Bytes; use quickcheck::{QuickCheck, TestResult}; use crate::{ - file_watcher::{tests::*, FileWatcher}, + file_watcher::{tests::*, FileWatcher, RawLineResult}, ReadFrom, }; @@ -63,17 +63,23 @@ fn experiment_no_truncations(actions: Vec) { Err(_) => { unreachable!(); } - Ok(Some(line)) if line.bytes.is_empty() => { + Ok(RawLineResult { + raw_line: Some(line), + .. + }) if line.bytes.is_empty() => { attempts -= 1; assert!(fwfiles[read_index].read_line().is_none()); continue; } - Ok(None) => { + Ok(RawLineResult { raw_line: None, .. }) => { attempts -= 1; assert!(fwfiles[read_index].read_line().is_none()); continue; } - Ok(Some(line)) => { + Ok(RawLineResult { + raw_line: Some(line), + .. + }) => { let exp = fwfiles[read_index].read_line().expect("could not readline"); assert_eq!(exp.into_bytes(), line.bytes); // assert_eq!(sz, buf.len() + 1); diff --git a/lib/file-source/src/fingerprinter.rs b/lib/file-source/src/fingerprinter.rs index b176f6e9964ac..a7c339d249760 100644 --- a/lib/file-source/src/fingerprinter.rs +++ b/lib/file-source/src/fingerprinter.rs @@ -392,6 +392,7 @@ mod test { time::Duration, }; + use bytes::BytesMut; use flate2::write::GzEncoder; use tempfile::{tempdir, TempDir}; @@ -813,5 +814,7 @@ mod test { fn emit_files_open(&self, _: usize) {} fn emit_path_globbing_failed(&self, _: &Path, _: &Error) {} + + fn emit_file_line_too_long(&self, _: &BytesMut, _: usize, _: usize) {} } } diff --git a/lib/file-source/src/internal_events.rs b/lib/file-source/src/internal_events.rs index 9eb60e65397a1..3077a51d8cab9 100644 --- a/lib/file-source/src/internal_events.rs +++ b/lib/file-source/src/internal_events.rs @@ -1,5 +1,7 @@ use std::{io::Error, path::Path, time::Duration}; +use bytes::BytesMut; + /// Every internal event in this crate has a corresponding /// method in this trait which should emit the event. pub trait FileSourceInternalEvents: Send + Sync + Clone + 'static { @@ -26,4 +28,11 @@ pub trait FileSourceInternalEvents: Send + Sync + Clone + 'static { fn emit_files_open(&self, count: usize); fn emit_path_globbing_failed(&self, path: &Path, error: &Error); + + fn emit_file_line_too_long( + &self, + truncated_bytes: &BytesMut, + configured_limit: usize, + encountered_size_so_far: usize, + ); } diff --git a/src/internal_events/file.rs b/src/internal_events/file.rs index 84cf828c89d7a..c6d80460143f2 100644 --- a/src/internal_events/file.rs +++ b/src/internal_events/file.rs @@ -105,8 +105,10 @@ impl InternalEvent for FileIoError<'_, P> { mod source { use std::{io::Error, path::Path, time::Duration}; + use bytes::BytesMut; use metrics::counter; use vector_lib::file_source::FileSourceInternalEvents; + use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL}; use super::{FileOpen, InternalEvent}; use vector_lib::emit; @@ -493,6 +495,38 @@ mod source { } } + #[derive(Debug)] + pub struct FileLineTooBigError<'a> { + pub truncated_bytes: &'a BytesMut, + pub configured_limit: usize, + pub encountered_size_so_far: usize, + } + + impl InternalEvent for FileLineTooBigError<'_> { + fn emit(self) { + error!( + message = "Found line that exceeds max_line_bytes; discarding.", + truncated_bytes = ?self.truncated_bytes, + configured_limit = self.configured_limit, + encountered_size_so_far = self.encountered_size_so_far, + internal_log_rate_limit = true, + error_type = error_type::CONDITION_FAILED, + stage = error_stage::RECEIVING, + ); + counter!( + "component_errors_total", + "error_code" => "reading_line_from_file", + "error_type" => error_type::CONDITION_FAILED, + "stage" => error_stage::RECEIVING, + ) + .increment(1); + emit!(ComponentEventsDropped:: { + count: 1, + reason: "Found line that exceeds max_line_bytes; discarding.", + }); + } + } + #[derive(Clone)] pub struct FileSourceInternalEventsEmitter { pub include_file_metric_tag: bool, @@ -575,5 +609,18 @@ mod source { fn emit_path_globbing_failed(&self, path: &Path, error: &Error) { emit!(PathGlobbingError { path, error }); } + + fn emit_file_line_too_long( + &self, + truncated_bytes: &bytes::BytesMut, + configured_limit: usize, + encountered_size_so_far: usize, + ) { + emit!(FileLineTooBigError { + truncated_bytes, + configured_limit, + encountered_size_so_far + }); + } } } diff --git a/src/internal_events/kubernetes_logs.rs b/src/internal_events/kubernetes_logs.rs index e70f3f47cc4a7..ed68c14790656 100644 --- a/src/internal_events/kubernetes_logs.rs +++ b/src/internal_events/kubernetes_logs.rs @@ -1,9 +1,10 @@ use metrics::counter; -use vector_lib::internal_event::InternalEvent; +use vector_lib::internal_event::{InternalEvent, INTENTIONAL}; use vector_lib::{ internal_event::{error_stage, error_type, ComponentEventsDropped, UNINTENTIONAL}, json_size::JsonSize, }; +use vrl::core::Value; use crate::event::Event; @@ -205,3 +206,35 @@ impl InternalEvent for KubernetesLifecycleError { }); } } + +#[derive(Debug)] +pub struct KubernetesMergedLineTooBigError<'a> { + pub event: &'a Value, + pub configured_limit: usize, + pub encountered_size_so_far: usize, +} + +impl InternalEvent for KubernetesMergedLineTooBigError<'_> { + fn emit(self) { + error!( + message = "Found line that exceeds max_merged_line_bytes; discarding.", + event = ?self.event, + configured_limit = self.configured_limit, + encountered_size_so_far = self.encountered_size_so_far, + internal_log_rate_limit = true, + error_type = error_type::CONDITION_FAILED, + stage = error_stage::RECEIVING, + ); + counter!( + "component_errors_total", + "error_code" => "reading_line_from_kubernetes_log", + "error_type" => error_type::CONDITION_FAILED, + "stage" => error_stage::RECEIVING, + ) + .increment(1); + emit!(ComponentEventsDropped:: { + count: 1, + reason: "Found line that exceeds max_merged_line_bytes; discarding.", + }); + } +} diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index fa0fedb80546a..4adf82d61905f 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -4,7 +4,7 @@ //! running inside the cluster as a DaemonSet. #![deny(missing_docs)] -use std::{path::PathBuf, time::Duration}; +use std::{cmp::min, path::PathBuf, time::Duration}; use bytes::Bytes; use chrono::Utc; @@ -193,6 +193,16 @@ pub struct Config { #[configurable(metadata(docs::type_unit = "bytes"))] max_line_bytes: usize, + /// The maximum number of bytes a line can contain - after merging - before being discarded. + /// + /// This protects against malformed lines or tailing incorrect files. + /// + /// Note that, if auto_partial_merge is false, this config will be ignored. Also, if max_line_bytes is too small to reach the continuation character, then this + /// config will have no practical impact (the same is true of `auto_partial_merge`). Finally, the smaller of `max_merged_line_bytes` and `max_line_bytes` will apply + /// if auto_partial_merge is true, so if this is set to be 1 MiB, for example, but `max_line_bytes` is set to ~2.5 MiB, then every line greater than 1 MiB will be dropped. + #[configurable(metadata(docs::type_unit = "bytes"))] + max_merged_line_bytes: Option, + /// The number of lines to read for generating the checksum. /// /// If your files share a common header that is not always a fixed size, @@ -294,6 +304,7 @@ impl Default for Config { max_read_bytes: default_max_read_bytes(), oldest_first: default_oldest_first(), max_line_bytes: default_max_line_bytes(), + max_merged_line_bytes: None, fingerprint_lines: default_fingerprint_lines(), glob_minimum_cooldown_ms: default_glob_minimum_cooldown_ms(), ingestion_timestamp_field: None, @@ -553,6 +564,7 @@ struct Source { max_read_bytes: usize, oldest_first: bool, max_line_bytes: usize, + max_merged_line_bytes: Option, fingerprint_lines: usize, glob_minimum_cooldown: Duration, use_apiserver_cache: bool, @@ -641,6 +653,7 @@ impl Source { max_read_bytes: config.max_read_bytes, oldest_first: config.oldest_first, max_line_bytes: config.max_line_bytes, + max_merged_line_bytes: config.max_merged_line_bytes, fingerprint_lines: config.fingerprint_lines, glob_minimum_cooldown, use_apiserver_cache: config.use_apiserver_cache, @@ -676,6 +689,7 @@ impl Source { max_read_bytes, oldest_first, max_line_bytes, + max_merged_line_bytes, fingerprint_lines, glob_minimum_cooldown, use_apiserver_cache, @@ -779,6 +793,14 @@ impl Source { let ignore_before = calculate_ignore_before(ignore_older_secs); + let mut resolved_max_line_bytes = max_line_bytes; + if auto_partial_merge { + resolved_max_line_bytes = min( + max_line_bytes, + max_merged_line_bytes.unwrap_or(max_line_bytes), + ); + } + // TODO: maybe more of the parameters have to be configurable. let checkpointer = Checkpointer::new(&data_dir); @@ -803,7 +825,7 @@ impl Source { ignore_before, // The maximum number of bytes a line can contain before being discarded. This // protects against malformed lines or tailing incorrect files. - max_line_bytes, + max_line_bytes: resolved_max_line_bytes, // Delimiter bytes that is used to read the file line-by-line line_delimiter: Bytes::from("\n"), // The directory where to keep the checkpoints. @@ -821,7 +843,7 @@ impl Source { ignored_header_bytes: 0, lines: fingerprint_lines, }, - max_line_length: max_line_bytes, + max_line_length: resolved_max_line_bytes, ignore_not_found: true, }, oldest_first, @@ -897,7 +919,7 @@ impl Source { let (events_count, _) = events.size_hint(); let mut stream = if auto_partial_merge { - merge_partial_events(events, log_namespace).left_stream() + merge_partial_events(events, log_namespace, max_merged_line_bytes).left_stream() } else { events.right_stream() }; diff --git a/src/sources/kubernetes_logs/partial_events_merger.rs b/src/sources/kubernetes_logs/partial_events_merger.rs index 14628df46ad6e..56752b49454cd 100644 --- a/src/sources/kubernetes_logs/partial_events_merger.rs +++ b/src/sources/kubernetes_logs/partial_events_merger.rs @@ -11,6 +11,7 @@ use vrl::owned_value_path; use crate::event; use crate::event::{Event, LogEvent, Value}; +use crate::internal_events::KubernetesMergedLineTooBigError; use crate::sources::kubernetes_logs::transform_utils::get_message_path; /// The key we use for `file` field. @@ -20,6 +21,7 @@ const EXPIRATION_TIME: Duration = Duration::from_secs(30); struct PartialEventMergeState { buckets: HashMap, + maybe_max_merged_line_bytes: Option, } impl PartialEventMergeState { @@ -30,38 +32,80 @@ impl PartialEventMergeState { message_path: &OwnedTargetPath, expiration_time: Duration, ) { + let mut bytes_mut = BytesMut::new(); if let Some(bucket) = self.buckets.get_mut(file) { + // don't bother continuing to process new partial events that match existing ones that are already too big + if bucket.exceeds_max_merged_line_limit { + return; + } + // merging with existing event if let (Some(Value::Bytes(prev_value)), Some(Value::Bytes(new_value))) = (bucket.event.get_mut(message_path), event.get(message_path)) { - let mut bytes_mut = BytesMut::new(); bytes_mut.extend_from_slice(prev_value); bytes_mut.extend_from_slice(new_value); + + // drop event if it's bigger than max allowed + if let Some(max_merged_line_bytes) = self.maybe_max_merged_line_bytes { + if bytes_mut.len() > max_merged_line_bytes { + bucket.exceeds_max_merged_line_limit = true; + // perf impact of clone should be minimal since being here means no further processing of this event will occur + emit!(KubernetesMergedLineTooBigError { + event: &Value::Bytes(new_value.clone()), + configured_limit: max_merged_line_bytes, + encountered_size_so_far: bytes_mut.len() + }); + } + } + *prev_value = bytes_mut.freeze(); } } else { // new event + + let mut exceeds_max_merged_line_limit = false; + + if let Some(Value::Bytes(event_bytes)) = event.get(message_path) { + bytes_mut.extend_from_slice(event_bytes); + if let Some(max_merged_line_bytes) = self.maybe_max_merged_line_bytes { + exceeds_max_merged_line_limit = bytes_mut.len() > max_merged_line_bytes; + + if exceeds_max_merged_line_limit { + // perf impact of clone should be minimal since being here means no further processing of this event will occur + emit!(KubernetesMergedLineTooBigError { + event: &Value::Bytes(event_bytes.clone()), + configured_limit: max_merged_line_bytes, + encountered_size_so_far: bytes_mut.len() + }); + } + } + } + self.buckets.insert( file.to_owned(), Bucket { event, expiration: Instant::now() + expiration_time, + exceeds_max_merged_line_limit, }, ); } } fn remove_event(&mut self, file: &str) -> Option { - self.buckets.remove(file).map(|bucket| bucket.event) + self.buckets + .remove(file) + .filter(|bucket| !bucket.exceeds_max_merged_line_limit) + .map(|bucket| bucket.event) } fn emit_expired_events(&mut self, emitter: &mut Emitter) { let now = Instant::now(); self.buckets.retain(|_key, bucket| { let expired = now >= bucket.expiration; - if expired { + if expired && !bucket.exceeds_max_merged_line_limit { emitter.emit(bucket.event.clone()); } !expired @@ -70,7 +114,9 @@ impl PartialEventMergeState { fn flush_events(&mut self, emitter: &mut Emitter) { for (_, bucket) in self.buckets.drain() { - emitter.emit(bucket.event); + if !bucket.exceeds_max_merged_line_limit { + emitter.emit(bucket.event); + } } } } @@ -78,13 +124,20 @@ impl PartialEventMergeState { struct Bucket { event: LogEvent, expiration: Instant, + exceeds_max_merged_line_limit: bool, } pub fn merge_partial_events( stream: impl Stream + 'static, log_namespace: LogNamespace, + maybe_max_merged_line_bytes: Option, ) -> impl Stream { - merge_partial_events_with_custom_expiration(stream, log_namespace, EXPIRATION_TIME) + merge_partial_events_with_custom_expiration( + stream, + log_namespace, + EXPIRATION_TIME, + maybe_max_merged_line_bytes, + ) } // internal function that allows customizing the expiration time (for testing) @@ -92,6 +145,7 @@ fn merge_partial_events_with_custom_expiration( stream: impl Stream + 'static, log_namespace: LogNamespace, expiration_time: Duration, + maybe_max_merged_line_bytes: Option, ) -> impl Stream { let partial_flag_path = match log_namespace { LogNamespace::Vector => { @@ -109,6 +163,7 @@ fn merge_partial_events_with_custom_expiration( let state = PartialEventMergeState { buckets: HashMap::new(), + maybe_max_merged_line_bytes, }; let message_path = get_message_path(log_namespace); @@ -164,7 +219,7 @@ mod test { e_1.insert("foo", 1); let input_stream = futures::stream::iter([e_1.into()]); - let output_stream = merge_partial_events(input_stream, LogNamespace::Legacy); + let output_stream = merge_partial_events(input_stream, LogNamespace::Legacy, None); let output: Vec = output_stream.collect().await; assert_eq!(output.len(), 1); @@ -174,6 +229,18 @@ mod test { ); } + #[tokio::test] + async fn merge_single_event_legacy_exceeds_max_merged_line_limit() { + let mut e_1 = LogEvent::from("test message 1"); + e_1.insert("foo", 1); + + let input_stream = futures::stream::iter([e_1.into()]); + let output_stream = merge_partial_events(input_stream, LogNamespace::Legacy, Some(1)); + + let output: Vec = output_stream.collect().await; + assert_eq!(output.len(), 0); + } + #[tokio::test] async fn merge_multiple_events_legacy() { let mut e_1 = LogEvent::from("test message 1"); @@ -184,7 +251,7 @@ mod test { e_2.insert("foo2", 1); let input_stream = futures::stream::iter([e_1.into(), e_2.into()]); - let output_stream = merge_partial_events(input_stream, LogNamespace::Legacy); + let output_stream = merge_partial_events(input_stream, LogNamespace::Legacy, None); let output: Vec = output_stream.collect().await; assert_eq!(output.len(), 1); @@ -194,6 +261,23 @@ mod test { ); } + #[tokio::test] + async fn merge_multiple_events_legacy_exceeds_max_merged_line_limit() { + let mut e_1 = LogEvent::from("test message 1"); + e_1.insert("foo", 1); + e_1.insert("_partial", true); + + let mut e_2 = LogEvent::from("test message 2"); + e_2.insert("foo2", 1); + + let input_stream = futures::stream::iter([e_1.into(), e_2.into()]); + // 24 > length of first message but less than the two combined + let output_stream = merge_partial_events(input_stream, LogNamespace::Legacy, Some(24)); + + let output: Vec = output_stream.collect().await; + assert_eq!(output.len(), 0); + } + #[tokio::test] async fn multiple_events_flush_legacy() { let mut e_1 = LogEvent::from("test message 1"); @@ -205,7 +289,7 @@ mod test { e_1.insert("_partial", true); let input_stream = futures::stream::iter([e_1.into(), e_2.into()]); - let output_stream = merge_partial_events(input_stream, LogNamespace::Legacy); + let output_stream = merge_partial_events(input_stream, LogNamespace::Legacy, None); let output: Vec = output_stream.collect().await; assert_eq!(output.len(), 1); @@ -215,6 +299,24 @@ mod test { ); } + #[tokio::test] + async fn multiple_events_flush_legacy_exceeds_max_merged_line_limit() { + let mut e_1 = LogEvent::from("test message 1"); + e_1.insert("foo", 1); + e_1.insert("_partial", true); + + let mut e_2 = LogEvent::from("test message 2"); + e_2.insert("foo2", 1); + e_1.insert("_partial", true); + + let input_stream = futures::stream::iter([e_1.into(), e_2.into()]); + // 24 > length of first message but less than the two combined + let output_stream = merge_partial_events(input_stream, LogNamespace::Legacy, Some(24)); + + let output: Vec = output_stream.collect().await; + assert_eq!(output.len(), 0); + } + #[tokio::test] async fn multiple_events_expire_legacy() { let mut e_1 = LogEvent::from("test message"); @@ -233,6 +335,7 @@ mod test { input_stream, LogNamespace::Legacy, Duration::from_secs(1), + None, ); let output: Vec = output_stream.take(2).collect().await; @@ -256,7 +359,7 @@ mod test { ); let input_stream = futures::stream::iter([e_1.into()]); - let output_stream = merge_partial_events(input_stream, LogNamespace::Vector); + let output_stream = merge_partial_events(input_stream, LogNamespace::Vector, None); let output: Vec = output_stream.collect().await; assert_eq!(output.len(), 1); @@ -286,7 +389,7 @@ mod test { ); let input_stream = futures::stream::iter([e_1.into(), e_2.into()]); - let output_stream = merge_partial_events(input_stream, LogNamespace::Vector); + let output_stream = merge_partial_events(input_stream, LogNamespace::Vector, None); let output: Vec = output_stream.collect().await; assert_eq!(output.len(), 1); diff --git a/website/cue/reference/components/sources/base/kubernetes_logs.cue b/website/cue/reference/components/sources/base/kubernetes_logs.cue index 5895f56908506..f07e3b317e5a0 100644 --- a/website/cue/reference/components/sources/base/kubernetes_logs.cue +++ b/website/cue/reference/components/sources/base/kubernetes_logs.cue @@ -191,6 +191,19 @@ base: components: sources: kubernetes_logs: configuration: { unit: "bytes" } } + max_merged_line_bytes: { + description: """ + The maximum number of bytes a line can contain - after merging - before being discarded. + + This protects against malformed lines or tailing incorrect files. + + Note that, if auto_partial_merge is false, this config will be ignored. Also, if max_line_bytes is too small to reach the continuation character, then this + config will have no practical impact (the same is true of `auto_partial_merge`). Finally, the smaller of `max_merged_line_bytes` and `max_line_bytes` will apply + if auto_partial_merge is true, so if this is set to be 1 MiB, for example, but `max_line_bytes` is set to ~2.5 MiB, then every line greater than 1 MiB will be dropped. + """ + required: false + type: uint: unit: "bytes" + } max_read_bytes: { description: """ Max amount of bytes to read from a single file before switching over to the next file. From 7205b4a7273ec4480d6e03375df01b29002df504 Mon Sep 17 00:00:00 2001 From: Aram Peres <6775216+aramperes@users.noreply.github.com> Date: Fri, 13 Jun 2025 09:55:42 -0400 Subject: [PATCH 061/305] fix(amqp sink): attempt one reconnect when channel has errored (#22971) * fix(amqp sink): attempt one reconnect when channel has errored * add changelog.d * fix changelog.d * Use vector_common::Error * use deadpool to manage channel recovery and multiple channels * Rename AmqpSink * Update integration tests * make amqp max_channels configurable; default to 4; added to docs and changelog * dupe word * Update 3rd party licenses for deadpool * Fix formatting in amqp.cue * Update DEVELOPING.md --- Cargo.lock | 17 +++- Cargo.toml | 3 +- LICENSE-3rdparty.csv | 2 + .../22971_reconnect_broken_amqp_sink.fix.md | 3 + docs/DEVELOPING.md | 4 +- src/sinks/amqp/channel.rs | 86 +++++++++++++++++++ src/sinks/amqp/config.rs | 20 ++++- src/sinks/amqp/integration_tests.rs | 7 +- src/sinks/amqp/mod.rs | 5 +- src/sinks/amqp/service.rs | 26 ++++-- src/sinks/amqp/sink.rs | 23 ++--- .../reference/components/sinks/base/amqp.cue | 5 ++ 12 files changed, 163 insertions(+), 38 deletions(-) create mode 100644 changelog.d/22971_reconnect_broken_amqp_sink.fix.md create mode 100644 src/sinks/amqp/channel.rs diff --git a/Cargo.lock b/Cargo.lock index 9698a0f4e00fb..0b1e6d2fd327e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2975,11 +2975,25 @@ dependencies = [ "tokio", ] +[[package]] +name = "deadpool" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ed5957ff93768adf7a65ab167a17835c3d2c3c50d084fe305174c112f468e2f" +dependencies = [ + "deadpool-runtime", + "num_cpus", + "tokio", +] + [[package]] name = "deadpool-runtime" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63dfa964fe2a66f3fde91fc70b267fe193d822c7e603e2a675a49a7f46ad3f49" +dependencies = [ + "tokio", +] [[package]] name = "der" @@ -11475,6 +11489,7 @@ dependencies = [ "crossterm 0.29.0", "csv", "databend-client", + "deadpool 0.12.2", "derivative", "dirs-next", "dnsmsg-parser", @@ -12771,7 +12786,7 @@ dependencies = [ "assert-json-diff", "async-trait", "base64 0.22.1", - "deadpool", + "deadpool 0.10.0", "futures 0.3.31", "http 1.1.0", "http-body-util", diff --git a/Cargo.toml b/Cargo.toml index e705a9f37d613..089f436bcc8f2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -290,6 +290,7 @@ smpl_jwt = { version = "0.8.0", default-features = false, optional = true } # AMQP lapin = { version = "2.5.3", default-features = false, features = ["native-tls"], optional = true } +deadpool = { version = "0.12.2", default-features = false, features = ["managed", "rt_tokio_1"], optional = true } # API async-graphql = { version = "7.0.17", default-features = false, optional = true, features = ["chrono", "playground"] } @@ -784,7 +785,7 @@ sinks-metrics = [ "sinks-splunk_hec" ] -sinks-amqp = ["lapin"] +sinks-amqp = ["deadpool", "lapin"] sinks-appsignal = [] sinks-aws_cloudwatch_logs = ["aws-core", "dep:aws-sdk-cloudwatchlogs", "dep:aws-sdk-kms"] sinks-aws_cloudwatch_metrics = ["aws-core", "dep:aws-sdk-cloudwatch"] diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index e107729c35ddc..aaaaac24bed82 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -180,6 +180,7 @@ data-encoding,https://github.com/ia0/data-encoding,MIT,Julien Cretin databend-client,https://github.com/databendlabs/bendsql,Apache-2.0,Databend Authors dbl,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers +deadpool,https://github.com/bikeshedder/deadpool,MIT OR Apache-2.0,Michael P. Jung der,https://github.com/RustCrypto/formats/tree/master/der,Apache-2.0 OR MIT,RustCrypto Developers deranged,https://github.com/jhpratt/deranged,MIT OR Apache-2.0,Jacob Pratt derivative,https://github.com/mcarton/rust-derivative,MIT OR Apache-2.0,mcarton @@ -423,6 +424,7 @@ num-integer,https://github.com/rust-num/num-integer,MIT OR Apache-2.0,The Rust P num-iter,https://github.com/rust-num/num-iter,MIT OR Apache-2.0,The Rust Project Developers num-rational,https://github.com/rust-num/num-rational,MIT OR Apache-2.0,The Rust Project Developers num-traits,https://github.com/rust-num/num-traits,MIT OR Apache-2.0,The Rust Project Developers +num_cpus,https://github.com/seanmonstar/num_cpus,MIT OR Apache-2.0,Sean McArthur num_enum,https://github.com/illicitonion/num_enum,BSD-3-Clause OR MIT OR Apache-2.0,"Daniel Wagner-Hall , Daniel Henry-Mantilla , Vincent Esche " num_threads,https://github.com/jhpratt/num_threads,MIT OR Apache-2.0,Jacob Pratt number_prefix,https://github.com/ogham/rust-number-prefix,MIT,Benjamin Sago diff --git a/changelog.d/22971_reconnect_broken_amqp_sink.fix.md b/changelog.d/22971_reconnect_broken_amqp_sink.fix.md new file mode 100644 index 0000000000000..acd01e4280f13 --- /dev/null +++ b/changelog.d/22971_reconnect_broken_amqp_sink.fix.md @@ -0,0 +1,3 @@ +The `amqp` sink now attempts to re-connect to the AMQP broker when the channel has been disconnected. It will also create up to 4 channels in a pool (configurable with the `max_channels` setting) to improve throughput. + +authors: aramperes diff --git a/docs/DEVELOPING.md b/docs/DEVELOPING.md index 7b9116a2a806b..356f780e54cb6 100644 --- a/docs/DEVELOPING.md +++ b/docs/DEVELOPING.md @@ -126,7 +126,7 @@ Loosely, you'll need the following: - **To run integration tests:** Have `docker` available, or a real live version of that service. (Use `AUTOSPAWN=false`) - **To run `make check-component-features`:** Have `remarshal` installed. - **To run `make check-licenses` or `cargo vdev build licenses`:** Have `dd-rust-license-tool` [installed](https://github.com/DataDog/rust-license-tool). -- **To run `cargo vdev build component-docs`:** Have `cue` [installed](https://cuelang.org/docs/install/). +- **To run `make generate-component-docs`:** Have `cue` [installed](https://cuelang.org/docs/install/). If you find yourself needing to run something inside the Docker environment described above, that's totally fine, they won't collide or hurt each other. In this case, you'd just run `make environment-generate`. @@ -161,7 +161,7 @@ cargo bench transforms::example make fmt cargo fmt # Build component documentation for the website -cargo vdev build component-docs +make generate-component-docs ``` If you run `make` you'll see a full list of all our tasks. Some of these will start Docker containers, sign commits, or even make releases. These are not common development commands and your mileage may vary. diff --git a/src/sinks/amqp/channel.rs b/src/sinks/amqp/channel.rs new file mode 100644 index 0000000000000..862ff06e5d0dc --- /dev/null +++ b/src/sinks/amqp/channel.rs @@ -0,0 +1,86 @@ +use super::config::AmqpSinkConfig; +use super::service::AmqpError; +use crate::amqp::AmqpConfig; +use deadpool::managed::Pool; +use lapin::options::ConfirmSelectOptions; + +pub type AmqpSinkChannels = Pool; + +pub(super) fn new_channel_pool(config: &AmqpSinkConfig) -> crate::Result { + let max_channels = config.max_channels.try_into().map_err(|_| { + Box::new(AmqpError::PoolError { + error: "max_channels must fit into usize".into(), + }) + })?; + if max_channels == 0 { + return Err(Box::new(AmqpError::PoolError { + error: "max_channels must be positive".into(), + })); + } + let channel_manager = AmqpSinkChannelManager::new(&config.connection); + let channels = Pool::builder(channel_manager) + .max_size(max_channels) + .runtime(deadpool::Runtime::Tokio1) + .build()?; + debug!("AMQP channel pool created with max size: {}", max_channels); + Ok(channels) +} + +/// A channel pool manager for the AMQP sink. +/// This manager is responsible for creating and recycling AMQP channels. +/// It uses the `deadpool` crate to manage the channels. +pub(crate) struct AmqpSinkChannelManager { + config: AmqpConfig, +} + +impl deadpool::managed::Manager for AmqpSinkChannelManager { + type Type = lapin::Channel; + type Error = AmqpError; + + async fn create(&self) -> Result { + let channel = Self::new_channel(&self.config).await?; + info!( + message = "Created a new channel to the AMQP broker.", + id = channel.id() + ); + Ok(channel) + } + + async fn recycle( + &self, + channel: &mut Self::Type, + _: &deadpool::managed::Metrics, + ) -> deadpool::managed::RecycleResult { + let state = channel.status().state(); + if state == lapin::ChannelState::Connected { + Ok(()) + } else { + Err((AmqpError::ChannelClosed { state }).into()) + } + } +} + +impl AmqpSinkChannelManager { + /// Creates a new channel pool manager for the AMQP sink. + pub fn new(config: &AmqpConfig) -> Self { + Self { + config: config.clone(), + } + } + + /// Creates a new AMQP channel using the configuration of this sink. + async fn new_channel(config: &AmqpConfig) -> Result { + let (_, channel) = config + .connect() + .await + .map_err(|e| AmqpError::ConnectFailed { error: e })?; + + // Enable confirmations on the channel. + channel + .confirm_select(ConfirmSelectOptions::default()) + .await + .map_err(|e| AmqpError::ConnectFailed { error: Box::new(e) })?; + + Ok(channel) + } +} diff --git a/src/sinks/amqp/config.rs b/src/sinks/amqp/config.rs index 50ea63104f836..afca8b47ccae5 100644 --- a/src/sinks/amqp/config.rs +++ b/src/sinks/amqp/config.rs @@ -1,7 +1,7 @@ //! Configuration functionality for the `AMQP` sink. +use super::channel::AmqpSinkChannels; use crate::{amqp::AmqpConfig, sinks::prelude::*}; use lapin::{types::ShortString, BasicProperties}; -use std::sync::Arc; use vector_lib::{ codecs::TextSerializerConfig, internal_event::{error_stage, error_type}, @@ -90,6 +90,14 @@ pub struct AmqpSinkConfig { skip_serializing_if = "crate::serde::is_default" )] pub(crate) acknowledgements: AcknowledgementsConfig, + + /// Maximum number of AMQP channels to keep active (channels are created as needed). + #[serde(default = "default_max_channels")] + pub(crate) max_channels: u32, +} + +const fn default_max_channels() -> u32 { + 4 } impl Default for AmqpSinkConfig { @@ -101,6 +109,7 @@ impl Default for AmqpSinkConfig { encoding: TextSerializerConfig::default().into(), connection: AmqpConfig::default(), acknowledgements: AcknowledgementsConfig::default(), + max_channels: default_max_channels(), } } } @@ -111,7 +120,8 @@ impl GenerateConfig for AmqpSinkConfig { r#"connection_string = "amqp://localhost:5672/%2f" routing_key = "user_id" exchange = "test" - encoding.codec = "json""#, + encoding.codec = "json" + max_channels = 4"#, ) .unwrap() } @@ -122,7 +132,7 @@ impl GenerateConfig for AmqpSinkConfig { impl SinkConfig for AmqpSinkConfig { async fn build(&self, _cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { let sink = AmqpSink::new(self.clone()).await?; - let hc = healthcheck(Arc::clone(&sink.channel)).boxed(); + let hc = healthcheck(sink.channels.clone()).boxed(); Ok((VectorSink::from_event_streamsink(sink), hc)) } @@ -135,9 +145,11 @@ impl SinkConfig for AmqpSinkConfig { } } -pub(super) async fn healthcheck(channel: Arc) -> crate::Result<()> { +pub(super) async fn healthcheck(channels: AmqpSinkChannels) -> crate::Result<()> { trace!("Healthcheck started."); + let channel = channels.get().await?; + if !channel.status().connected() { return Err(Box::new(std::io::Error::new( std::io::ErrorKind::BrokenPipe, diff --git a/src/sinks/amqp/integration_tests.rs b/src/sinks/amqp/integration_tests.rs index e919c5b88e92e..c974dbccb87cc 100644 --- a/src/sinks/amqp/integration_tests.rs +++ b/src/sinks/amqp/integration_tests.rs @@ -3,6 +3,7 @@ use crate::{ amqp::await_connection, config::{SinkConfig, SinkContext}, shutdown::ShutdownSignal, + sinks::amqp::channel::new_channel_pool, template::{Template, UnsignedIntTemplate}, test_util::{ components::{run_and_assert_sink_compliance, SINK_TAGS}, @@ -12,7 +13,7 @@ use crate::{ }; use config::AmqpPropertiesConfig; use futures::StreamExt; -use std::{collections::HashSet, sync::Arc, time::Duration}; +use std::{collections::HashSet, time::Duration}; use vector_lib::{config::LogNamespace, event::LogEvent}; pub fn make_config() -> AmqpSinkConfig { @@ -37,8 +38,8 @@ async fn healthcheck() { let mut config = make_config(); config.exchange = Template::try_from(exchange.as_str()).unwrap(); await_connection(&config.connection).await; - let (_conn, channel) = config.connection.connect().await.unwrap(); - super::config::healthcheck(Arc::new(channel)).await.unwrap(); + let channels = new_channel_pool(&config).unwrap(); + super::config::healthcheck(channels).await.unwrap(); } #[tokio::test] diff --git a/src/sinks/amqp/mod.rs b/src/sinks/amqp/mod.rs index 749f892f1ccd2..6b478c4adc10d 100644 --- a/src/sinks/amqp/mod.rs +++ b/src/sinks/amqp/mod.rs @@ -1,5 +1,6 @@ //! `AMQP` sink. //! Handles version AMQP 0.9.1 which is used by RabbitMQ. +mod channel; mod config; mod encoder; mod request_builder; @@ -15,7 +16,5 @@ use snafu::Snafu; #[derive(Debug, Snafu)] enum BuildError { #[snafu(display("creating amqp producer failed: {}", source))] - AmqpCreateFailed { - source: Box, - }, + AmqpCreateFailed { source: vector_common::Error }, } diff --git a/src/sinks/amqp/service.rs b/src/sinks/amqp/service.rs index 44c475f208cae..cee7d20344bd7 100644 --- a/src/sinks/amqp/service.rs +++ b/src/sinks/amqp/service.rs @@ -5,10 +5,9 @@ use bytes::Bytes; use futures::future::BoxFuture; use lapin::{options::BasicPublishOptions, BasicProperties}; use snafu::Snafu; -use std::{ - sync::Arc, - task::{Context, Poll}, -}; +use std::task::{Context, Poll}; + +use super::channel::AmqpSinkChannels; /// The request contains the data to send to `AMQP` together /// with the information need to route the message. @@ -79,11 +78,11 @@ impl DriverResponse for AmqpResponse { /// The tower service that handles the actual sending of data to `AMQP`. pub(super) struct AmqpService { - pub(super) channel: Arc, + pub(super) channels: AmqpSinkChannels, } #[derive(Debug, Snafu)] -pub(super) enum AmqpError { +pub enum AmqpError { #[snafu(display("Failed retrieving Acknowledgement: {}", error))] AcknowledgementFailed { error: lapin::Error }, @@ -92,6 +91,15 @@ pub(super) enum AmqpError { #[snafu(display("Received Negative Acknowledgement from AMQP broker."))] Nack, + + #[snafu(display("Failed to open AMQP channel: {}", error))] + ConnectFailed { error: vector_common::Error }, + + #[snafu(display("Channel is not writeable: {:?}", state))] + ChannelClosed { state: lapin::ChannelState }, + + #[snafu(display("Channel pool error: {}", error))] + PoolError { error: vector_common::Error }, } impl Service for AmqpService { @@ -106,9 +114,13 @@ impl Service for AmqpService { } fn call(&mut self, req: AmqpRequest) -> Self::Future { - let channel = Arc::clone(&self.channel); + let channel = self.channels.clone(); Box::pin(async move { + let channel = channel.get().await.map_err(|error| AmqpError::PoolError { + error: Box::new(error), + })?; + let byte_size = req.body.len(); let fut = channel .basic_publish( diff --git a/src/sinks/amqp/sink.rs b/src/sinks/amqp/sink.rs index 1537713be206c..ca40fe46b21e5 100644 --- a/src/sinks/amqp/sink.rs +++ b/src/sinks/amqp/sink.rs @@ -1,10 +1,10 @@ //! The sink for the `AMQP` sink that wires together the main stream that takes the //! event and sends it to `AMQP`. use crate::sinks::prelude::*; -use lapin::{options::ConfirmSelectOptions, BasicProperties}; +use lapin::BasicProperties; use serde::Serialize; -use std::sync::Arc; +use super::channel::AmqpSinkChannels; use super::{ config::{AmqpPropertiesConfig, AmqpSinkConfig}, encoder::AmqpEncoder, @@ -27,7 +27,7 @@ pub(super) struct AmqpEvent { } pub(super) struct AmqpSink { - pub(super) channel: Arc, + pub(super) channels: AmqpSinkChannels, exchange: Template, routing_key: Option