Skip to content

Commit 00c4b71

Browse files
author
Devdutt Shenoi
authored
Merge branch 'main' into hottier-perf
2 parents 1b41e6b + 4af4e6c commit 00c4b71

File tree

9 files changed

+111
-444
lines changed

9 files changed

+111
-444
lines changed

helm/templates/service-monitor.yaml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,10 @@ kind: ServiceMonitor
44
metadata:
55
name: {{ include "parseable.fullname" . }}
66
namespace: {{ default .Release.Namespace .Values.parseable.metrics.serviceMonitor.namespace | quote }}
7-
labels:
7+
labels:
8+
{{- with .Values.parseable.metrics.serviceMonitor.labels }}
9+
{{- toYaml . | nindent 4 }}
10+
{{- end }}
811
{{- include "parseable.labels" . | nindent 4 }}
912
spec:
1013
{{ if .Values.parseable.metrics.serviceMonitor.spec.jobLabel }}

helm/values.yaml

Lines changed: 15 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ parseable:
33
repository: containers.parseable.com/parseable/parseable
44
tag: v1.6.3
55
pullPolicy: Always
6-
## object store can be local, s3 or blob.
6+
## object store can be local-store, s3-store or blob-store.
77
## local needs to be false if set to object store.
88
store: local-store
99
## Set to true if you want to deploy Parseable in a HA mode (multiple ingestors)
@@ -192,6 +192,7 @@ parseable:
192192
metrics:
193193
serviceMonitor:
194194
enabled: false
195+
labels: {}
195196
namespace: ""
196197
spec:
197198
jobLabel: ""
@@ -300,7 +301,7 @@ fluent-bit:
300301
replicaCount: 1
301302
image:
302303
repository: parseable/fluent-bit
303-
tag: "v1"
304+
tag: "v2"
304305
pullPolicy: Always
305306
testFramework:
306307
enabled: true
@@ -382,25 +383,21 @@ fluent-bit:
382383
[OUTPUT]
383384
Name parseable
384385
Match kube.*
385-
P_Server parseable.parseable.svc.cluster.local
386-
P_Port 80
387-
P_Username admin
388-
P_Password admin
389-
P_Stream $NAMESPACE
386+
Server_Host parseable.parseable.svc.cluster.local
387+
Username admin
388+
Password admin
389+
Server_Port 80
390+
Stream $NAMESPACE
391+
Exclude_Namespaces kube-system, default
390392
391393
[OUTPUT]
392-
Name http
394+
Name parseable
393395
Match k8s_events
394-
host parseable.parseable.svc.cluster.local
395-
http_User admin
396-
http_Passwd admin
397-
format json
398-
port 80
399-
header Content-Type application/json
400-
header X-P-Stream k8s-events
401-
uri /api/v1/ingest
402-
json_date_key timestamp
403-
json_date_format iso8601
396+
Server_Host parseable.parseable.svc.cluster.local
397+
Server_Port 80
398+
Username admin
399+
Password admin
400+
Stream k8s-events
404401
405402
upstream: {}
406403

src/handlers/http/ingest.rs

Lines changed: 7 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818

1919
use super::logstream::error::{CreateStreamError, StreamError};
2020
use super::modal::utils::ingest_utils::{flatten_and_push_logs, push_logs};
21-
use super::otel;
2221
use super::users::dashboards::DashboardError;
2322
use super::users::filters::FiltersError;
2423
use crate::event::{
@@ -27,7 +26,7 @@ use crate::event::{
2726
format::{self, EventFormat},
2827
};
2928
use crate::handlers::http::modal::utils::logstream_utils::create_stream_and_schema_from_storage;
30-
use crate::handlers::{LOG_SOURCE_KEY, LOG_SOURCE_OTEL, STREAM_NAME_HEADER_KEY};
29+
use crate::handlers::STREAM_NAME_HEADER_KEY;
3130
use crate::localcache::CacheError;
3231
use crate::metadata::error::stream_info::MetadataError;
3332
use crate::metadata::STREAM_INFO;
@@ -115,25 +114,7 @@ pub async fn ingest_otel_logs(req: HttpRequest, body: Bytes) -> Result<HttpRespo
115114
{
116115
let stream_name = stream_name.to_str().unwrap().to_owned();
117116
create_stream_if_not_exists(&stream_name, &StreamType::UserDefined.to_string()).await?;
118-
119-
//flatten logs
120-
if let Some((_, log_source)) = req.headers().iter().find(|&(key, _)| key == LOG_SOURCE_KEY)
121-
{
122-
let log_source: String = log_source.to_str().unwrap().to_owned();
123-
if log_source == LOG_SOURCE_OTEL {
124-
let mut json = otel::flatten_otel_logs(&body);
125-
for record in json.iter_mut() {
126-
let body: Bytes = serde_json::to_vec(record).unwrap().into();
127-
push_logs(stream_name.to_string(), req.clone(), body).await?;
128-
}
129-
} else {
130-
return Err(PostError::CustomError("Unknown log source".to_string()));
131-
}
132-
} else {
133-
return Err(PostError::CustomError(
134-
"log source key header is missing".to_string(),
135-
));
136-
}
117+
push_logs(stream_name.to_string(), req.clone(), body).await?;
137118
} else {
138119
return Err(PostError::Header(ParseHeaderError::MissingStreamName));
139120
}
@@ -293,9 +274,7 @@ mod tests {
293274
use std::{collections::HashMap, sync::Arc};
294275

295276
use actix_web::test::TestRequest;
296-
use arrow_array::{
297-
types::Int64Type, ArrayRef, Float64Array, Int64Array, ListArray, StringArray,
298-
};
277+
use arrow_array::{ArrayRef, Float64Array, Int64Array, StringArray};
299278
use arrow_schema::{DataType, Field};
300279
use serde_json::json;
301280

@@ -689,25 +668,14 @@ mod tests {
689668
])
690669
);
691670

692-
let c_a = vec![None, None, Some(vec![Some(1i64)]), Some(vec![Some(1)])];
693-
let c_b = vec![None, None, None, Some(vec![Some(2i64)])];
694-
695671
assert_eq!(
696-
rb.column_by_name("c_a")
697-
.unwrap()
698-
.as_any()
699-
.downcast_ref::<ListArray>()
700-
.unwrap(),
701-
&ListArray::from_iter_primitive::<Int64Type, _, _>(c_a)
672+
rb.column_by_name("c_a").unwrap().as_int64_arr(),
673+
&Int64Array::from(vec![None, None, Some(1), Some(1)])
702674
);
703675

704676
assert_eq!(
705-
rb.column_by_name("c_b")
706-
.unwrap()
707-
.as_any()
708-
.downcast_ref::<ListArray>()
709-
.unwrap(),
710-
&ListArray::from_iter_primitive::<Int64Type, _, _>(c_b)
677+
rb.column_by_name("c_b").unwrap().as_int64_arr(),
678+
&Int64Array::from(vec![None, None, None, Some(2)])
711679
);
712680
}
713681
}

src/handlers/http/mod.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ pub mod logstream;
3636
pub mod middleware;
3737
pub mod modal;
3838
pub mod oidc;
39-
mod otel;
4039
pub mod query;
4140
pub mod rbac;
4241
pub mod role;

src/handlers/http/modal/utils/ingest_utils.rs

Lines changed: 13 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,7 @@
1616
*
1717
*/
1818

19-
use std::{
20-
collections::{BTreeMap, HashMap},
21-
sync::Arc,
22-
};
19+
use std::{collections::HashMap, sync::Arc};
2320

2421
use actix_web::HttpRequest;
2522
use arrow_schema::Field;
@@ -33,8 +30,8 @@ use crate::{
3330
format::{self, EventFormat},
3431
},
3532
handlers::{
36-
http::{ingest::PostError, kinesis, otel},
37-
LOG_SOURCE_KEY, LOG_SOURCE_KINESIS, LOG_SOURCE_OTEL, PREFIX_META, PREFIX_TAGS, SEPARATOR,
33+
http::{ingest::PostError, kinesis},
34+
LOG_SOURCE_KEY, LOG_SOURCE_KINESIS, PREFIX_META, PREFIX_TAGS, SEPARATOR,
3835
},
3936
metadata::STREAM_INFO,
4037
storage::StreamType,
@@ -46,26 +43,19 @@ pub async fn flatten_and_push_logs(
4643
body: Bytes,
4744
stream_name: String,
4845
) -> Result<(), PostError> {
49-
//flatten logs
50-
if let Some((_, log_source)) = req.headers().iter().find(|&(key, _)| key == LOG_SOURCE_KEY) {
51-
let mut json: Vec<BTreeMap<String, Value>> = Vec::new();
52-
let log_source: String = log_source.to_str().unwrap().to_owned();
53-
match log_source.as_str() {
54-
LOG_SOURCE_KINESIS => json = kinesis::flatten_kinesis_logs(&body),
55-
LOG_SOURCE_OTEL => {
56-
json = otel::flatten_otel_logs(&body);
57-
}
58-
_ => {
59-
log::warn!("Unknown log source: {}", log_source);
60-
push_logs(stream_name.to_string(), req.clone(), body).await?;
61-
}
62-
}
63-
for record in json.iter_mut() {
46+
let log_source = req
47+
.headers()
48+
.get(LOG_SOURCE_KEY)
49+
.map(|header| header.to_str().unwrap_or_default())
50+
.unwrap_or_default();
51+
if log_source == LOG_SOURCE_KINESIS {
52+
let json = kinesis::flatten_kinesis_logs(&body);
53+
for record in json.iter() {
6454
let body: Bytes = serde_json::to_vec(record).unwrap().into();
65-
push_logs(stream_name.to_string(), req.clone(), body).await?;
55+
push_logs(stream_name.clone(), req.clone(), body.clone()).await?;
6656
}
6757
} else {
68-
push_logs(stream_name.to_string(), req, body).await?;
58+
push_logs(stream_name, req, body).await?;
6959
}
7060
Ok(())
7161
}

0 commit comments

Comments
 (0)