diff --git a/packages/api/.env.e2e b/packages/api/.env.e2e index a637e1f22..bbc6d3f7d 100644 --- a/packages/api/.env.e2e +++ b/packages/api/.env.e2e @@ -16,5 +16,5 @@ OPAMP_PORT=24320 # Auto-create default connections and sources for new teams # Uses public demo ClickHouse instance with pre-populated data DEFAULT_CONNECTIONS='[{"name":"local","host":"https://sql-clickhouse.clickhouse.com","username":"otel_demo","password":""}]' -DEFAULT_SOURCES='[{"kind":"log","name":"Demo Logs","connection":"local","from":{"databaseName":"otel_v2","tableName":"otel_logs"},"timestampValueExpression":"TimestampTime","defaultTableSelectExpression":"Timestamp, ServiceName, SeverityText, Body","serviceNameExpression":"ServiceName","severityTextExpression":"SeverityText","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"Body","displayedTimestampValueExpression":"Timestamp","sessionSourceId":"Demo Sessions","traceSourceId":"Demo Traces","metricSourceId":"Demo Metrics"},{"kind":"trace","name":"Demo Traces","connection":"local","from":{"databaseName":"otel_v2","tableName":"otel_traces"},"timestampValueExpression":"Timestamp","defaultTableSelectExpression":"Timestamp, ServiceName, StatusCode, round(Duration / 1e6), SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"SpanName","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanKindExpression":"SpanKind","spanNameExpression":"SpanName","logSourceId":"Demo Logs","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","metricSourceId":"Demo Metrics","sessionSourceId":"Demo Sessions"},{"kind":"metric","name":"Demo Metrics","connection":"local","from":{"databaseName":"otel_v2","tableName":""},"timestampValueExpression":"TimeUnix","serviceNameExpression":"ServiceName","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponential histogram":"otel_metrics_exponential_histogram"},"resourceAttributesExpression":"ResourceAttributes","logSourceId":"Demo Logs"},{"kind":"session","name":"Demo Sessions","connection":"local","from":{"databaseName":"otel_v2","tableName":"hyperdx_sessions"},"timestampValueExpression":"TimestampTime","defaultTableSelectExpression":"Timestamp, ServiceName, Body","serviceNameExpression":"ServiceName","severityTextExpression":"SeverityText","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","traceSourceId":"Demo Traces","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"Body"},{"kind":"trace","name":"ClickPy Traces","connection":"local","from":{"databaseName":"otel_clickpy","tableName":"otel_traces"},"timestampValueExpression":"Timestamp","defaultTableSelectExpression":"Timestamp, ServiceName, StatusCode, round(Duration / 1e6), SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"SpanName","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanKindExpression":"SpanKind","spanNameExpression":"SpanName","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","highlightedTraceAttributeExpressions":[{"sqlExpression":"if((SpanAttributes['http.route']) LIKE '%dashboard%', concat('https://clickpy.clickhouse.com', path(SpanAttributes['http.target'])), '')","alias":"clickpy_link"}],"sessionSourceId":"ClickPy Sessions"},{"kind":"session","name":"ClickPy Sessions","connection":"local","from":{"databaseName":"otel_clickpy","tableName":"hyperdx_sessions"},"timestampValueExpression":"TimestampTime","defaultTableSelectExpression":"Timestamp, ServiceName, Body","serviceNameExpression":"ServiceName","severityTextExpression":"SeverityText","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","traceSourceId":"ClickPy Traces","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"Body"}]' +DEFAULT_SOURCES='[{"kind":"log","name":"Demo Logs","connection":"local","from":{"databaseName":"otel_v2","tableName":"otel_logs"},"timestampValueExpression":"TimestampTime","defaultTableSelectExpression":"Timestamp, ServiceName, SeverityText, Body","serviceNameExpression":"ServiceName","severityTextExpression":"SeverityText","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"Body","displayedTimestampValueExpression":"Timestamp","sessionSourceId":"Demo Sessions","traceSourceId":"Demo Traces","metricSourceId":"Demo Metrics"},{"kind":"trace","name":"Demo Traces","connection":"local","from":{"databaseName":"otel_v2","tableName":"otel_traces"},"timestampValueExpression":"Timestamp","defaultTableSelectExpression":"Timestamp, ServiceName, StatusCode, round(Duration / 1e6), SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"SpanName","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanKindExpression":"SpanKind","spanNameExpression":"SpanName","logSourceId":"Demo Logs","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","metricSourceId":"Demo Metrics","sessionSourceId":"Demo Sessions","materializedViews": [{"databaseName": "otel_v2","tableName": "otel_traces_1m","dimensionColumns": "ServiceName, StatusCode","minGranularity": "1 minute","timestampColumn": "Timestamp","aggregatedColumns": [{"mvColumn": "count","aggFn": "count","sourceColumn": ""},{"mvColumn": "max__Duration","aggFn": "max","sourceColumn": "Duration"},{"mvColumn": "avg__Duration","aggFn": "avg","sourceColumn": "Duration"}]},{"databaseName": "otel_v2","tableName": "otel_traces_1m_v2","dimensionColumns": "ServiceName, SpanName, SpanKind","minGranularity": "1 minute","timestampColumn": "Timestamp","aggregatedColumns": [{"mvColumn": "count","aggFn": "count","sourceColumn": ""},{"mvColumn": "max__Duration","aggFn": "max","sourceColumn": "Duration"},{"mvColumn": "avg__Duration","aggFn": "avg","sourceColumn": "Duration"},{"mvColumn": "quantile__Duration","aggFn": "quantile","sourceColumn": "Duration"}]}]},{"kind":"metric","name":"Demo Metrics","connection":"local","from":{"databaseName":"otel_v2","tableName":""},"timestampValueExpression":"TimeUnix","serviceNameExpression":"ServiceName","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponential histogram":"otel_metrics_exponential_histogram"},"resourceAttributesExpression":"ResourceAttributes","logSourceId":"Demo Logs"},{"kind":"session","name":"Demo Sessions","connection":"local","from":{"databaseName":"otel_v2","tableName":"hyperdx_sessions"},"timestampValueExpression":"TimestampTime","defaultTableSelectExpression":"Timestamp, ServiceName, Body","serviceNameExpression":"ServiceName","severityTextExpression":"SeverityText","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","traceSourceId":"Demo Traces","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"Body"},{"kind":"trace","name":"ClickPy Traces","connection":"local","from":{"databaseName":"otel_clickpy","tableName":"otel_traces"},"timestampValueExpression":"Timestamp","defaultTableSelectExpression":"Timestamp, ServiceName, StatusCode, round(Duration / 1e6), SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"SpanName","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanKindExpression":"SpanKind","spanNameExpression":"SpanName","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","highlightedTraceAttributeExpressions":[{"sqlExpression":"if((SpanAttributes['http.route']) LIKE '%dashboard%', concat('https://clickpy.clickhouse.com', path(SpanAttributes['http.target'])), '')","alias":"clickpy_link"}],"sessionSourceId":"ClickPy Sessions"},{"kind":"session","name":"ClickPy Sessions","connection":"local","from":{"databaseName":"otel_clickpy","tableName":"hyperdx_sessions"},"timestampValueExpression":"TimestampTime","defaultTableSelectExpression":"Timestamp, ServiceName, Body","serviceNameExpression":"ServiceName","severityTextExpression":"SeverityText","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","traceSourceId":"ClickPy Traces","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"Body"}]' diff --git a/packages/app/src/components/OnboardingModal.tsx b/packages/app/src/components/OnboardingModal.tsx index 15508b07a..a493965f0 100644 --- a/packages/app/src/components/OnboardingModal.tsx +++ b/packages/app/src/components/OnboardingModal.tsx @@ -40,6 +40,7 @@ async function addOtelDemoSources({ traceSourceName, traceSourceTableName, traceSourceHighlightedTraceAttributes, + traceSourceMaterializedViews, }: { connectionId: string; createSourceMutation: ReturnType; @@ -62,6 +63,7 @@ async function addOtelDemoSources({ traceSourceName: string; traceSourceTableName: string; traceSourceHighlightedTraceAttributes?: TSource['highlightedTraceAttributeExpressions']; + traceSourceMaterializedViews?: TSource['materializedViews']; }) { const hasLogSource = logSourceDatabaseName && logSourceName && logSourceTableName; @@ -121,6 +123,7 @@ async function addOtelDemoSources({ spanEventsValueExpression: 'Events', highlightedTraceAttributeExpressions: traceSourceHighlightedTraceAttributes, + materializedViews: traceSourceMaterializedViews, }, }); let metricsSource: TSource | undefined; @@ -287,6 +290,53 @@ function OnboardingModalComponent({ traceSourceDatabaseName: 'otel_v2', traceSourceName: 'Demo Traces', traceSourceTableName: 'otel_traces', + traceSourceMaterializedViews: [ + { + databaseName: 'otel_v2', + tableName: 'otel_traces_1m', + dimensionColumns: 'ServiceName, StatusCode', + minGranularity: '1 minute', + timestampColumn: 'Timestamp', + aggregatedColumns: [ + { mvColumn: 'count', aggFn: 'count', sourceColumn: '' }, + { + mvColumn: 'max__Duration', + aggFn: 'max', + sourceColumn: 'Duration', + }, + { + mvColumn: 'avg__Duration', + aggFn: 'avg', + sourceColumn: 'Duration', + }, + ], + }, + { + databaseName: 'otel_v2', + tableName: 'otel_traces_1m_v2', + dimensionColumns: 'ServiceName, SpanName, SpanKind', + minGranularity: '1 minute', + timestampColumn: 'Timestamp', + aggregatedColumns: [ + { mvColumn: 'count', aggFn: 'count', sourceColumn: '' }, + { + mvColumn: 'max__Duration', + aggFn: 'max', + sourceColumn: 'Duration', + }, + { + mvColumn: 'avg__Duration', + aggFn: 'avg', + sourceColumn: 'Duration', + }, + { + mvColumn: 'quantile__Duration', + aggFn: 'quantile', + sourceColumn: 'Duration', + }, + ], + }, + ], updateSourceMutation, }); diff --git a/packages/app/src/components/SearchTotalCountChart.tsx b/packages/app/src/components/SearchTotalCountChart.tsx index fec663247..f7604b7a9 100644 --- a/packages/app/src/components/SearchTotalCountChart.tsx +++ b/packages/app/src/components/SearchTotalCountChart.tsx @@ -1,4 +1,9 @@ import { useMemo } from 'react'; +import { + filterColumnMetaByType, + JSDataType, + ResponseJSON, +} from '@hyperdx/common-utils/dist/clickhouse'; import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { Text } from '@mantine/core'; import { keepPreviousData } from '@tanstack/react-query'; @@ -7,6 +12,18 @@ import api from '@/api'; import { convertToTimeChartConfig } from '@/ChartUtils'; import { useQueriedChartConfig } from '@/hooks/useChartConfig'; +function inferCountColumn(meta: ResponseJSON['meta'] | undefined): string { + if (!meta) return 'count()'; + if (meta.find(col => col.name === 'count()')) { + return 'count()'; + } + + // The column may be named differently, particularly when using Materialized Views. + return ( + filterColumnMetaByType(meta, [JSDataType.Number])?.[0].name ?? 'count()' + ); +} + export function useSearchTotalCount( config: ChartConfigWithDateRange, queryKeyPrefix: string, @@ -50,8 +67,9 @@ export function useSearchTotalCount( const isTotalCountComplete = !!totalCountData?.isComplete; const totalCount = useMemo(() => { + const countColumn = inferCountColumn(totalCountData?.meta); return totalCountData?.data?.reduce( - (p: number, v: any) => p + Number.parseInt(v['count()']), + (p: number, v: any) => p + Number.parseInt(v[countColumn]), 0, ); }, [totalCountData]);