feat: Add MVs to Demo Traces source config

packages/api/.env.e2e

@@ -16,5 +16,5 @@ OPAMP_PORT=24320
 # Auto-create default connections and sources for new teams
 # Uses public demo ClickHouse instance with pre-populated data
 DEFAULT_CONNECTIONS='[{"name":"local","host":"https://sql-clickhouse.clickhouse.com","username":"otel_demo","password":""}]'
-DEFAULT_SOURCES='[{"kind":"log","name":"Demo Logs","connection":"local","from":{"databaseName":"otel_v2","tableName":"otel_logs"},"timestampValueExpression":"TimestampTime","defaultTableSelectExpression":"Timestamp, ServiceName, SeverityText, Body","serviceNameExpression":"ServiceName","severityTextExpression":"SeverityText","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"Body","displayedTimestampValueExpression":"Timestamp","sessionSourceId":"Demo Sessions","traceSourceId":"Demo Traces","metricSourceId":"Demo Metrics"},{"kind":"trace","name":"Demo Traces","connection":"local","from":{"databaseName":"otel_v2","tableName":"otel_traces"},"timestampValueExpression":"Timestamp","defaultTableSelectExpression":"Timestamp, ServiceName, StatusCode, round(Duration / 1e6), SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"SpanName","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanKindExpression":"SpanKind","spanNameExpression":"SpanName","logSourceId":"Demo Logs","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","metricSourceId":"Demo Metrics","sessionSourceId":"Demo Sessions"},{"kind":"metric","name":"Demo Metrics","connection":"local","from":{"databaseName":"otel_v2","tableName":""},"timestampValueExpression":"TimeUnix","serviceNameExpression":"ServiceName","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponential histogram":"otel_metrics_exponential_histogram"},"resourceAttributesExpression":"ResourceAttributes","logSourceId":"Demo Logs"},{"kind":"session","name":"Demo Sessions","connection":"local","from":{"databaseName":"otel_v2","tableName":"hyperdx_sessions"},"timestampValueExpression":"TimestampTime","defaultTableSelectExpression":"Timestamp, ServiceName, Body","serviceNameExpression":"ServiceName","severityTextExpression":"SeverityText","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","traceSourceId":"Demo Traces","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"Body"},{"kind":"trace","name":"ClickPy Traces","connection":"local","from":{"databaseName":"otel_clickpy","tableName":"otel_traces"},"timestampValueExpression":"Timestamp","defaultTableSelectExpression":"Timestamp, ServiceName, StatusCode, round(Duration / 1e6), SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"SpanName","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanKindExpression":"SpanKind","spanNameExpression":"SpanName","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","highlightedTraceAttributeExpressions":[{"sqlExpression":"if((SpanAttributes['http.route']) LIKE '%dashboard%', concat('https://clickpy.clickhouse.com', path(SpanAttributes['http.target'])), '')","alias":"clickpy_link"}],"sessionSourceId":"ClickPy Sessions"},{"kind":"session","name":"ClickPy Sessions","connection":"local","from":{"databaseName":"otel_clickpy","tableName":"hyperdx_sessions"},"timestampValueExpression":"TimestampTime","defaultTableSelectExpression":"Timestamp, ServiceName, Body","serviceNameExpression":"ServiceName","severityTextExpression":"SeverityText","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","traceSourceId":"ClickPy Traces","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"Body"}]'
+DEFAULT_SOURCES='[{"kind":"log","name":"Demo Logs","connection":"local","from":{"databaseName":"otel_v2","tableName":"otel_logs"},"timestampValueExpression":"TimestampTime","defaultTableSelectExpression":"Timestamp, ServiceName, SeverityText, Body","serviceNameExpression":"ServiceName","severityTextExpression":"SeverityText","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"Body","displayedTimestampValueExpression":"Timestamp","sessionSourceId":"Demo Sessions","traceSourceId":"Demo Traces","metricSourceId":"Demo Metrics"},{"kind":"trace","name":"Demo Traces","connection":"local","from":{"databaseName":"otel_v2","tableName":"otel_traces"},"timestampValueExpression":"Timestamp","defaultTableSelectExpression":"Timestamp, ServiceName, StatusCode, round(Duration / 1e6), SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"SpanName","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanKindExpression":"SpanKind","spanNameExpression":"SpanName","logSourceId":"Demo Logs","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","metricSourceId":"Demo Metrics","sessionSourceId":"Demo Sessions","materializedViews": [{"databaseName": "otel_v2","tableName": "otel_traces_1m","dimensionColumns": "ServiceName, StatusCode","minGranularity": "1 minute","timestampColumn": "Timestamp","aggregatedColumns": [{"mvColumn": "count","aggFn": "count","sourceColumn": ""},{"mvColumn": "max__Duration","aggFn": "max","sourceColumn": "Duration"},{"mvColumn": "avg__Duration","aggFn": "avg","sourceColumn": "Duration"}]},{"databaseName": "otel_v2","tableName": "otel_traces_1m_v2","dimensionColumns": "ServiceName, SpanName, SpanKind","minGranularity": "1 minute","timestampColumn": "Timestamp","aggregatedColumns": [{"mvColumn": "count","aggFn": "count","sourceColumn": ""},{"mvColumn": "max__Duration","aggFn": "max","sourceColumn": "Duration"},{"mvColumn": "avg__Duration","aggFn": "avg","sourceColumn": "Duration"},{"mvColumn": "quantile__Duration","aggFn": "quantile","sourceColumn": "Duration"}]}]},{"kind":"metric","name":"Demo Metrics","connection":"local","from":{"databaseName":"otel_v2","tableName":""},"timestampValueExpression":"TimeUnix","serviceNameExpression":"ServiceName","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponential histogram":"otel_metrics_exponential_histogram"},"resourceAttributesExpression":"ResourceAttributes","logSourceId":"Demo Logs"},{"kind":"session","name":"Demo Sessions","connection":"local","from":{"databaseName":"otel_v2","tableName":"hyperdx_sessions"},"timestampValueExpression":"TimestampTime","defaultTableSelectExpression":"Timestamp, ServiceName, Body","serviceNameExpression":"ServiceName","severityTextExpression":"SeverityText","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","traceSourceId":"Demo Traces","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"Body"},{"kind":"trace","name":"ClickPy Traces","connection":"local","from":{"databaseName":"otel_clickpy","tableName":"otel_traces"},"timestampValueExpression":"Timestamp","defaultTableSelectExpression":"Timestamp, ServiceName, StatusCode, round(Duration / 1e6), SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"SpanName","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanKindExpression":"SpanKind","spanNameExpression":"SpanName","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","highlightedTraceAttributeExpressions":[{"sqlExpression":"if((SpanAttributes['http.route']) LIKE '%dashboard%', concat('https://clickpy.clickhouse.com', path(SpanAttributes['http.target'])), '')","alias":"clickpy_link"}],"sessionSourceId":"ClickPy Sessions"},{"kind":"session","name":"ClickPy Sessions","connection":"local","from":{"databaseName":"otel_clickpy","tableName":"hyperdx_sessions"},"timestampValueExpression":"TimestampTime","defaultTableSelectExpression":"Timestamp, ServiceName, Body","serviceNameExpression":"ServiceName","severityTextExpression":"SeverityText","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","traceSourceId":"ClickPy Traces","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"Body"}]'
 

packages/app/src/components/OnboardingModal.tsx

@@ -40,6 +40,7 @@ async function addOtelDemoSources({
   traceSourceName,
   traceSourceTableName,
   traceSourceHighlightedTraceAttributes,
+  traceSourceMaterializedViews,
 }: {
   connectionId: string;
   createSourceMutation: ReturnType<typeof useCreateSource>;
@@ -62,6 +63,7 @@ async function addOtelDemoSources({
   traceSourceName: string;
   traceSourceTableName: string;
   traceSourceHighlightedTraceAttributes?: TSource['highlightedTraceAttributeExpressions'];
+  traceSourceMaterializedViews?: TSource['materializedViews'];
 }) {
   const hasLogSource =
     logSourceDatabaseName && logSourceName && logSourceTableName;
@@ -121,6 +123,7 @@ async function addOtelDemoSources({
       spanEventsValueExpression: 'Events',
       highlightedTraceAttributeExpressions:
         traceSourceHighlightedTraceAttributes,
+      materializedViews: traceSourceMaterializedViews,
     },
   });
   let metricsSource: TSource | undefined;
@@ -287,6 +290,53 @@ function OnboardingModalComponent({
         traceSourceDatabaseName: 'otel_v2',
         traceSourceName: 'Demo Traces',
         traceSourceTableName: 'otel_traces',
+        traceSourceMaterializedViews: [
+          {
+            databaseName: 'otel_v2',
+            tableName: 'otel_traces_1m',
+            dimensionColumns: 'ServiceName, StatusCode',
+            minGranularity: '1 minute',
+            timestampColumn: 'Timestamp',
+            aggregatedColumns: [
+              { mvColumn: 'count', aggFn: 'count', sourceColumn: '' },
+              {
+                mvColumn: 'max__Duration',
+                aggFn: 'max',
+                sourceColumn: 'Duration',
+              },
+              {
+                mvColumn: 'avg__Duration',
+                aggFn: 'avg',
+                sourceColumn: 'Duration',
+              },
+            ],
+          },
+          {
+            databaseName: 'otel_v2',
+            tableName: 'otel_traces_1m_v2',
+            dimensionColumns: 'ServiceName, SpanName, SpanKind',
+            minGranularity: '1 minute',
+            timestampColumn: 'Timestamp',
+            aggregatedColumns: [
+              { mvColumn: 'count', aggFn: 'count', sourceColumn: '' },
+              {
+                mvColumn: 'max__Duration',
+                aggFn: 'max',
+                sourceColumn: 'Duration',
+              },
+              {
+                mvColumn: 'avg__Duration',
+                aggFn: 'avg',
+                sourceColumn: 'Duration',
+              },
+              {
+                mvColumn: 'quantile__Duration',
+                aggFn: 'quantile',
+                sourceColumn: 'Duration',
+              },
+            ],
+          },
+        ],
 
         updateSourceMutation,
       });

packages/app/src/components/SearchTotalCountChart.tsx

@@ -1,4 +1,9 @@
 import { useMemo } from 'react';
+import {
+  filterColumnMetaByType,
+  JSDataType,
+  ResponseJSON,
+} from '@hyperdx/common-utils/dist/clickhouse';
 import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
 import { Text } from '@mantine/core';
 import { keepPreviousData } from '@tanstack/react-query';
@@ -7,6 +12,18 @@ import api from '@/api';
 import { convertToTimeChartConfig } from '@/ChartUtils';
 import { useQueriedChartConfig } from '@/hooks/useChartConfig';
 
+function inferCountColumn(meta: ResponseJSON['meta'] | undefined): string {
+  if (!meta) return 'count()';
+  if (meta.find(col => col.name === 'count()')) {
+    return 'count()';
+  }
+
+  // The column may be named differently, particularly when using Materialized Views.
+  return (
+    filterColumnMetaByType(meta, [JSDataType.Number])?.[0].name ?? 'count()'
+  );
+}
+
 export function useSearchTotalCount(
   config: ChartConfigWithDateRange,
   queryKeyPrefix: string,
@@ -50,8 +67,9 @@ export function useSearchTotalCount(
   const isTotalCountComplete = !!totalCountData?.isComplete;
 
   const totalCount = useMemo(() => {
+    const countColumn = inferCountColumn(totalCountData?.meta);
     return totalCountData?.data?.reduce(
-      (p: number, v: any) => p + Number.parseInt(v['count()']),
+      (p: number, v: any) => p + Number.parseInt(v[countColumn]),
       0,
     );
   }, [totalCountData]);