diff --git a/.changeset/itchy-zebras-train.md b/.changeset/itchy-zebras-train.md new file mode 100644 index 000000000..695bd79b4 --- /dev/null +++ b/.changeset/itchy-zebras-train.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/api": patch +"@hyperdx/app": patch +--- + +feat: Align line/bar chart date ranges to chart granularity diff --git a/packages/app/src/ChartUtils.tsx b/packages/app/src/ChartUtils.tsx index 834cbffa5..6f04d266c 100644 --- a/packages/app/src/ChartUtils.tsx +++ b/packages/app/src/ChartUtils.tsx @@ -1,5 +1,5 @@ import { useMemo } from 'react'; -import { add } from 'date-fns'; +import { add, differenceInSeconds } from 'date-fns'; import { omit } from 'lodash'; import SqlString from 'sqlstring'; import { z } from 'zod'; @@ -135,20 +135,45 @@ export const DEFAULT_CHART_CONFIG: Omit< whereLanguage: 'lucene', displayType: DisplayType.Line, granularity: 'auto', + alignDateRangeToGranularity: true, }; export const isGranularity = (value: string): value is Granularity => { return Object.values(Granularity).includes(value as Granularity); }; +export function getAlignedDateRange( + [originalStart, originalEnd]: [Date, Date], + granularity: SQLInterval, +): [Date, Date] { + // Round the start time down to the previous interval boundary + const alignedStart = toStartOfInterval(originalStart, granularity); + + // Round the end time up to the next interval boundary + let alignedEnd = toStartOfInterval(originalEnd, granularity); + if (alignedEnd.getTime() < originalEnd.getTime()) { + const intervalSeconds = convertGranularityToSeconds(granularity); + alignedEnd = add(alignedEnd, { seconds: intervalSeconds }); + } + + return [alignedStart, alignedEnd]; +} + export function convertToTimeChartConfig(config: ChartConfigWithDateRange) { const granularity = config.granularity === 'auto' || config.granularity == null ? convertDateRangeToGranularityString(config.dateRange, 80) : config.granularity; + const dateRange = + config.alignDateRangeToGranularity === false + ? config.dateRange + : getAlignedDateRange(config.dateRange, granularity); + return { ...config, + dateRange, + dateRangeEndInclusive: false, granularity, limit: { limit: 100000 }, }; @@ -549,16 +574,9 @@ function inferGroupColumns(meta: Array<{ name: string; type: string }>) { ]); } -export function getPreviousPeriodOffsetSeconds( - dateRange: [Date, Date], -): number { - const [start, end] = dateRange; - return Math.round((end.getTime() - start.getTime()) / 1000); -} - export function getPreviousDateRange(currentRange: [Date, Date]): [Date, Date] { const [start, end] = currentRange; - const offsetSeconds = getPreviousPeriodOffsetSeconds(currentRange); + const offsetSeconds = differenceInSeconds(end, start); return [ new Date(start.getTime() - offsetSeconds * 1000), new Date(end.getTime() - offsetSeconds * 1000), @@ -622,7 +640,7 @@ function addResponseToFormattedData({ lineDataMap, tsBucketMap, source, - currentPeriodDateRange, + previousPeriodOffsetSeconds, isPreviousPeriod, hiddenSeries = [], }: { @@ -631,7 +649,7 @@ function addResponseToFormattedData({ response: ResponseJSON>; source?: TSource; isPreviousPeriod: boolean; - currentPeriodDateRange: [Date, Date]; + previousPeriodOffsetSeconds: number; hiddenSeries?: string[]; }) { const { meta, data } = response; @@ -655,9 +673,7 @@ function addResponseToFormattedData({ const date = new Date(row[timestampColumn.name]); // Previous period data needs to be shifted forward to align with current period - const offsetSeconds = isPreviousPeriod - ? getPreviousPeriodOffsetSeconds(currentPeriodDateRange) - : 0; + const offsetSeconds = isPreviousPeriod ? previousPeriodOffsetSeconds : 0; const ts = Math.round(date.getTime() / 1000 + offsetSeconds); for (const valueColumn of valueColumns) { @@ -712,6 +728,7 @@ export function formatResponseForTimeChart({ generateEmptyBuckets = true, source, hiddenSeries = [], + previousPeriodOffsetSeconds = 0, }: { dateRange: [Date, Date]; granularity?: SQLInterval; @@ -720,6 +737,7 @@ export function formatResponseForTimeChart({ generateEmptyBuckets?: boolean; source?: TSource; hiddenSeries?: string[]; + previousPeriodOffsetSeconds?: number; }) { const meta = currentPeriodResponse.meta; @@ -750,7 +768,7 @@ export function formatResponseForTimeChart({ tsBucketMap, source, isPreviousPeriod: false, - currentPeriodDateRange: dateRange, + previousPeriodOffsetSeconds, hiddenSeries, }); @@ -761,7 +779,7 @@ export function formatResponseForTimeChart({ tsBucketMap, source, isPreviousPeriod: true, - currentPeriodDateRange: dateRange, + previousPeriodOffsetSeconds, hiddenSeries, }); } diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index 4e7bb649d..a70e6446b 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -1367,6 +1367,9 @@ function DBSearchPage() { with: aliasWith, // Preserve the original table select string for "View Events" links eventTableSelect: searchedConfig.select, + // In live mode, when the end date is aligned to the granularity, the end date does + // not change on every query, resulting in cached data being re-used. + alignDateRangeToGranularity: !isLive, ...variableConfig, }; }, [ @@ -1375,6 +1378,7 @@ function DBSearchPage() { aliasWith, searchedTimeRange, searchedConfig.select, + isLive, ]); const onFormSubmit = useCallback>( diff --git a/packages/app/src/__tests__/ChartUtils.test.ts b/packages/app/src/__tests__/ChartUtils.test.ts index 928f1d4d0..0c6443484 100644 --- a/packages/app/src/__tests__/ChartUtils.test.ts +++ b/packages/app/src/__tests__/ChartUtils.test.ts @@ -9,6 +9,7 @@ import { convertToTableChartConfig, convertToTimeChartConfig, formatResponseForTimeChart, + getAlignedDateRange, } from '@/ChartUtils'; if (!globalThis.structuredClone) { @@ -503,6 +504,7 @@ describe('ChartUtils', () => { ], granularity: '1 minute', generateEmptyBuckets: false, + previousPeriodOffsetSeconds: 120, }); expect(actual.graphResults).toEqual([ @@ -652,4 +654,111 @@ describe('ChartUtils', () => { expect(convertedConfig.limit).toEqual({ limit: 200 }); }); }); + + describe('getAlignedDateRange', () => { + it('should align start time down to the previous minute boundary', () => { + const dateRange: [Date, Date] = [ + new Date('2025-11-26T12:23:37Z'), // 37 seconds + new Date('2025-11-26T12:25:00Z'), + ]; + + const [alignedStart, alignedEnd] = getAlignedDateRange( + dateRange, + '1 minute', + ); + + expect(alignedStart.toISOString()).toBe('2025-11-26T12:23:00.000Z'); + expect(alignedEnd.toISOString()).toBe('2025-11-26T12:25:00.000Z'); + }); + + it('should align end time up to the next minute boundary', () => { + const dateRange: [Date, Date] = [ + new Date('2025-11-26T12:23:00Z'), + new Date('2025-11-26T12:25:42Z'), // 42 seconds + ]; + + const [alignedStart, alignedEnd] = getAlignedDateRange( + dateRange, + '1 minute', + ); + + expect(alignedStart.toISOString()).toBe('2025-11-26T12:23:00.000Z'); + expect(alignedEnd.toISOString()).toBe('2025-11-26T12:26:00.000Z'); + }); + + it('should align both start and end times with 5 minute granularity', () => { + const dateRange: [Date, Date] = [ + new Date('2025-11-26T12:23:17Z'), // Should round down to 12:20:00 + new Date('2025-11-26T12:27:42Z'), // Should round up to 12:30:00 + ]; + + const [alignedStart, alignedEnd] = getAlignedDateRange( + dateRange, + '5 minute', + ); + + expect(alignedStart.toISOString()).toBe('2025-11-26T12:20:00.000Z'); + expect(alignedEnd.toISOString()).toBe('2025-11-26T12:30:00.000Z'); + }); + + it('should align with 30 second granularity', () => { + const dateRange: [Date, Date] = [ + new Date('2025-11-26T12:23:17Z'), // Should round down to 12:23:00 + new Date('2025-11-26T12:25:42Z'), // Should round up to 12:26:00 + ]; + + const [alignedStart, alignedEnd] = getAlignedDateRange( + dateRange, + '30 second', + ); + + expect(alignedStart.toISOString()).toBe('2025-11-26T12:23:00.000Z'); + expect(alignedEnd.toISOString()).toBe('2025-11-26T12:26:00.000Z'); + }); + + it('should align with 1 day granularity', () => { + const dateRange: [Date, Date] = [ + new Date('2025-11-26T12:23:17Z'), // Should round down to start of day + new Date('2025-11-28T08:15:00Z'), // Should round up to start of next day + ]; + + const [alignedStart, alignedEnd] = getAlignedDateRange( + dateRange, + '1 day', + ); + + expect(alignedStart.toISOString()).toBe('2025-11-26T00:00:00.000Z'); + expect(alignedEnd.toISOString()).toBe('2025-11-29T00:00:00.000Z'); + }); + + it('should not change range when already aligned to the interval', () => { + const dateRange: [Date, Date] = [ + new Date('2025-11-26T12:23:00Z'), // Already aligned + new Date('2025-11-26T12:25:00Z'), // Already aligned + ]; + + const [alignedStart, alignedEnd] = getAlignedDateRange( + dateRange, + '1 minute', + ); + + expect(alignedStart.toISOString()).toBe('2025-11-26T12:23:00.000Z'); + expect(alignedEnd.toISOString()).toBe('2025-11-26T12:25:00.000Z'); + }); + + it('should align with 15 minute granularity', () => { + const dateRange: [Date, Date] = [ + new Date('2025-11-26T12:23:17Z'), // Should round down to 12:15:00 + new Date('2025-11-26T12:47:42Z'), // Should round up to 13:00:00 + ]; + + const [alignedStart, alignedEnd] = getAlignedDateRange( + dateRange, + '15 minute', + ); + + expect(alignedStart.toISOString()).toBe('2025-11-26T12:15:00.000Z'); + expect(alignedEnd.toISOString()).toBe('2025-11-26T13:00:00.000Z'); + }); + }); }); diff --git a/packages/app/src/__tests__/DBSearchPageQueryKey.test.tsx b/packages/app/src/__tests__/DBSearchPageQueryKey.test.tsx index d529b50b1..10595f5ff 100644 --- a/packages/app/src/__tests__/DBSearchPageQueryKey.test.tsx +++ b/packages/app/src/__tests__/DBSearchPageQueryKey.test.tsx @@ -52,7 +52,7 @@ jest.mock('@/ChartUtils', () => ({ new Date('2023-12-31'), new Date('2024-01-01'), ], - getPreviousPeriodOffsetSeconds: () => 86400, + getAlignedDateRange: (dateRange: [Date, Date]) => dateRange, convertToTimeChartConfig: jest.requireActual('@/ChartUtils').convertToTimeChartConfig, })); diff --git a/packages/app/src/components/DBEditTimeChartForm.tsx b/packages/app/src/components/DBEditTimeChartForm.tsx index 06e168ce3..b6e409077 100644 --- a/packages/app/src/components/DBEditTimeChartForm.tsx +++ b/packages/app/src/components/DBEditTimeChartForm.tsx @@ -485,6 +485,10 @@ export default function EditTimeChartForm({ control, name: 'compareToPreviousPeriod', }); + const alignDateRangeToGranularity = useWatch({ + control, + name: 'alignDateRangeToGranularity', + }); const groupBy = useWatch({ control, name: 'groupBy' }); const displayType = useWatch({ control, name: 'displayType' }) ?? DisplayType.Line; @@ -496,9 +500,6 @@ export default function EditTimeChartForm({ const databaseName = tableSource?.from.databaseName; const tableName = tableSource?.from.tableName; - // const tableSource = tableSourceWatch(); - // const databaseName = tableSourceWatch('from.databaseName'); - // const tableName = tableSourceWatch('from.tableName'); const activeTab = useMemo(() => { switch (displayType) { case DisplayType.Search: @@ -523,19 +524,6 @@ export default function EditTimeChartForm({ const showGeneratedSql = ['table', 'time', 'number'].includes(activeTab); // Whether to show the generated SQL preview const showSampleEvents = tableSource?.kind !== SourceKind.Metric; - // const queriedConfig: ChartConfigWithDateRange | undefined = useMemo(() => { - // if (queriedTableSource == null) { - // return undefined; - // } - - // return { - // ...chartConfig, - // from: queriedTableSource.from, - // timestampValueExpression: queriedTableSource?.timestampValueExpression, - // dateRange, - // }; - // }, [dateRange, chartConfig, queriedTableSource]); - // Only update this on submit, otherwise we'll have issues // with using the source value from the last submit // (ex. ignoring local custom source updates) @@ -707,6 +695,20 @@ export default function EditTimeChartForm({ }); }, [dateRange]); + // Trigger a search when "Show Complete Intervals" changes + useEffect(() => { + setQueriedConfig((config: ChartConfigWithDateRange | undefined) => { + if (config == null) { + return config; + } + + return { + ...config, + alignDateRangeToGranularity, + }; + }); + }, [alignDateRangeToGranularity]); + // Trigger a search when "compare to previous period" changes useEffect(() => { setQueriedConfig((config: ChartConfigWithDateRange | undefined) => { @@ -1213,6 +1215,11 @@ export default function EditTimeChartForm({ {activeTab === 'time' && ( + { + const previousPeriodDateRange = + queriedConfig.alignDateRangeToGranularity === false + ? getPreviousDateRange(originalDateRange) + : getAlignedDateRange( + getPreviousDateRange(originalDateRange), + queriedConfig.granularity, + ); + return { ...queriedConfig, - dateRange: getPreviousDateRange(dateRange), + dateRange: previousPeriodDateRange, }; - }, [queriedConfig, dateRange]); + }, [queriedConfig, originalDateRange]); const previousPeriodOffsetSeconds = useMemo(() => { return config.compareToPreviousPeriod - ? getPreviousPeriodOffsetSeconds(dateRange) + ? differenceInSeconds( + dateRange[0], + previousPeriodChartConfig.dateRange[0], + ) : undefined; - }, [dateRange, config.compareToPreviousPeriod]); + }, [ + config.compareToPreviousPeriod, + dateRange, + previousPeriodChartConfig.dateRange, + ]); const { data: previousPeriodData, isLoading: isPreviousPeriodLoading } = useQueriedChartConfig(previousPeriodChartConfig, { @@ -332,6 +349,7 @@ function DBTimeChartComponent({ generateEmptyBuckets: fillNulls !== false, source, hiddenSeries, + previousPeriodOffsetSeconds, }); } catch (e) { console.error(e); @@ -347,6 +365,7 @@ function DBTimeChartComponent({ config.compareToPreviousPeriod, previousPeriodData, hiddenSeries, + previousPeriodOffsetSeconds, ]); // To enable backward compatibility, allow non-controlled usage of displayType diff --git a/packages/app/src/components/MaterializedViews/MVConfigSummary.tsx b/packages/app/src/components/MaterializedViews/MVConfigSummary.tsx index ef8a34fa4..dd8ac3314 100644 --- a/packages/app/src/components/MaterializedViews/MVConfigSummary.tsx +++ b/packages/app/src/components/MaterializedViews/MVConfigSummary.tsx @@ -1,7 +1,8 @@ import { useMemo } from 'react'; import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; import { MaterializedViewConfiguration } from '@hyperdx/common-utils/dist/types'; -import { Grid, Group, Pill, Stack, Table, Text } from '@mantine/core'; +import { Grid, Group, Pill, Stack, Table, Text, Tooltip } from '@mantine/core'; +import { IconInfoCircle } from '@tabler/icons-react'; import { FormatTime } from '@/useFormatTime'; @@ -33,9 +34,23 @@ export default function MVConfigSummary({ - - Granularity - + + + Granularity + + + + + + {config.minGranularity} {config.minDate && ( diff --git a/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts b/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts index b9a683e2f..0983e5993 100644 --- a/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts +++ b/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts @@ -105,7 +105,7 @@ describe('materializedViews', () => { }); }); - it('should return mvConfig and errors if selecting a column which is not in the materialized view', async () => { + it('should return errors if selecting a column which is not in the materialized view', async () => { const chartConfig: ChartConfigWithOptDateRange = { from: { databaseName: 'default', @@ -133,7 +133,7 @@ describe('materializedViews', () => { ]); }); - it('should return mvConfig and errors if selecting an aggregation which is not supported for the specified column', async () => { + it('should return errors if selecting an aggregation which is not supported for the specified column', async () => { const chartConfig: ChartConfigWithOptDateRange = { from: { databaseName: 'default', @@ -627,7 +627,7 @@ describe('materializedViews', () => { expect(result.errors).toBeUndefined(); }); - it('should return mvConfig and errors if the granularity of the query is less than the materialized view granularity', async () => { + it('should return errors if the granularity of the query is less than the materialized view granularity', async () => { const chartConfig: ChartConfigWithOptDateRange = { from: { databaseName: 'default', @@ -641,7 +641,7 @@ describe('materializedViews', () => { ], where: '', connection: 'test-connection', - granularity: '30 seconds', + granularity: '30 second', }; const result = await tryConvertConfigToMaterializedViewSelect( @@ -651,10 +651,41 @@ describe('materializedViews', () => { ); expect(result.optimizedConfig).toBeUndefined(); - expect(result.errors).toEqual(['Granularity must be at least 1 minute.']); + expect(result.errors).toEqual([ + "Granularity must be a multiple of the view's granularity (1 minute).", + ]); }); - it('should return mvConfig and errors if no granularity is specified but the date range is too short for the MV granularity', async () => { + it('should return errors if the granularity of the query is greater than but not a multiple of the materialized view granularity', async () => { + const chartConfig: ChartConfigWithOptDateRange = { + from: { + databaseName: 'default', + tableName: 'otel_spans', + }, + select: [ + { + valueExpression: '', + aggFn: 'count', + }, + ], + where: '', + connection: 'test-connection', + granularity: '90 second', + }; + + const result = await tryConvertConfigToMaterializedViewSelect( + chartConfig, + MV_CONFIG_METRIC_ROLLUP_1M, + metadata, + ); + + expect(result.optimizedConfig).toBeUndefined(); + expect(result.errors).toEqual([ + "Granularity must be a multiple of the view's granularity (1 minute).", + ]); + }); + + it('should return errors if no granularity is specified but the date range is too short for the MV granularity', async () => { const chartConfig: ChartConfigWithOptDateRange = { from: { databaseName: 'default', @@ -894,6 +925,54 @@ describe('materializedViews', () => { ], where: '', connection: 'test-connection', + dateRangeEndInclusive: false, + }); + expect(result.errors).toBeUndefined(); + }); + + it('should set dateRangeEndInclusive to false when optimizing a config with dateRange', async () => { + const chartConfig: ChartConfigWithOptDateRange = { + from: { + databaseName: 'default', + tableName: 'otel_spans', + }, + select: [ + { + valueExpression: '', + aggFn: 'count', + }, + ], + where: '', + connection: 'test-connection', + granularity: '1 minute', + dateRange: [ + new Date('2023-01-01T00:00:00Z'), + new Date('2023-01-02T01:00:00Z'), + ], + }; + + const result = await tryConvertConfigToMaterializedViewSelect( + chartConfig, + MV_CONFIG_METRIC_ROLLUP_1M, + metadata, + ); + + expect(result.optimizedConfig).toEqual({ + from: { + databaseName: 'default', + tableName: 'metrics_rollup_1m', + }, + select: [ + { + valueExpression: 'count', + aggFn: 'sum', + }, + ], + where: '', + connection: 'test-connection', + granularity: '1 minute', + dateRange: chartConfig.dateRange, + dateRangeEndInclusive: false, }); expect(result.errors).toBeUndefined(); }); @@ -1371,7 +1450,7 @@ describe('materializedViews', () => { ).not.toHaveBeenCalled(); }); - it('should return mvConfig and errors if the generated MV config is not valid', async () => { + it('should return errors if the generated MV config is not valid', async () => { const chartConfig: ChartConfigWithOptDateRange = { from: { databaseName: 'default', @@ -1539,7 +1618,9 @@ describe('materializedViews', () => { expect(result.explanations).toEqual([ { mvConfig: MV_CONFIG_METRIC_ROLLUP_1M, - errors: ['Granularity must be at least 1 minute.'], + errors: [ + "Granularity must be a multiple of the view's granularity (1 minute).", + ], success: false, }, { diff --git a/packages/common-utils/src/core/materializedViews.ts b/packages/common-utils/src/core/materializedViews.ts index 187a124e3..dcbb90b3f 100644 --- a/packages/common-utils/src/core/materializedViews.ts +++ b/packages/common-utils/src/core/materializedViews.ts @@ -150,7 +150,13 @@ function mvConfigSupportsGranularity( mvConfig.minGranularity, ); - return chartGranularitySeconds >= mvGranularitySeconds; + // The chart granularity must be a multiple of the MV granularity, + // to avoid unequal distribution of data across chart time buckets + // which don't align with the MV time buckets. + return ( + chartGranularitySeconds >= mvGranularitySeconds && + chartGranularitySeconds % mvGranularitySeconds === 0 + ); } function mvConfigSupportsDateRange( @@ -296,7 +302,7 @@ export async function tryConvertConfigToMaterializedViewSelect< if (!mvConfigSupportsGranularity(mvConfig, chartConfig)) { const error = chartConfig.granularity - ? `Granularity must be at least ${mvConfig.minGranularity}.` + ? `Granularity must be a multiple of the view's granularity (${mvConfig.minGranularity}).` : 'The selected date range is too short for the granularity of this materialized view.'; return { errors: [error] }; } @@ -334,13 +340,15 @@ export async function tryConvertConfigToMaterializedViewSelect< }; } - const clonedConfig = { + const clonedConfig: C = { ...structuredClone(chartConfig), select, from: { databaseName: mvConfig.databaseName, tableName: mvConfig.tableName, }, + // Make the date range end exclusive to avoid selecting the entire next time bucket from the MV + ...('dateRange' in chartConfig ? { dateRangeEndInclusive: false } : {}), }; return { diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 64ae03077..5e429b3f4 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -422,6 +422,7 @@ export const _ChartConfigSchema = z.object({ eventTableSelect: z.string().optional(), compareToPreviousPeriod: z.boolean().optional(), source: z.string().optional(), + alignDateRangeToGranularity: z.boolean().optional(), }); // This is a ChartConfig type without the `with` CTE clause included.