feat: Align date ranges to chart and MV granularity (Line/Bar charts) (#1533)

Closes HDX-3067
Closes #1331 
Closes #1212 
Closes #1468

# Summary

This PR makes a number of improvements around the way we handle date ranges and granularities, in an effort to minimize discrepancies between aggregate values queried from original data and aggregate values queried from materialized views.

1. Date ranges for Line and Bar chart queries are now (by default) auto-aligned to the chart's granularity. **This is not limited to materialized view queries.** Since the chart granularity is a multiple of the MV granularity, this ensures that the date range is aligned to the MV granularity as well. This also address a number of related issues that point out 0-values or low-values in the first or last data points. This PR also includes an option to disable this behavior for charts in Chart Explorer or Dashboard Tiles.
2. All materialized view queries and all time chart queries are now end-exclusive, to avoid selecting the entirety of the next "time bucket" from the materialized view when the date range is aligned with the materialized view granularity
3. Materialized views are only used for a query with a granularity if the chart query granularity is a multiple of the MV granularity. Previously, we'd use the MV as long as the chart query granularity was at least as large as the MV granularity, but this could cause unequal distributions of data across time buckets. Nearly all available granularities are multiples of all smaller available granularities - so this should only impact queries with granularity 15 minutes with MVs with granularity 10 minutes. 10m granularity support is being removed in #1551 

## Demo

<details>
<summary>Show Complete Intervals Option</summary>

https://github.com/user-attachments/assets/4b903adb-4edf-4481-93d6-2a0c42589a37
</details>

Author: pulpdrew

.changeset/itchy-zebras-train.md

@@ -0,0 +1,7 @@
+---
+"@hyperdx/common-utils": patch
+"@hyperdx/api": patch
+"@hyperdx/app": patch
+---
+
+feat: Align line/bar chart date ranges to chart granularity

packages/app/src/ChartUtils.tsx

@@ -1,5 +1,5 @@
 import { useMemo } from 'react';
-import { add } from 'date-fns';
+import { add, differenceInSeconds } from 'date-fns';
 import { omit } from 'lodash';
 import SqlString from 'sqlstring';
 import { z } from 'zod';
@@ -135,20 +135,45 @@ export const DEFAULT_CHART_CONFIG: Omit<
   whereLanguage: 'lucene',
   displayType: DisplayType.Line,
   granularity: 'auto',
+  alignDateRangeToGranularity: true,
 };
 
 export const isGranularity = (value: string): value is Granularity => {
   return Object.values(Granularity).includes(value as Granularity);
 };
 
+export function getAlignedDateRange(
+  [originalStart, originalEnd]: [Date, Date],
+  granularity: SQLInterval,
+): [Date, Date] {
+  // Round the start time down to the previous interval boundary
+  const alignedStart = toStartOfInterval(originalStart, granularity);
+
+  // Round the end time up to the next interval boundary
+  let alignedEnd = toStartOfInterval(originalEnd, granularity);
+  if (alignedEnd.getTime() < originalEnd.getTime()) {
+    const intervalSeconds = convertGranularityToSeconds(granularity);
+    alignedEnd = add(alignedEnd, { seconds: intervalSeconds });
+  }
+
+  return [alignedStart, alignedEnd];
+}
+
 export function convertToTimeChartConfig(config: ChartConfigWithDateRange) {
   const granularity =
     config.granularity === 'auto' || config.granularity == null
       ? convertDateRangeToGranularityString(config.dateRange, 80)
       : config.granularity;
 
+  const dateRange =
+    config.alignDateRangeToGranularity === false
+      ? config.dateRange
+      : getAlignedDateRange(config.dateRange, granularity);
+
   return {
     ...config,
+    dateRange,
+    dateRangeEndInclusive: false,
     granularity,
     limit: { limit: 100000 },
   };
@@ -549,16 +574,9 @@ function inferGroupColumns(meta: Array<{ name: string; type: string }>) {
   ]);
 }
 
-export function getPreviousPeriodOffsetSeconds(
-  dateRange: [Date, Date],
-): number {
-  const [start, end] = dateRange;
-  return Math.round((end.getTime() - start.getTime()) / 1000);
-}
-
 export function getPreviousDateRange(currentRange: [Date, Date]): [Date, Date] {
   const [start, end] = currentRange;
-  const offsetSeconds = getPreviousPeriodOffsetSeconds(currentRange);
+  const offsetSeconds = differenceInSeconds(end, start);
   return [
     new Date(start.getTime() - offsetSeconds * 1000),
     new Date(end.getTime() - offsetSeconds * 1000),
@@ -622,7 +640,7 @@ function addResponseToFormattedData({
   lineDataMap,
   tsBucketMap,
   source,
-  currentPeriodDateRange,
+  previousPeriodOffsetSeconds,
   isPreviousPeriod,
   hiddenSeries = [],
 }: {
@@ -631,7 +649,7 @@ function addResponseToFormattedData({
   response: ResponseJSON<Record<string, any>>;
   source?: TSource;
   isPreviousPeriod: boolean;
-  currentPeriodDateRange: [Date, Date];
+  previousPeriodOffsetSeconds: number;
   hiddenSeries?: string[];
 }) {
   const { meta, data } = response;
@@ -655,9 +673,7 @@ function addResponseToFormattedData({
     const date = new Date(row[timestampColumn.name]);
 
     // Previous period data needs to be shifted forward to align with current period
-    const offsetSeconds = isPreviousPeriod
-      ? getPreviousPeriodOffsetSeconds(currentPeriodDateRange)
-      : 0;
+    const offsetSeconds = isPreviousPeriod ? previousPeriodOffsetSeconds : 0;
     const ts = Math.round(date.getTime() / 1000 + offsetSeconds);
 
     for (const valueColumn of valueColumns) {
@@ -712,6 +728,7 @@ export function formatResponseForTimeChart({
   generateEmptyBuckets = true,
   source,
   hiddenSeries = [],
+  previousPeriodOffsetSeconds = 0,
 }: {
   dateRange: [Date, Date];
   granularity?: SQLInterval;
@@ -720,6 +737,7 @@ export function formatResponseForTimeChart({
   generateEmptyBuckets?: boolean;
   source?: TSource;
   hiddenSeries?: string[];
+  previousPeriodOffsetSeconds?: number;
 }) {
   const meta = currentPeriodResponse.meta;
 
@@ -750,7 +768,7 @@ export function formatResponseForTimeChart({
     tsBucketMap,
     source,
     isPreviousPeriod: false,
-    currentPeriodDateRange: dateRange,
+    previousPeriodOffsetSeconds,
     hiddenSeries,
   });
 
@@ -761,7 +779,7 @@ export function formatResponseForTimeChart({
       tsBucketMap,
       source,
       isPreviousPeriod: true,
-      currentPeriodDateRange: dateRange,
+      previousPeriodOffsetSeconds,
       hiddenSeries,
     });
   }

packages/app/src/DBSearchPage.tsx

@@ -1367,6 +1367,9 @@ function DBSearchPage() {
       with: aliasWith,
       // Preserve the original table select string for "View Events" links
       eventTableSelect: searchedConfig.select,
+      // In live mode, when the end date is aligned to the granularity, the end date does
+      // not change on every query, resulting in cached data being re-used.
+      alignDateRangeToGranularity: !isLive,
       ...variableConfig,
     };
   }, [
@@ -1375,6 +1378,7 @@ function DBSearchPage() {
     aliasWith,
     searchedTimeRange,
     searchedConfig.select,
+    isLive,
   ]);
 
   const onFormSubmit = useCallback<FormEventHandler<HTMLFormElement>>(

packages/app/src/__tests__/ChartUtils.test.ts

@@ -9,6 +9,7 @@ import {
   convertToTableChartConfig,
   convertToTimeChartConfig,
   formatResponseForTimeChart,
+  getAlignedDateRange,
 } from '@/ChartUtils';
 
 if (!globalThis.structuredClone) {
@@ -503,6 +504,7 @@ describe('ChartUtils', () => {
         ],
         granularity: '1 minute',
         generateEmptyBuckets: false,
+        previousPeriodOffsetSeconds: 120,
       });
 
       expect(actual.graphResults).toEqual([
@@ -652,4 +654,111 @@ describe('ChartUtils', () => {
       expect(convertedConfig.limit).toEqual({ limit: 200 });
     });
   });
+
+  describe('getAlignedDateRange', () => {
+    it('should align start time down to the previous minute boundary', () => {
+      const dateRange: [Date, Date] = [
+        new Date('2025-11-26T12:23:37Z'), // 37 seconds
+        new Date('2025-11-26T12:25:00Z'),
+      ];
+
+      const [alignedStart, alignedEnd] = getAlignedDateRange(
+        dateRange,
+        '1 minute',
+      );
+
+      expect(alignedStart.toISOString()).toBe('2025-11-26T12:23:00.000Z');
+      expect(alignedEnd.toISOString()).toBe('2025-11-26T12:25:00.000Z');
+    });
+
+    it('should align end time up to the next minute boundary', () => {
+      const dateRange: [Date, Date] = [
+        new Date('2025-11-26T12:23:00Z'),
+        new Date('2025-11-26T12:25:42Z'), // 42 seconds
+      ];
+
+      const [alignedStart, alignedEnd] = getAlignedDateRange(
+        dateRange,
+        '1 minute',
+      );
+
+      expect(alignedStart.toISOString()).toBe('2025-11-26T12:23:00.000Z');
+      expect(alignedEnd.toISOString()).toBe('2025-11-26T12:26:00.000Z');
+    });
+
+    it('should align both start and end times with 5 minute granularity', () => {
+      const dateRange: [Date, Date] = [
+        new Date('2025-11-26T12:23:17Z'), // Should round down to 12:20:00
+        new Date('2025-11-26T12:27:42Z'), // Should round up to 12:30:00
+      ];
+
+      const [alignedStart, alignedEnd] = getAlignedDateRange(
+        dateRange,
+        '5 minute',
+      );
+
+      expect(alignedStart.toISOString()).toBe('2025-11-26T12:20:00.000Z');
+      expect(alignedEnd.toISOString()).toBe('2025-11-26T12:30:00.000Z');
+    });
+
+    it('should align with 30 second granularity', () => {
+      const dateRange: [Date, Date] = [
+        new Date('2025-11-26T12:23:17Z'), // Should round down to 12:23:00
+        new Date('2025-11-26T12:25:42Z'), // Should round up to 12:26:00
+      ];
+
+      const [alignedStart, alignedEnd] = getAlignedDateRange(
+        dateRange,
+        '30 second',
+      );
+
+      expect(alignedStart.toISOString()).toBe('2025-11-26T12:23:00.000Z');
+      expect(alignedEnd.toISOString()).toBe('2025-11-26T12:26:00.000Z');
+    });
+
+    it('should align with 1 day granularity', () => {
+      const dateRange: [Date, Date] = [
+        new Date('2025-11-26T12:23:17Z'), // Should round down to start of day
+        new Date('2025-11-28T08:15:00Z'), // Should round up to start of next day
+      ];
+
+      const [alignedStart, alignedEnd] = getAlignedDateRange(
+        dateRange,
+        '1 day',
+      );
+
+      expect(alignedStart.toISOString()).toBe('2025-11-26T00:00:00.000Z');
+      expect(alignedEnd.toISOString()).toBe('2025-11-29T00:00:00.000Z');
+    });
+
+    it('should not change range when already aligned to the interval', () => {
+      const dateRange: [Date, Date] = [
+        new Date('2025-11-26T12:23:00Z'), // Already aligned
+        new Date('2025-11-26T12:25:00Z'), // Already aligned
+      ];
+
+      const [alignedStart, alignedEnd] = getAlignedDateRange(
+        dateRange,
+        '1 minute',
+      );
+
+      expect(alignedStart.toISOString()).toBe('2025-11-26T12:23:00.000Z');
+      expect(alignedEnd.toISOString()).toBe('2025-11-26T12:25:00.000Z');
+    });
+
+    it('should align with 15 minute granularity', () => {
+      const dateRange: [Date, Date] = [
+        new Date('2025-11-26T12:23:17Z'), // Should round down to 12:15:00
+        new Date('2025-11-26T12:47:42Z'), // Should round up to 13:00:00
+      ];
+
+      const [alignedStart, alignedEnd] = getAlignedDateRange(
+        dateRange,
+        '15 minute',
+      );
+
+      expect(alignedStart.toISOString()).toBe('2025-11-26T12:15:00.000Z');
+      expect(alignedEnd.toISOString()).toBe('2025-11-26T13:00:00.000Z');
+    });
+  });
 });

packages/app/src/__tests__/DBSearchPageQueryKey.test.tsx

@@ -52,7 +52,7 @@ jest.mock('@/ChartUtils', () => ({
     new Date('2023-12-31'),
     new Date('2024-01-01'),
   ],
-  getPreviousPeriodOffsetSeconds: () => 86400,
+  getAlignedDateRange: (dateRange: [Date, Date]) => dateRange,
   convertToTimeChartConfig:
     jest.requireActual('@/ChartUtils').convertToTimeChartConfig,
 }));

packages/app/src/components/DBEditTimeChartForm.tsx

@@ -485,6 +485,10 @@ export default function EditTimeChartForm({
     control,
     name: 'compareToPreviousPeriod',
   });
+  const alignDateRangeToGranularity = useWatch({
+    control,
+    name: 'alignDateRangeToGranularity',
+  });
   const groupBy = useWatch({ control, name: 'groupBy' });
   const displayType =
     useWatch({ control, name: 'displayType' }) ?? DisplayType.Line;
@@ -496,9 +500,6 @@ export default function EditTimeChartForm({
   const databaseName = tableSource?.from.databaseName;
   const tableName = tableSource?.from.tableName;
 
-  // const tableSource = tableSourceWatch();
-  // const databaseName = tableSourceWatch('from.databaseName');
-  // const tableName = tableSourceWatch('from.tableName');
   const activeTab = useMemo(() => {
     switch (displayType) {
       case DisplayType.Search:
@@ -523,19 +524,6 @@ export default function EditTimeChartForm({
   const showGeneratedSql = ['table', 'time', 'number'].includes(activeTab); // Whether to show the generated SQL preview
   const showSampleEvents = tableSource?.kind !== SourceKind.Metric;
 
-  // const queriedConfig: ChartConfigWithDateRange | undefined = useMemo(() => {
-  //   if (queriedTableSource == null) {
-  //     return undefined;
-  //   }
-
-  //   return {
-  //     ...chartConfig,
-  //     from: queriedTableSource.from,
-  //     timestampValueExpression: queriedTableSource?.timestampValueExpression,
-  //     dateRange,
-  //   };
-  // }, [dateRange, chartConfig, queriedTableSource]);
-
   // Only update this on submit, otherwise we'll have issues
   // with using the source value from the last submit
   // (ex. ignoring local custom source updates)
@@ -707,6 +695,20 @@ export default function EditTimeChartForm({
     });
   }, [dateRange]);
 
+  // Trigger a search when "Show Complete Intervals" changes
+  useEffect(() => {
+    setQueriedConfig((config: ChartConfigWithDateRange | undefined) => {
+      if (config == null) {
+        return config;
+      }
+
+      return {
+        ...config,
+        alignDateRangeToGranularity,
+      };
+    });
+  }, [alignDateRangeToGranularity]);
+
   // Trigger a search when "compare to previous period" changes
   useEffect(() => {
     setQueriedConfig((config: ChartConfigWithDateRange | undefined) => {
@@ -1213,6 +1215,11 @@ export default function EditTimeChartForm({
       </Flex>
       {activeTab === 'time' && (
         <Group justify="end" mb="xs">
+          <SwitchControlled
+            control={control}
+            name="alignDateRangeToGranularity"
+            label="Show Complete Intervals"
+          />
           <SwitchControlled
             control={control}
             name="compareToPreviousPeriod"