feat: use 1 minute window for searches

.changeset/sweet-bears-fold.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/app": patch
+---
+
+feat: use 1 minute window for searches

packages/app/src/DBSearchPage.tsx

@@ -151,7 +151,7 @@ const LIVE_TAIL_REFRESH_FREQUENCY_OPTIONS = [
   { value: '10000', label: '10s' },
   { value: '30000', label: '30s' },
 ];
-const DEFAULT_REFRESH_FREQUENCY = 4000;
+const DEFAULT_REFRESH_FREQUENCY = 10000;
 
 const ALLOWED_SOURCE_KINDS = [SourceKind.Log, SourceKind.Trace];
 const SearchConfigSchema = z.object({
@@ -2101,6 +2101,7 @@ function DBSearchPage() {
                             collapseAllRows={collapseAllRows}
                             onSortingChange={onSortingChange}
                             initialSortBy={initialSortBy}
+                            enableSmallFirstWindow
                           />
                         )}
                     </Box>

packages/app/src/components/DBRowTable.tsx

@@ -1482,6 +1482,7 @@ function DBSqlRowTableComponent({
   onSortingChange,
   initialSortBy,
   variant = 'default',
+  enableSmallFirstWindow,
 }: {
   config: BuilderChartConfigWithDateRange;
   sourceId?: string;
@@ -1505,6 +1506,7 @@ function DBSqlRowTableComponent({
   initialSortBy?: SortingState;
   onSortingChange?: (v: SortingState | null) => void;
   variant?: DBRowTableVariant;
+  enableSmallFirstWindow?: boolean;
 }) {
   const { data: me } = api.useMe();
   const { toggleColumn, displayedColumns: contextDisplayedColumns } =
@@ -1570,6 +1572,7 @@ function DBSqlRowTableComponent({
         enabled && mergedConfig != null && getSelectLength(config.select) > 0,
       isLive,
       queryKeyPrefix,
+      enableSmallFirstWindow,
     });
 
   // The first N columns are the select columns from the user

packages/app/src/components/DBSqlRowTableWithSidebar.tsx

@@ -42,6 +42,7 @@ interface Props {
   onSortingChange?: (v: SortingState | null) => void;
   initialSortBy?: SortingState;
   variant?: DBRowTableVariant;
+  enableSmallFirstWindow?: boolean;
 }
 
 export default function DBSqlRowTableWithSideBar({
@@ -61,6 +62,7 @@ export default function DBSqlRowTableWithSideBar({
   onSortingChange,
   initialSortBy,
   variant,
+  enableSmallFirstWindow,
 }: Props) {
   const { data: sourceData } = useSource({ id: sourceId });
   const [rowId, setRowId] = useQueryState('rowWhere', parseAsStringEncoded);
@@ -139,6 +141,7 @@ export default function DBSqlRowTableWithSideBar({
         onExpandedRowsChange={onExpandedRowsChange}
         collapseAllRows={collapseAllRows}
         variant={variant}
+        enableSmallFirstWindow={enableSmallFirstWindow}
       />
     </RowSidePanelContext.Provider>
   );

packages/app/src/hooks/__tests__/useChartConfig.test.tsx

@@ -18,6 +18,31 @@ import {
 } from '../useChartConfig';
 import { useMVOptimizationExplanation } from '../useMVOptimizationExplanation';
 
+// Mock DEFAULT_TIME_WINDOWS_SECONDS to remove the 15m window
+jest.mock('@/utils/searchWindows', () => {
+  const original = jest.requireActual('@/utils/searchWindows');
+  const mockWindows = [
+    6 * 60 * 60, // 6h
+    6 * 60 * 60, // 6h
+    12 * 60 * 60, // 12h
+    24 * 60 * 60, // 24h
+  ];
+  return {
+    ...original,
+    DEFAULT_TIME_WINDOWS_SECONDS: mockWindows,
+    generateTimeWindowsDescending: (
+      startDate: Date,
+      endDate: Date,
+      windowDurationsSeconds?: number[],
+    ) =>
+      original.generateTimeWindowsDescending(
+        startDate,
+        endDate,
+        windowDurationsSeconds ?? mockWindows,
+      ),
+  };
+});
+
 // Mock the clickhouse module
 jest.mock('@/clickhouse', () => ({
   useClickhouseClient: jest.fn(),

packages/app/src/hooks/__tests__/useOffsetPaginatedQuery.test.tsx

@@ -186,11 +186,11 @@ describe('useOffsetPaginatedQuery', () => {
 
       await waitFor(() => expect(result.current.isLoading).toBe(false));
 
-      // Should have data from the first 6-hour window (working backwards from end date)
+      // Should have data from the first 15-min window (working backwards from end date)
       expect(result.current.data).toBeDefined();
       expect(result.current.data?.window.windowIndex).toBe(0);
       expect(result.current.data?.window.startTime).toEqual(
-        new Date('2024-01-01T18:00:00Z'), // endDate - 6h
+        new Date('2024-01-01T23:45:00Z'), // endDate - 15m
       );
       expect(result.current.data?.window.endTime).toEqual(
         new Date('2024-01-02T00:00:00Z'), // endDate
@@ -226,14 +226,14 @@ describe('useOffsetPaginatedQuery', () => {
 
       await waitFor(() => expect(result.current.isLoading).toBe(false));
 
-      // Should have data from the first 6-hour window (working forwards from start date)
+      // Should have data from the first 15-min window (working forwards from start date)
       expect(result.current.data).toBeDefined();
       expect(result.current.data?.window.windowIndex).toBe(0);
       expect(result.current.data?.window.startTime).toEqual(
         new Date('2024-01-01T00:00:00Z'), // startDate
       );
       expect(result.current.data?.window.endTime).toEqual(
-        new Date('2024-01-01T06:00:00Z'), // endDate + 6h
+        new Date('2024-01-01T00:15:00Z'), // endDate + 15m
       );
       expect(result.current.data?.window.direction).toEqual('ASC');
     });
@@ -473,7 +473,7 @@ describe('useOffsetPaginatedQuery', () => {
       // Verify we're in the first window
       expect(result.current.data?.window.windowIndex).toBe(0);
       expect(result.current.data?.window.startTime).toEqual(
-        new Date('2024-01-01T18:00:00Z'), // endDate - 6h
+        new Date('2024-01-01T23:45:00Z'), // endDate - 15m
       );
       expect(result.current.data?.window.endTime).toEqual(
         new Date('2024-01-02T00:00:00Z'), // endDate
@@ -509,9 +509,9 @@ describe('useOffsetPaginatedQuery', () => {
 
       await waitFor(() => expect(result.current.isLoading).toBe(false));
 
-      // First window: 6h (working backwards from end date)
+      // First window: 15-min (working backwards from end date)
       expect(result.current.data?.window.startTime).toEqual(
-        new Date('2024-01-02T18:00:00Z'), // endDate - 6h
+        new Date('2024-01-02T23:45:00Z'), // endDate - 15m
       );
       expect(result.current.data?.window.endTime).toEqual(
         new Date('2024-01-03T00:00:00Z'), // endDate

packages/app/src/hooks/useOffsetPaginatedQuery.tsx

@@ -39,8 +39,10 @@ import { useMVOptimizationExplanation } from '@/hooks/useMVOptimizationExplanati
 import { useSource } from '@/source';
 import { omit } from '@/utils';
 import {
+  DEFAULT_TIME_WINDOWS_SECONDS,
   generateTimeWindowsAscending,
   generateTimeWindowsDescending,
+  ONE_MIN_WINDOW,
   TimeWindow,
 } from '@/utils/searchWindows';
 
@@ -78,6 +80,7 @@ type TData = {
 type QueryMeta = {
   queryClient: QueryClient;
   hasPreviousQueries: boolean;
+  windowDurationsSeconds: number[];
   metadata: Metadata;
   optimizedConfig?: ChartConfigWithOptTimestamp;
   source: TSource | undefined;
@@ -87,12 +90,17 @@ type QueryMeta = {
 function getTimeWindowFromPageParam(
   config: ChartConfigWithOptTimestamp,
   pageParam: TPageParam,
+  windowDurationsSeconds: number[],
 ): TimeWindow {
   const [startDate, endDate] = config.dateRange;
   const windows =
     isBuilderChartConfig(config) && isFirstOrderByAscending(config.orderBy)
-      ? generateTimeWindowsAscending(startDate, endDate)
-      : generateTimeWindowsDescending(startDate, endDate);
+      ? generateTimeWindowsAscending(startDate, endDate, windowDurationsSeconds)
+      : generateTimeWindowsDescending(
+          startDate,
+          endDate,
+          windowDurationsSeconds,
+        );
   const window = windows[pageParam.windowIndex];
   if (window == null) {
     throw new Error('Invalid time window for page param');
@@ -105,6 +113,7 @@ function getNextPageParam(
   lastPage: TQueryFnData | null,
   allPages: TQueryFnData[],
   config: ChartConfigWithOptTimestamp,
+  windowDurationsSeconds: number[],
 ): TPageParam | undefined {
   // Pagination is not supported for raw SQL tables since they may not be ordered at all.
   if (lastPage == null || isRawSqlChartConfig(config)) {
@@ -113,8 +122,8 @@ function getNextPageParam(
 
   const [startDate, endDate] = config.dateRange;
   const windows = isFirstOrderByAscending(config.orderBy)
-    ? generateTimeWindowsAscending(startDate, endDate)
-    : generateTimeWindowsDescending(startDate, endDate);
+    ? generateTimeWindowsAscending(startDate, endDate, windowDurationsSeconds)
+    : generateTimeWindowsDescending(startDate, endDate, windowDurationsSeconds);
   const currentWindow = lastPage.window;
 
   // Calculate total results from all pages in current window
@@ -158,8 +167,14 @@ const queryFn: QueryFunction<TQueryFnData, TQueryKey, TPageParam> = async ({
     throw new Error('Query missing client meta');
   }
 
-  const { queryClient, metadata, hasPreviousQueries, optimizedConfig, source } =
-    meta as QueryMeta;
+  const {
+    queryClient,
+    windowDurationsSeconds,
+    metadata,
+    hasPreviousQueries,
+    optimizedConfig,
+    source,
+  } = meta as QueryMeta;
 
   // Only stream incrementally if this is a fresh query with no previous
   // response or if it's a paginated query
@@ -177,7 +192,7 @@ const queryFn: QueryFunction<TQueryFnData, TQueryKey, TPageParam> = async ({
   const shouldUseWindowing =
     isBuilderChartConfig(config) && isTimestampExpressionInFirstOrderBy(config);
   const timeWindow = shouldUseWindowing
-    ? getTimeWindowFromPageParam(config, pageParam)
+    ? getTimeWindowFromPageParam(config, pageParam, windowDurationsSeconds)
     : {
         startTime: config.dateRange[0],
         endTime: config.dateRange[1],
@@ -423,10 +438,12 @@ export default function useOffsetPaginatedQuery(
     isLive,
     enabled = true,
     queryKeyPrefix = '',
+    enableSmallFirstWindow,
   }: {
     isLive?: boolean;
     enabled?: boolean;
     queryKeyPrefix?: string;
+    enableSmallFirstWindow?: boolean;
   } = {},
 ) {
   const { data: meData, isLoading: isLoadingMe } = api.useMe();
@@ -440,6 +457,11 @@ export default function useOffsetPaginatedQuery(
   const hasPreviousQueries =
     matchedQueries.filter(([_, data]) => data != null).length > 0;
 
+  const windowDurationsSeconds = DEFAULT_TIME_WINDOWS_SECONDS.slice();
+  if (enableSmallFirstWindow) {
+    windowDurationsSeconds.unshift(ONE_MIN_WINDOW);
+  }
+
   const builderConfig = isBuilderChartConfig(config) ? config : undefined;
   const { data: mvOptimizationData, isLoading: isLoadingMVOptimization } =
     useMVOptimizationExplanation(builderConfig, {
@@ -475,12 +497,18 @@ export default function useOffsetPaginatedQuery(
       enabled && !isLoadingMe && !isLoadingMVOptimization && !isSourceLoading,
     initialPageParam: { windowIndex: 0, offset: 0 } as TPageParam,
     getNextPageParam: (lastPage, allPages) => {
-      return getNextPageParam(lastPage, allPages, config);
+      return getNextPageParam(
+        lastPage,
+        allPages,
+        config,
+        windowDurationsSeconds,
+      );
     },
     staleTime: Infinity, // TODO: Pick a correct time
     meta: {
       queryClient,
       hasPreviousQueries,
+      windowDurationsSeconds,
       metadata,
       optimizedConfig: mvOptimizationData?.optimizedConfig,
       source,

packages/app/src/utils/searchWindows.ts

@@ -1,4 +1,6 @@
-const DEFAULT_TIME_WINDOWS_SECONDS = [
+export const ONE_MIN_WINDOW = 1 * 60;
+export const DEFAULT_TIME_WINDOWS_SECONDS = [
+  15 * 60, // 15m
   6 * 60 * 60, // 6h
   6 * 60 * 60, // 6h
   12 * 60 * 60, // 12h