diff --git a/services/libs/tinybird/datasources/activities.datasource b/services/libs/tinybird/datasources/activities.datasource index 68dc8dc893..25f570c3fd 100644 --- a/services/libs/tinybird/datasources/activities.datasource +++ b/services/libs/tinybird/datasources/activities.datasource @@ -48,6 +48,9 @@ SCHEMA > `updatedById` UUID `json:$.updatedById` DEFAULT toUUID('00000000-0000-0000-0000-000000000000'), `segmentId` LowCardinality(String) `json:$.segmentId` DEFAULT '' +INDEXES > + idx_updatedAt updatedAt TYPE minmax GRANULARITY 1 + ENGINE ReplacingMergeTree ENGINE_PARTITION_KEY toYear(createdAt) ENGINE_SORTING_KEY id diff --git a/services/libs/tinybird/datasources/activities_deduplicated_ds.datasource b/services/libs/tinybird/datasources/activities_deduplicated_ds.datasource index 50342803c7..a86732be18 100644 --- a/services/libs/tinybird/datasources/activities_deduplicated_ds.datasource +++ b/services/libs/tinybird/datasources/activities_deduplicated_ds.datasource @@ -35,6 +35,7 @@ SCHEMA > INDEXES > idx_body_ngram3 body TYPE ngrambf_v1(3, 2048, 6, 0) GRANULARITY 64 idx_title_ngram3 title TYPE ngrambf_v1(3, 512, 6, 0) GRANULARITY 64 + idx_updatedAt updatedAt TYPE minmax GRANULARITY 1 ENGINE MergeTree ENGINE_PARTITION_KEY toYear(timestamp) diff --git a/services/libs/tinybird/pipes/activities_deduplicated_copy_pipe_append_mode.pipe b/services/libs/tinybird/pipes/activities_deduplicated_copy_pipe_append_mode.pipe index 277c4ddbc5..34f152704c 100644 --- a/services/libs/tinybird/pipes/activities_deduplicated_copy_pipe_append_mode.pipe +++ b/services/libs/tinybird/pipes/activities_deduplicated_copy_pipe_append_mode.pipe @@ -15,7 +15,12 @@ SQL > a.url, a.updatedAt FROM activities a - WHERE a.updatedAt > (SELECT max("updatedAt") FROM activities_deduplicated_ds) + WHERE + a.updatedAt > ( + SELECT greatest(max(updatedAt), now() - INTERVAL 3 HOUR) + FROM activities_deduplicated_ds + WHERE updatedAt > now() - INTERVAL 3 HOUR + ) TYPE COPY TARGET_DATASOURCE activities_deduplicated_ds