diff --git a/.changeset/fresh-geckos-develop.md b/.changeset/fresh-geckos-develop.md new file mode 100644 index 000000000..dd071398b --- /dev/null +++ b/.changeset/fresh-geckos-develop.md @@ -0,0 +1,13 @@ +--- +'@powersync/service-module-postgres-storage': minor +'@powersync/service-module-mongodb-storage': minor +'@powersync/service-core-tests': minor +'@powersync/service-module-postgres': minor +'@powersync/service-module-mongodb': minor +'@powersync/service-core': minor +'@powersync/service-module-mssql': minor +'@powersync/service-module-mysql': minor +'@powersync/service-sync-rules': minor +--- + +[Internal] Refactor sync rule representation to split out the parsed definitions from the hydrated state. diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index 30952ff46..2961b2abb 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -1,14 +1,14 @@ import { mongo } from '@powersync/lib-service-mongodb'; -import { SqlEventDescriptor, SqliteRow, SqliteValue, SqlSyncRules } from '@powersync/service-sync-rules'; +import { SqlEventDescriptor, SqliteRow, SqliteValue, HydratedSyncRules } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { BaseObserver, container, + logger as defaultLogger, ErrorCode, errors, Logger, - logger as defaultLogger, ReplicationAssertionError, ServiceError } from '@powersync/lib-services-framework'; @@ -22,13 +22,13 @@ import { utils } from '@powersync/service-core'; import * as timers from 'node:timers/promises'; +import { idPrefixFilter } from '../../utils/util.js'; import { PowerSyncMongo } from './db.js'; import { CurrentBucket, CurrentDataDocument, SourceKey, SyncRuleDocument } from './models.js'; import { MongoIdSequence } from './MongoIdSequence.js'; import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js'; import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js'; import { PersistedBatch } from './PersistedBatch.js'; -import { idPrefixFilter } from '../../utils/util.js'; /** * 15MB @@ -44,7 +44,7 @@ const replicationMutex = new utils.Mutex(); export interface MongoBucketBatchOptions { db: PowerSyncMongo; - syncRules: SqlSyncRules; + syncRules: HydratedSyncRules; groupId: number; slotName: string; lastCheckpointLsn: string | null; @@ -71,7 +71,7 @@ export class MongoBucketBatch private readonly client: mongo.MongoClient; public readonly db: PowerSyncMongo; public readonly session: mongo.ClientSession; - private readonly sync_rules: SqlSyncRules; + private readonly sync_rules: HydratedSyncRules; private readonly group_id: number; @@ -474,8 +474,7 @@ export class MongoBucketBatch if (sourceTable.syncData) { const { results: evaluated, errors: syncErrors } = this.sync_rules.evaluateRowWithErrors({ record: after, - sourceTable, - bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer(`${this.group_id}`) + sourceTable }); for (let error of syncErrors) { diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts index ce38cb683..77796b143 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts @@ -1,6 +1,7 @@ -import { SqlSyncRules } from '@powersync/service-sync-rules'; +import { SqlSyncRules, HydratedSyncRules } from '@powersync/service-sync-rules'; import { storage } from '@powersync/service-core'; +import { versionedHydrationState } from '@powersync/service-sync-rules/src/HydrationState.js'; export class MongoPersistedSyncRules implements storage.PersistedSyncRules { public readonly slot_name: string; @@ -13,4 +14,8 @@ export class MongoPersistedSyncRules implements storage.PersistedSyncRules { ) { this.slot_name = slot_name ?? `powersync_${id}`; } + + hydratedSyncRules(): HydratedSyncRules { + return this.sync_rules.hydrate({ hydrationState: versionedHydrationState(this.id) }); + } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index c36a27322..eb217683f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -10,7 +10,6 @@ import { BroadcastIterable, CHECKPOINT_INVALIDATE_ALL, CheckpointChanges, - CompactOptions, deserializeParameterLookup, GetCheckpointChangesOptions, InternalOpId, @@ -25,10 +24,11 @@ import { WatchWriteCheckpointOptions } from '@powersync/service-core'; import { JSONBig } from '@powersync/service-jsonbig'; -import { ParameterLookup, SqliteJsonRow, SqlSyncRules } from '@powersync/service-sync-rules'; +import { HydratedSyncRules, ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { LRUCache } from 'lru-cache'; import * as timers from 'timers/promises'; +import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../utils/util.js'; import { MongoBucketStorage } from '../MongoBucketStorage.js'; import { PowerSyncMongo } from './db.js'; import { BucketDataDocument, BucketDataKey, BucketStateDocument, SourceKey, SourceTableDocument } from './models.js'; @@ -37,7 +37,6 @@ import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js'; import { MongoCompactor } from './MongoCompactor.js'; import { MongoParameterCompactor } from './MongoParameterCompactor.js'; import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js'; -import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../utils/util.js'; export interface MongoSyncBucketStorageOptions { checksumOptions?: MongoChecksumOptions; @@ -61,7 +60,7 @@ export class MongoSyncBucketStorage private readonly db: PowerSyncMongo; readonly checksums: MongoChecksums; - private parsedSyncRulesCache: { parsed: SqlSyncRules; options: storage.ParseSyncRulesOptions } | undefined; + private parsedSyncRulesCache: { parsed: HydratedSyncRules; options: storage.ParseSyncRulesOptions } | undefined; private writeCheckpointAPI: MongoWriteCheckpointAPI; constructor( @@ -101,14 +100,14 @@ export class MongoSyncBucketStorage }); } - getParsedSyncRules(options: storage.ParseSyncRulesOptions): SqlSyncRules { + getParsedSyncRules(options: storage.ParseSyncRulesOptions): HydratedSyncRules { const { parsed, options: cachedOptions } = this.parsedSyncRulesCache ?? {}; /** * Check if the cached sync rules, if present, had the same options. * Parse sync rules if the options are different or if there is no cached value. */ if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) { - this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).sync_rules, options }; + this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).hydratedSyncRules(), options }; } return this.parsedSyncRulesCache!.parsed; @@ -170,7 +169,7 @@ export class MongoSyncBucketStorage await using batch = new MongoBucketBatch({ logger: options.logger, db: this.db, - syncRules: this.sync_rules.parsed(options).sync_rules, + syncRules: this.sync_rules.parsed(options).hydratedSyncRules(), groupId: this.group_id, slotName: this.slot_name, lastCheckpointLsn: checkpoint_lsn, @@ -293,7 +292,10 @@ export class MongoSyncBucketStorage return result!; } - async getParameterSets(checkpoint: MongoReplicationCheckpoint, lookups: ParameterLookup[]): Promise { + async getParameterSets( + checkpoint: MongoReplicationCheckpoint, + lookups: ScopedParameterLookup[] + ): Promise { return this.db.client.withSession({ snapshot: true }, async (session) => { // Set the session's snapshot time to the checkpoint's snapshot time. // An alternative would be to create the session when the checkpoint is created, but managing @@ -1025,7 +1027,7 @@ class MongoReplicationCheckpoint implements ReplicationCheckpoint { public snapshotTime: mongo.Timestamp ) {} - async getParameterSets(lookups: ParameterLookup[]): Promise { + async getParameterSets(lookups: ScopedParameterLookup[]): Promise { return this.storage.getParameterSets(this, lookups); } } @@ -1034,7 +1036,7 @@ class EmptyReplicationCheckpoint implements ReplicationCheckpoint { readonly checkpoint: InternalOpId = 0n; readonly lsn: string | null = null; - async getParameterSets(lookups: ParameterLookup[]): Promise { + async getParameterSets(lookups: ScopedParameterLookup[]): Promise { return []; } } diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 0fa47f8a2..0347edf3c 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -10,7 +10,6 @@ import { ServiceError } from '@powersync/lib-services-framework'; import { - InternalOpId, MetricsEngine, RelationCache, SaveOperationTag, @@ -18,7 +17,13 @@ import { SourceTable, storage } from '@powersync/service-core'; -import { DatabaseInputRow, SqliteInputRow, SqliteRow, SqlSyncRules, TablePattern } from '@powersync/service-sync-rules'; +import { + DatabaseInputRow, + SqliteInputRow, + SqliteRow, + HydratedSyncRules, + TablePattern +} from '@powersync/service-sync-rules'; import { ReplicationMetric } from '@powersync/service-types'; import { MongoLSN } from '../common/MongoLSN.js'; import { PostImagesOption } from '../types/types.js'; @@ -75,7 +80,7 @@ export class ChangeStreamInvalidatedError extends DatabaseConnectionError { } export class ChangeStream { - sync_rules: SqlSyncRules; + sync_rules: HydratedSyncRules; group_id: number; connection_id = 1; diff --git a/modules/module-mssql/src/replication/CDCStream.ts b/modules/module-mssql/src/replication/CDCStream.ts index ca668aeff..d2b109f62 100644 --- a/modules/module-mssql/src/replication/CDCStream.ts +++ b/modules/module-mssql/src/replication/CDCStream.ts @@ -10,7 +10,13 @@ import { } from '@powersync/lib-services-framework'; import { getUuidReplicaIdentityBson, MetricsEngine, SourceEntityDescriptor, storage } from '@powersync/service-core'; -import { SqliteInputRow, SqliteRow, SqlSyncRules, TablePattern } from '@powersync/service-sync-rules'; +import { + SqliteInputRow, + SqliteRow, + SqlSyncRules, + HydratedSyncRules, + TablePattern +} from '@powersync/service-sync-rules'; import { ReplicationMetric } from '@powersync/service-types'; import { BatchedSnapshotQuery, MSSQLSnapshotQuery, SimpleSnapshotQuery } from './MSSQLSnapshotQuery.js'; @@ -82,7 +88,7 @@ export class CDCDataExpiredError extends DatabaseConnectionError { } export class CDCStream { - private readonly syncRules: SqlSyncRules; + private readonly syncRules: HydratedSyncRules; private readonly storage: storage.SyncRulesBucketStorage; private readonly connections: MSSQLConnectionManager; private readonly abortSignal: AbortSignal; diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index 98d9dc665..13c40062d 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -61,7 +61,7 @@ function createTableId(schema: string, tableName: string): string { } export class BinLogStream { - private readonly syncRules: sync_rules.SqlSyncRules; + private readonly syncRules: sync_rules.HydratedSyncRules; private readonly groupId: number; private readonly storage: storage.SyncRulesBucketStorage; diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index 73600b0b8..c7a3c2c29 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -4,7 +4,6 @@ import { BucketChecksum, CHECKPOINT_INVALIDATE_ALL, CheckpointChanges, - CompactOptions, GetCheckpointChangesOptions, InternalOpId, internalToExternalOpId, @@ -60,7 +59,9 @@ export class PostgresSyncRulesStorage protected writeCheckpointAPI: PostgresWriteCheckpointAPI; // TODO we might be able to share this in an abstract class - private parsedSyncRulesCache: { parsed: sync_rules.SqlSyncRules; options: storage.ParseSyncRulesOptions } | undefined; + private parsedSyncRulesCache: + | { parsed: sync_rules.HydratedSyncRules; options: storage.ParseSyncRulesOptions } + | undefined; private _checksumCache: storage.ChecksumCache | undefined; constructor(protected options: PostgresSyncRulesStorageOptions) { @@ -96,14 +97,14 @@ export class PostgresSyncRulesStorage } // TODO we might be able to share this in an abstract class - getParsedSyncRules(options: storage.ParseSyncRulesOptions): sync_rules.SqlSyncRules { + getParsedSyncRules(options: storage.ParseSyncRulesOptions): sync_rules.HydratedSyncRules { const { parsed, options: cachedOptions } = this.parsedSyncRulesCache ?? {}; /** * Check if the cached sync rules, if present, had the same options. * Parse sync rules if the options are different or if there is no cached value. */ if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) { - this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).sync_rules, options }; + this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).hydratedSyncRules(), options }; } return this.parsedSyncRulesCache!.parsed; @@ -349,7 +350,7 @@ export class PostgresSyncRulesStorage const batch = new PostgresBucketBatch({ logger: options.logger ?? framework.logger, db: this.db, - sync_rules: this.sync_rules.parsed(options).sync_rules, + sync_rules: this.sync_rules.parsed(options).hydratedSyncRules(), group_id: this.group_id, slot_name: this.slot_name, last_checkpoint_lsn: checkpoint_lsn, @@ -374,7 +375,7 @@ export class PostgresSyncRulesStorage async getParameterSets( checkpoint: ReplicationCheckpoint, - lookups: sync_rules.ParameterLookup[] + lookups: sync_rules.ScopedParameterLookup[] ): Promise { const rows = await this.db.sql` SELECT DISTINCT @@ -879,7 +880,7 @@ class PostgresReplicationCheckpoint implements storage.ReplicationCheckpoint { public readonly lsn: string | null ) {} - getParameterSets(lookups: sync_rules.ParameterLookup[]): Promise { + getParameterSets(lookups: sync_rules.ScopedParameterLookup[]): Promise { return this.storage.getParameterSets(this, lookups); } } diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts index 55ff1ccb6..27d4deb5a 100644 --- a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts @@ -24,7 +24,7 @@ import { PostgresPersistedBatch } from './PostgresPersistedBatch.js'; export interface PostgresBucketBatchOptions { logger: Logger; db: lib_postgres.DatabaseClient; - sync_rules: sync_rules.SqlSyncRules; + sync_rules: sync_rules.HydratedSyncRules; group_id: number; slot_name: string; last_checkpoint_lsn: string | null; @@ -72,7 +72,7 @@ export class PostgresBucketBatch protected persisted_op: InternalOpId | null; protected write_checkpoint_batch: storage.CustomWriteCheckpointOptions[]; - protected readonly sync_rules: sync_rules.SqlSyncRules; + protected readonly sync_rules: sync_rules.HydratedSyncRules; protected batch: OperationBatch | null; private lastWaitingLogThrottled = 0; private markRecordUnavailable: BucketStorageMarkRecordUnavailable | undefined; @@ -840,8 +840,7 @@ export class PostgresBucketBatch if (sourceTable.syncData) { const { results: evaluated, errors: syncErrors } = this.sync_rules.evaluateRowWithErrors({ record: after, - sourceTable, - bucketIdTransformer: sync_rules.SqlSyncRules.versionedBucketIdTransformer(`${this.group_id}`) + sourceTable }); for (const error of syncErrors) { diff --git a/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts b/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts index c55cea108..2548b03b7 100644 --- a/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts +++ b/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts @@ -4,6 +4,7 @@ import { storage } from '@powersync/service-core'; import { SqlSyncRules } from '@powersync/service-sync-rules'; import { models } from '../../types/types.js'; +import { versionedHydrationState } from '@powersync/service-sync-rules/src/HydrationState.js'; export class PostgresPersistedSyncRulesContent implements storage.PersistedSyncRulesContent { public readonly slot_name: string; @@ -35,7 +36,12 @@ export class PostgresPersistedSyncRulesContent implements storage.PersistedSyncR return { id: this.id, slot_name: this.slot_name, - sync_rules: SqlSyncRules.fromYaml(this.sync_rules_content, options) + sync_rules: SqlSyncRules.fromYaml(this.sync_rules_content, options), + hydratedSyncRules() { + return this.sync_rules.hydrate({ + hydrationState: versionedHydrationState(this.id) + }); + } }; } diff --git a/modules/module-postgres/src/replication/WalStream.ts b/modules/module-postgres/src/replication/WalStream.ts index b56dad78e..1dd2e23be 100644 --- a/modules/module-postgres/src/replication/WalStream.ts +++ b/modules/module-postgres/src/replication/WalStream.ts @@ -29,6 +29,7 @@ import { SqliteRow, SqliteValue, SqlSyncRules, + HydratedSyncRules, TablePattern, ToastableSqliteRow, toSyncRulesRow, @@ -111,7 +112,7 @@ export class MissingReplicationSlotError extends Error { } export class WalStream { - sync_rules: SqlSyncRules; + sync_rules: HydratedSyncRules; group_id: number; connection_id = 1; diff --git a/packages/service-core-tests/src/test-utils/general-utils.ts b/packages/service-core-tests/src/test-utils/general-utils.ts index 6f93fffb9..fb79c5fae 100644 --- a/packages/service-core-tests/src/test-utils/general-utils.ts +++ b/packages/service-core-tests/src/test-utils/general-utils.ts @@ -1,5 +1,6 @@ import { storage, utils } from '@powersync/service-core'; import { GetQuerierOptions, RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules'; +import { versionedHydrationState } from '@powersync/service-sync-rules/src/HydrationState.js'; import * as bson from 'bson'; export const ZERO_LSN = '0/0'; @@ -25,7 +26,10 @@ export function testRules(content: string): storage.PersistedSyncRulesContent { return { id: 1, sync_rules: SqlSyncRules.fromYaml(content, options), - slot_name: 'test' + slot_name: 'test', + hydratedSyncRules() { + return this.sync_rules.hydrate({ hydrationState: versionedHydrationState(1) }); + } }; }, lock() { @@ -107,7 +111,6 @@ export function querierOptions(globalParameters: RequestParameters): GetQuerierO return { globalParameters, hasDefaultStreams: true, - streams: {}, - bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1') + streams: {} }; } diff --git a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts index d079aaa8c..c65ee68d6 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts @@ -1,9 +1,9 @@ import { storage } from '@powersync/service-core'; -import { ParameterLookup, RequestParameters } from '@powersync/service-sync-rules'; -import { SqlBucketDescriptor } from '@powersync/service-sync-rules/src/SqlBucketDescriptor.js'; +import { RequestParameters, ScopedParameterLookup } from '@powersync/service-sync-rules'; import { expect, test } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; import { TEST_TABLE } from './util.js'; +import { ParameterLookupScope } from '@powersync/service-sync-rules/src/HydrationState.js'; /** * @example @@ -16,6 +16,8 @@ import { TEST_TABLE } from './util.js'; * ``` */ export function registerDataStorageParameterTests(generateStorageFactory: storage.TestStorageFactory) { + const MYBUCKET_1: ParameterLookupScope = { lookupName: 'mybucket', queryId: '1' }; + test('save and load parameters', async () => { await using factory = await generateStorageFactory(); const syncRules = await factory.updateSyncRules({ @@ -58,7 +60,7 @@ bucket_definitions: }); const checkpoint = await bucketStorage.getCheckpoint(); - const parameters = await checkpoint.getParameterSets([ParameterLookup.normalized('mybucket', '1', ['user1'])]); + const parameters = await checkpoint.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); expect(parameters).toEqual([ { group_id: 'group1a' @@ -106,7 +108,7 @@ bucket_definitions: }); const checkpoint2 = await bucketStorage.getCheckpoint(); - const parameters = await checkpoint2.getParameterSets([ParameterLookup.normalized('mybucket', '1', ['user1'])]); + const parameters = await checkpoint2.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); expect(parameters).toEqual([ { group_id: 'group2' @@ -114,7 +116,7 @@ bucket_definitions: ]); // Use the checkpoint to get older data if relevant - const parameters2 = await checkpoint1.getParameterSets([ParameterLookup.normalized('mybucket', '1', ['user1'])]); + const parameters2 = await checkpoint1.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); expect(parameters2).toEqual([ { group_id: 'group1' @@ -183,8 +185,8 @@ bucket_definitions: // association of `list1`::`todo2` const checkpoint = await bucketStorage.getCheckpoint(); const parameters = await checkpoint.getParameterSets([ - ParameterLookup.normalized('mybucket', '1', ['list1']), - ParameterLookup.normalized('mybucket', '1', ['list2']) + ScopedParameterLookup.direct(MYBUCKET_1, ['list1']), + ScopedParameterLookup.direct(MYBUCKET_1, ['list2']) ]); expect(parameters.sort((a, b) => (a.todo_id as string).localeCompare(b.todo_id as string))).toEqual([ @@ -232,16 +234,14 @@ bucket_definitions: const checkpoint = await bucketStorage.getCheckpoint(); const parameters1 = await checkpoint.getParameterSets([ - ParameterLookup.normalized('mybucket', '1', [314n, 314, 3.14]) + ScopedParameterLookup.direct(MYBUCKET_1, [314n, 314, 3.14]) ]); expect(parameters1).toEqual([TEST_PARAMS]); const parameters2 = await checkpoint.getParameterSets([ - ParameterLookup.normalized('mybucket', '1', [314, 314n, 3.14]) + ScopedParameterLookup.direct(MYBUCKET_1, [314, 314n, 3.14]) ]); expect(parameters2).toEqual([TEST_PARAMS]); - const parameters3 = await checkpoint.getParameterSets([ - ParameterLookup.normalized('mybucket', '1', [314n, 314, 3]) - ]); + const parameters3 = await checkpoint.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, [314n, 314, 3])]); expect(parameters3).toEqual([]); }); @@ -295,7 +295,7 @@ bucket_definitions: const checkpoint = await bucketStorage.getCheckpoint(); const parameters1 = await checkpoint.getParameterSets([ - ParameterLookup.normalized('mybucket', '1', [1152921504606846976n]) + ScopedParameterLookup.direct(MYBUCKET_1, [1152921504606846976n]) ]); expect(parameters1).toEqual([TEST_PARAMS]); }); @@ -314,7 +314,7 @@ bucket_definitions: data: [] ` }); - const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).sync_rules; + const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); const bucketStorage = factory.getInstance(syncRules); await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -333,21 +333,19 @@ bucket_definitions: const parameters = new RequestParameters({ sub: 'u1' }, {}); - const q1 = (sync_rules.bucketSources[0] as SqlBucketDescriptor).parameterQueries[0]; + const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; - const lookups = q1.getLookups(parameters); - expect(lookups).toEqual([ParameterLookup.normalized('by_workspace', '1', ['u1'])]); + const lookups = querier.parameterQueryLookups; + expect(lookups).toEqual([ScopedParameterLookup.direct({ lookupName: 'by_workspace', queryId: '1' }, ['u1'])]); const parameter_sets = await checkpoint.getParameterSets(lookups); expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }]); - const buckets = await sync_rules - .getBucketParameterQuerier(test_utils.querierOptions(parameters)) - .querier.queryDynamicBucketDescriptions({ - getParameterSets(lookups) { - return checkpoint.getParameterSets(lookups); - } - }); + const buckets = await querier.queryDynamicBucketDescriptions({ + getParameterSets(lookups) { + return checkpoint.getParameterSets(lookups); + } + }); expect(buckets).toEqual([ { bucket: 'by_workspace["workspace1"]', priority: 3, definition: 'by_workspace', inclusion_reasons: ['default'] } ]); @@ -367,7 +365,7 @@ bucket_definitions: data: [] ` }); - const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).sync_rules; + const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); const bucketStorage = factory.getInstance(syncRules); await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -408,22 +406,20 @@ bucket_definitions: const parameters = new RequestParameters({ sub: 'unknown' }, {}); - const q1 = (sync_rules.bucketSources[0] as SqlBucketDescriptor).parameterQueries[0]; + const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; - const lookups = q1.getLookups(parameters); - expect(lookups).toEqual([ParameterLookup.normalized('by_public_workspace', '1', [])]); + const lookups = querier.parameterQueryLookups; + expect(lookups).toEqual([ScopedParameterLookup.direct({ lookupName: 'by_public_workspace', queryId: '1' }, [])]); const parameter_sets = await checkpoint.getParameterSets(lookups); parameter_sets.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }, { workspace_id: 'workspace3' }]); - const buckets = await sync_rules - .getBucketParameterQuerier(test_utils.querierOptions(parameters)) - .querier.queryDynamicBucketDescriptions({ - getParameterSets(lookups) { - return checkpoint.getParameterSets(lookups); - } - }); + const buckets = await querier.queryDynamicBucketDescriptions({ + getParameterSets(lookups) { + return checkpoint.getParameterSets(lookups); + } + }); buckets.sort((a, b) => a.bucket.localeCompare(b.bucket)); expect(buckets).toEqual([ { @@ -457,7 +453,7 @@ bucket_definitions: data: [] ` }); - const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).sync_rules; + const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); const bucketStorage = factory.getInstance(syncRules); await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -511,31 +507,25 @@ bucket_definitions: const parameters = new RequestParameters({ sub: 'u1' }, {}); // Test intermediate values - could be moved to sync_rules.test.ts - const q1 = (sync_rules.bucketSources[0] as SqlBucketDescriptor).parameterQueries[0]; - const lookups1 = q1.getLookups(parameters); - expect(lookups1).toEqual([ParameterLookup.normalized('by_workspace', '1', [])]); + const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; - const parameter_sets1 = await checkpoint.getParameterSets(lookups1); - parameter_sets1.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); - expect(parameter_sets1).toEqual([{ workspace_id: 'workspace1' }]); - - const q2 = (sync_rules.bucketSources[0] as SqlBucketDescriptor).parameterQueries[1]; - const lookups2 = q2.getLookups(parameters); - expect(lookups2).toEqual([ParameterLookup.normalized('by_workspace', '2', ['u1'])]); + const lookups = querier.parameterQueryLookups; + expect(lookups).toEqual([ + ScopedParameterLookup.direct({ lookupName: 'by_workspace', queryId: '1' }, []), + ScopedParameterLookup.direct({ lookupName: 'by_workspace', queryId: '2' }, ['u1']) + ]); - const parameter_sets2 = await checkpoint.getParameterSets(lookups2); - parameter_sets2.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); - expect(parameter_sets2).toEqual([{ workspace_id: 'workspace3' }]); + const parameter_sets = await checkpoint.getParameterSets(lookups); + parameter_sets.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); + expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }, { workspace_id: 'workspace3' }]); // Test final values - the important part const buckets = ( - await sync_rules - .getBucketParameterQuerier(test_utils.querierOptions(parameters)) - .querier.queryDynamicBucketDescriptions({ - getParameterSets(lookups) { - return checkpoint.getParameterSets(lookups); - } - }) + await querier.queryDynamicBucketDescriptions({ + getParameterSets(lookups) { + return checkpoint.getParameterSets(lookups); + } + }) ).map((e) => e.bucket); buckets.sort(); expect(buckets).toEqual(['by_workspace["workspace1"]', 'by_workspace["workspace3"]']); @@ -572,7 +562,7 @@ bucket_definitions: const checkpoint = await bucketStorage.getCheckpoint(); - const parameters = await checkpoint.getParameterSets([ParameterLookup.normalized('mybucket', '1', ['user1'])]); + const parameters = await checkpoint.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); expect(parameters).toEqual([]); }); diff --git a/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts b/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts index 59499fa02..10263fc7d 100644 --- a/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts +++ b/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts @@ -1,5 +1,5 @@ import { storage } from '@powersync/service-core'; -import { ParameterLookup } from '@powersync/service-sync-rules'; +import { ScopedParameterLookup } from '@powersync/service-sync-rules'; import { expect, test } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; @@ -40,7 +40,7 @@ bucket_definitions: await batch.commit('1/1'); }); - const lookup = ParameterLookup.normalized('test', '1', ['t1']); + const lookup = ScopedParameterLookup.direct({ lookupName: 'test', queryId: '1' }, ['t1']); const checkpoint1 = await bucketStorage.getCheckpoint(); const parameters1 = await checkpoint1.getParameterSets([lookup]); @@ -151,7 +151,7 @@ bucket_definitions: await batch.commit('3/1'); }); - const lookup = ParameterLookup.normalized('test', '1', ['u1']); + const lookup = ScopedParameterLookup.direct({ lookupName: 'test', queryId: '1' }, ['u1']); const checkpoint1 = await bucketStorage.getCheckpoint(); const parameters1 = await checkpoint1.getParameterSets([lookup]); diff --git a/packages/service-core-tests/src/tests/register-sync-tests.ts b/packages/service-core-tests/src/tests/register-sync-tests.ts index e721a2b31..25e323ace 100644 --- a/packages/service-core-tests/src/tests/register-sync-tests.ts +++ b/packages/service-core-tests/src/tests/register-sync-tests.ts @@ -82,10 +82,7 @@ export function registerSyncTests(factory: storage.TestStorageFactory) { const stream = sync.streamResponse({ syncContext, bucketStorage: bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -146,10 +143,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -212,10 +206,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -325,10 +316,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -469,10 +457,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -588,10 +573,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -657,10 +639,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -689,10 +668,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -723,10 +699,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -800,10 +773,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -879,10 +849,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -949,10 +916,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -1020,10 +984,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -1086,10 +1047,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -1217,10 +1175,7 @@ bucket_definitions: const params: sync.SyncStreamParameters = { syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -1293,10 +1248,7 @@ config: const stream = sync.streamResponse({ syncContext, bucketStorage: bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, diff --git a/packages/service-core-tests/tsconfig.json b/packages/service-core-tests/tsconfig.json index d2e14207b..cde5249d4 100644 --- a/packages/service-core-tests/tsconfig.json +++ b/packages/service-core-tests/tsconfig.json @@ -29,6 +29,9 @@ }, { "path": "../../libs/lib-services" + }, + { + "path": "../sync-rules" } ] } diff --git a/packages/service-core/src/routes/endpoints/admin.ts b/packages/service-core/src/routes/endpoints/admin.ts index 8adab1795..6d162a072 100644 --- a/packages/service-core/src/routes/endpoints/admin.ts +++ b/packages/service-core/src/routes/endpoints/admin.ts @@ -177,7 +177,10 @@ export const validate = routeDefinition({ sync_rules: SqlSyncRules.fromYaml(content, { ...apiHandler.getParseSyncRulesOptions(), schema - }) + }), + hydratedSyncRules() { + return this.sync_rules.hydrate(); + } }; }, sync_rules_content: content, diff --git a/packages/service-core/src/routes/endpoints/socket-route.ts b/packages/service-core/src/routes/endpoints/socket-route.ts index 81ac9d15b..e6367e09b 100644 --- a/packages/service-core/src/routes/endpoints/socket-route.ts +++ b/packages/service-core/src/routes/endpoints/socket-route.ts @@ -109,10 +109,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => for await (const data of sync.streamResponse({ syncContext: syncContext, bucketStorage: bucketStorage, - syncRules: { - syncRules, - version: bucketStorage.group_id - }, + syncRules, params: { ...params }, diff --git a/packages/service-core/src/routes/endpoints/sync-rules.ts b/packages/service-core/src/routes/endpoints/sync-rules.ts index ebac1c23e..e3ed68422 100644 --- a/packages/service-core/src/routes/endpoints/sync-rules.ts +++ b/packages/service-core/src/routes/endpoints/sync-rules.ts @@ -202,7 +202,7 @@ async function debugSyncRules(apiHandler: RouteAPI, sync_rules: string) { return { valid: true, - bucket_definitions: rules.bucketSources.map((source) => source.debugRepresentation()), + bucket_definitions: rules.debugRepresentation(), source_tables: resolved_tables, data_tables: rules.debugGetOutputTables() }; diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index 46d02d1e4..fb1df9e96 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -92,10 +92,7 @@ export const syncStreamed = routeDefinition({ const syncLines = sync.streamResponse({ syncContext: syncContext, bucketStorage, - syncRules: { - syncRules, - version: bucketStorage.group_id - }, + syncRules, params: payload.params, token: payload.context.token_payload!, tracker, diff --git a/packages/service-core/src/storage/PersistedSyncRulesContent.ts b/packages/service-core/src/storage/PersistedSyncRulesContent.ts index 6c77ba03a..dd3922764 100644 --- a/packages/service-core/src/storage/PersistedSyncRulesContent.ts +++ b/packages/service-core/src/storage/PersistedSyncRulesContent.ts @@ -1,4 +1,4 @@ -import { SqlSyncRules } from '@powersync/service-sync-rules'; +import { SqlSyncRules, HydratedSyncRules } from '@powersync/service-sync-rules'; import { ReplicationLock } from './ReplicationLock.js'; export interface ParseSyncRulesOptions { @@ -30,4 +30,6 @@ export interface PersistedSyncRules { readonly id: number; readonly sync_rules: SqlSyncRules; readonly slot_name: string; + + hydratedSyncRules(): HydratedSyncRules; } diff --git a/packages/service-core/src/storage/SyncRulesBucketStorage.ts b/packages/service-core/src/storage/SyncRulesBucketStorage.ts index 37015766a..175427449 100644 --- a/packages/service-core/src/storage/SyncRulesBucketStorage.ts +++ b/packages/service-core/src/storage/SyncRulesBucketStorage.ts @@ -1,5 +1,5 @@ import { Logger, ObserverClient } from '@powersync/lib-services-framework'; -import { ParameterLookup, SqlSyncRules, SqliteJsonRow } from '@powersync/service-sync-rules'; +import { HydratedSyncRules, ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules'; import * as util from '../util/util-index.js'; import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js'; import { BucketStorageFactory } from './BucketStorageFactory.js'; @@ -32,7 +32,7 @@ export interface SyncRulesBucketStorage callback: (batch: BucketStorageBatch) => Promise ): Promise; - getParsedSyncRules(options: ParseSyncRulesOptions): SqlSyncRules; + getParsedSyncRules(options: ParseSyncRulesOptions): HydratedSyncRules; /** * Terminate the sync rules. @@ -139,7 +139,7 @@ export interface ResolveTableOptions { connection_tag: string; entity_descriptor: SourceEntityDescriptor; - sync_rules: SqlSyncRules; + sync_rules: HydratedSyncRules; } export interface ResolveTableResult { @@ -284,7 +284,7 @@ export interface ReplicationCheckpoint { * * This gets parameter sets specific to this checkpoint. */ - getParameterSets(lookups: ParameterLookup[]): Promise; + getParameterSets(lookups: ScopedParameterLookup[]): Promise; } export interface WatchWriteCheckpointOptions { diff --git a/packages/service-core/src/storage/bson.ts b/packages/service-core/src/storage/bson.ts index db4b732be..ad7ee3e16 100644 --- a/packages/service-core/src/storage/bson.ts +++ b/packages/service-core/src/storage/bson.ts @@ -1,6 +1,6 @@ import * as bson from 'bson'; -import { ParameterLookup, SqliteJsonValue } from '@powersync/service-sync-rules'; +import { ScopedParameterLookup, SqliteJsonValue } from '@powersync/service-sync-rules'; import { ReplicaId } from './BucketStorageBatch.js'; type NodeBuffer = Buffer; @@ -27,11 +27,11 @@ export const BSON_DESERIALIZE_DATA_OPTIONS: bson.DeserializeOptions = { * Lookup serialization must be number-agnostic. I.e. normalize numbers, instead of preserving numbers. * @param lookup */ -export const serializeLookupBuffer = (lookup: ParameterLookup): NodeBuffer => { +export const serializeLookupBuffer = (lookup: ScopedParameterLookup): NodeBuffer => { return bson.serialize({ l: lookup.values }) as NodeBuffer; }; -export const serializeLookup = (lookup: ParameterLookup) => { +export const serializeLookup = (lookup: ScopedParameterLookup) => { return new bson.Binary(serializeLookupBuffer(lookup)); }; diff --git a/packages/service-core/src/sync/BucketChecksumState.ts b/packages/service-core/src/sync/BucketChecksumState.ts index 4ba687500..d9afa0efb 100644 --- a/packages/service-core/src/sync/BucketChecksumState.ts +++ b/packages/service-core/src/sync/BucketChecksumState.ts @@ -2,37 +2,32 @@ import { BucketDescription, BucketPriority, BucketSource, + HydratedSyncRules, RequestedStream, RequestJwtPayload, RequestParameters, - ResolvedBucket, - SqlSyncRules + ResolvedBucket } from '@powersync/service-sync-rules'; import * as storage from '../storage/storage-index.js'; import * as util from '../util/util-index.js'; import { + logger as defaultLogger, ErrorCode, Logger, ServiceAssertionError, - ServiceError, - logger as defaultLogger + ServiceError } from '@powersync/lib-services-framework'; import { JSONBig } from '@powersync/service-jsonbig'; import { BucketParameterQuerier, QuerierError } from '@powersync/service-sync-rules/src/BucketParameterQuerier.js'; import { SyncContext } from './SyncContext.js'; import { getIntersection, hasIntersection } from './util.js'; -export interface VersionedSyncRules { - syncRules: SqlSyncRules; - version: number; -} - export interface BucketChecksumStateOptions { syncContext: SyncContext; bucketStorage: BucketChecksumStateStorage; - syncRules: VersionedSyncRules; + syncRules: HydratedSyncRules; tokenPayload: RequestJwtPayload; syncRequest: util.StreamingSyncRequest; logger?: Logger; @@ -253,7 +248,7 @@ export class BucketChecksumState { const streamNameToIndex = new Map(); this.streamNameToIndex = streamNameToIndex; - for (const source of this.parameterState.syncRules.syncRules.bucketSources) { + for (const source of this.parameterState.syncRules.definition.bucketSources) { if (this.parameterState.isSubscribedToStream(source)) { streamNameToIndex.set(source.name, subscriptions.length); @@ -381,7 +376,7 @@ export interface CheckpointUpdate { export class BucketParameterState { private readonly context: SyncContext; public readonly bucketStorage: BucketChecksumStateStorage; - public readonly syncRules: VersionedSyncRules; + public readonly syncRules: HydratedSyncRules; public readonly syncParams: RequestParameters; private readonly querier: BucketParameterQuerier; /** @@ -404,7 +399,7 @@ export class BucketParameterState { constructor( context: SyncContext, bucketStorage: BucketChecksumStateStorage, - syncRules: VersionedSyncRules, + syncRules: HydratedSyncRules, tokenPayload: RequestJwtPayload, request: util.StreamingSyncRequest, logger: Logger @@ -436,11 +431,10 @@ export class BucketParameterState { this.includeDefaultStreams = subscriptions?.include_defaults ?? true; this.explicitStreamSubscriptions = explicitStreamSubscriptions; - const { querier, errors } = syncRules.syncRules.getBucketParameterQuerier({ + const { querier, errors } = syncRules.getBucketParameterQuerier({ globalParameters: this.syncParams, hasDefaultStreams: this.includeDefaultStreams, - streams: streamsByName, - bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer(`${syncRules.version}`) + streams: streamsByName }); this.querier = querier; this.streamErrors = Object.groupBy(errors, (e) => e.descriptor) as Record; diff --git a/packages/service-core/src/sync/sync.ts b/packages/service-core/src/sync/sync.ts index 01b848da0..ecd8e7b64 100644 --- a/packages/service-core/src/sync/sync.ts +++ b/packages/service-core/src/sync/sync.ts @@ -1,5 +1,5 @@ import { JSONBig, JsonContainer } from '@powersync/service-jsonbig'; -import { BucketDescription, BucketPriority, RequestJwtPayload } from '@powersync/service-sync-rules'; +import { BucketDescription, BucketPriority, RequestJwtPayload, HydratedSyncRules } from '@powersync/service-sync-rules'; import { AbortError } from 'ix/aborterror.js'; @@ -9,7 +9,7 @@ import * as util from '../util/util-index.js'; import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework'; import { mergeAsyncIterables } from '../streams/streams-index.js'; -import { BucketChecksumState, CheckpointLine, VersionedSyncRules } from './BucketChecksumState.js'; +import { BucketChecksumState, CheckpointLine } from './BucketChecksumState.js'; import { OperationsSentStats, RequestTracker, statsForBatch } from './RequestTracker.js'; import { SyncContext } from './SyncContext.js'; import { TokenStreamOptions, acquireSemaphoreAbortable, settledPromise, tokenStream } from './util.js'; @@ -17,7 +17,7 @@ import { TokenStreamOptions, acquireSemaphoreAbortable, settledPromise, tokenStr export interface SyncStreamParameters { syncContext: SyncContext; bucketStorage: storage.SyncRulesBucketStorage; - syncRules: VersionedSyncRules; + syncRules: HydratedSyncRules; params: util.StreamingSyncRequest; token: auth.JwtPayload; logger?: Logger; @@ -94,7 +94,7 @@ export async function* streamResponse( async function* streamResponseInner( syncContext: SyncContext, bucketStorage: storage.SyncRulesBucketStorage, - syncRules: VersionedSyncRules, + syncRules: HydratedSyncRules, params: util.StreamingSyncRequest, tokenPayload: RequestJwtPayload, tracker: RequestTracker, diff --git a/packages/service-core/test/src/routes/stream.test.ts b/packages/service-core/test/src/routes/stream.test.ts index baa74fc82..5e273d61c 100644 --- a/packages/service-core/test/src/routes/stream.test.ts +++ b/packages/service-core/test/src/routes/stream.test.ts @@ -45,7 +45,7 @@ describe('Stream Route', () => { const storage = { getParsedSyncRules() { - return new SqlSyncRules('bucket_definitions: {}'); + return new SqlSyncRules('bucket_definitions: {}').hydrate(); }, watchCheckpointChanges: async function* (options) { throw new Error('Simulated storage error'); @@ -83,7 +83,7 @@ describe('Stream Route', () => { it('logs the application metadata', async () => { const storage = { getParsedSyncRules() { - return new SqlSyncRules('bucket_definitions: {}'); + return new SqlSyncRules('bucket_definitions: {}').hydrate(); }, watchCheckpointChanges: async function* (options) { throw new Error('Simulated storage error'); diff --git a/packages/service-core/test/src/sync/BucketChecksumState.test.ts b/packages/service-core/test/src/sync/BucketChecksumState.test.ts index a27bf9a37..26c986521 100644 --- a/packages/service-core/test/src/sync/BucketChecksumState.test.ts +++ b/packages/service-core/test/src/sync/BucketChecksumState.test.ts @@ -12,14 +12,9 @@ import { WatchFilterEvent } from '@/index.js'; import { JSONBig } from '@powersync/service-jsonbig'; -import { - SqliteJsonRow, - ParameterLookup, - SqlSyncRules, - RequestJwtPayload, - BucketSource -} from '@powersync/service-sync-rules'; -import { describe, expect, test, beforeEach } from 'vitest'; +import { RequestJwtPayload, ScopedParameterLookup, SqliteJsonRow, SqlSyncRules } from '@powersync/service-sync-rules'; +import { versionedHydrationState } from '@powersync/service-sync-rules/src/HydrationState.js'; +import { beforeEach, describe, expect, test } from 'vitest'; describe('BucketChecksumState', () => { // Single global[] bucket. @@ -31,7 +26,7 @@ bucket_definitions: data: [] `, { defaultSchema: 'public' } - ); + ).hydrate({ hydrationState: versionedHydrationState(1) }); // global[1] and global[2] const SYNC_RULES_GLOBAL_TWO = SqlSyncRules.fromYaml( @@ -44,7 +39,7 @@ bucket_definitions: data: [] `, { defaultSchema: 'public' } - ); + ).hydrate({ hydrationState: versionedHydrationState(2) }); // by_project[n] const SYNC_RULES_DYNAMIC = SqlSyncRules.fromYaml( @@ -55,7 +50,7 @@ bucket_definitions: data: [] `, { defaultSchema: 'public' } - ); + ).hydrate({ hydrationState: versionedHydrationState(3) }); const syncContext = new SyncContext({ maxBuckets: 100, @@ -75,10 +70,7 @@ bucket_definitions: syncContext, syncRequest, tokenPayload, - syncRules: { - syncRules: SYNC_RULES_GLOBAL, - version: 1 - }, + syncRules: SYNC_RULES_GLOBAL, bucketStorage: storage }); @@ -148,10 +140,7 @@ bucket_definitions: tokenPayload, // Client sets the initial state here syncRequest: { buckets: [{ name: 'global[]', after: '1' }] }, - syncRules: { - syncRules: SYNC_RULES_GLOBAL, - version: 1 - }, + syncRules: SYNC_RULES_GLOBAL, bucketStorage: storage }); @@ -189,10 +178,7 @@ bucket_definitions: syncContext, tokenPayload, syncRequest, - syncRules: { - syncRules: SYNC_RULES_GLOBAL_TWO, - version: 2 - }, + syncRules: SYNC_RULES_GLOBAL_TWO, bucketStorage: storage }); @@ -260,10 +246,7 @@ bucket_definitions: tokenPayload, // Client sets the initial state here syncRequest: { buckets: [{ name: 'something_here[]', after: '1' }] }, - syncRules: { - syncRules: SYNC_RULES_GLOBAL, - version: 1 - }, + syncRules: SYNC_RULES_GLOBAL, bucketStorage: storage }); @@ -304,10 +287,7 @@ bucket_definitions: syncContext, tokenPayload, syncRequest, - syncRules: { - syncRules: SYNC_RULES_GLOBAL_TWO, - version: 1 - }, + syncRules: SYNC_RULES_GLOBAL_TWO, bucketStorage: storage }); @@ -360,10 +340,7 @@ bucket_definitions: syncContext, tokenPayload, syncRequest, - syncRules: { - syncRules: SYNC_RULES_GLOBAL_TWO, - version: 2 - }, + syncRules: SYNC_RULES_GLOBAL_TWO, bucketStorage: storage }); @@ -418,10 +395,7 @@ bucket_definitions: syncContext, tokenPayload, syncRequest, - syncRules: { - syncRules: SYNC_RULES_GLOBAL_TWO, - version: 2 - }, + syncRules: SYNC_RULES_GLOBAL_TWO, bucketStorage: storage }); @@ -525,16 +499,13 @@ bucket_definitions: syncContext, tokenPayload: { sub: 'u1' }, syncRequest, - syncRules: { - syncRules: SYNC_RULES_DYNAMIC, - version: 1 - }, + syncRules: SYNC_RULES_DYNAMIC, bucketStorage: storage }); const line = (await state.buildNextCheckpointLine({ base: storage.makeCheckpoint(1n, (lookups) => { - expect(lookups).toEqual([ParameterLookup.normalized('by_project', '1', ['u1'])]); + expect(lookups).toEqual([ScopedParameterLookup.direct({ lookupName: 'by_project', queryId: '1' }, ['u1'])]); return [{ id: 1 }, { id: 2 }]; }), writeCheckpoint: null, @@ -595,7 +566,7 @@ bucket_definitions: // Now we get a new line const line2 = (await state.buildNextCheckpointLine({ base: storage.makeCheckpoint(2n, (lookups) => { - expect(lookups).toEqual([ParameterLookup.normalized('by_project', '1', ['u1'])]); + expect(lookups).toEqual([ScopedParameterLookup.direct({ lookupName: 'by_project', queryId: '1' }, ['u1'])]); return [{ id: 1 }, { id: 2 }, { id: 3 }]; }), writeCheckpoint: null, @@ -627,7 +598,6 @@ bucket_definitions: }); describe('streams', () => { - let source: { -readonly [P in keyof BucketSource]: BucketSource[P] }; let storage: MockBucketChecksumStateStorage; function checksumState(source: string | boolean, options?: Partial) { @@ -645,13 +615,13 @@ config: const rules = SqlSyncRules.fromYaml(source, { defaultSchema: 'public' - }); + }).hydrate({ hydrationState: versionedHydrationState(1) }); return new BucketChecksumState({ syncContext, syncRequest, tokenPayload, - syncRules: { syncRules: rules, version: 1 }, + syncRules: rules, bucketStorage: storage, ...options }); @@ -884,12 +854,12 @@ class MockBucketChecksumStateStorage implements BucketChecksumStateStorage { makeCheckpoint( opId: InternalOpId, - parameters?: (lookups: ParameterLookup[]) => SqliteJsonRow[] + parameters?: (lookups: ScopedParameterLookup[]) => SqliteJsonRow[] ): ReplicationCheckpoint { return { checkpoint: opId, lsn: String(opId), - getParameterSets: async (lookups: ParameterLookup[]) => { + getParameterSets: async (lookups: ScopedParameterLookup[]) => { if (parameters == null) { throw new Error(`getParametersSets not defined for checkpoint ${opId}`); } diff --git a/packages/sync-rules/src/BaseSqlDataQuery.ts b/packages/sync-rules/src/BaseSqlDataQuery.ts index b426cea03..24ab1138b 100644 --- a/packages/sync-rules/src/BaseSqlDataQuery.ts +++ b/packages/sync-rules/src/BaseSqlDataQuery.ts @@ -3,19 +3,19 @@ import { SqlRuleError } from './errors.js'; import { ColumnDefinition } from './ExpressionType.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; +import { castAsText } from './sql_functions.js'; import { TablePattern } from './TablePattern.js'; import { - BucketIdTransformer, - EvaluationResult, QueryParameters, QuerySchema, + UnscopedEvaluatedRow, + UnscopedEvaluationResult, SourceSchema, SourceSchemaTable, SqliteJsonRow, SqliteRow } from './types.js'; import { filterJsonRow } from './utils.js'; -import { castAsText } from './sql_functions.js'; export interface RowValueExtractor { extract(tables: QueryParameters, into: SqliteRow): void; @@ -25,8 +25,7 @@ export interface RowValueExtractor { export interface EvaluateRowOptions { table: SourceTableInterface; row: SqliteRow; - bucketIds: (params: QueryParameters) => string[]; - bucketIdTransformer: BucketIdTransformer | null; + serializedBucketParameters: (params: QueryParameters) => string[]; } export interface BaseSqlDataQueryOptions { @@ -35,7 +34,6 @@ export interface BaseSqlDataQueryOptions { sql: string; columns: SelectedColumn[]; extractors: RowValueExtractor[]; - descriptorName: string; bucketParameters: string[]; tools: SqlTools; errors?: SqlRuleError[]; @@ -71,10 +69,6 @@ export class BaseSqlDataQuery { */ readonly extractors: RowValueExtractor[]; - /** - * Bucket definition name. - */ - readonly descriptorName: string; /** * Bucket parameter names, without the `bucket.` prefix. * @@ -95,7 +89,6 @@ export class BaseSqlDataQuery { this.sql = options.sql; this.columns = options.columns; this.extractors = options.extractors; - this.descriptorName = options.descriptorName; this.bucketParameters = options.bucketParameters; this.tools = options.tools; this.errors = options.errors ?? []; @@ -177,12 +170,17 @@ export class BaseSqlDataQuery { } } - evaluateRowWithOptions(options: Omit): EvaluationResult[] { + evaluateRowWithOptions(options: EvaluateRowOptions): UnscopedEvaluationResult[] { try { - const { table, row, bucketIds } = options; + const { table, row, serializedBucketParameters } = options; const tables = { [this.table.nameInSchema]: this.addSpecialParameters(table, row) }; - const resolvedBucketIds = bucketIds(tables); + // Array of _serialized_ parameters, one per output result. + const resolvedBucketParameters = serializedBucketParameters(tables); + if (resolvedBucketParameters.length == 0) { + // Short-circuit: No need to transform the row if there are no matching buckets. + return []; + } const data = this.transformRow(tables); let id = data.id; @@ -197,13 +195,13 @@ export class BaseSqlDataQuery { } const outputTable = this.getOutputName(table.name); - return resolvedBucketIds.map((bucketId) => { + return resolvedBucketParameters.map((serializedBucketParameters) => { return { - bucket: bucketId, + serializedBucketParameters, table: outputTable, id: id, data - } as EvaluationResult; + } satisfies UnscopedEvaluatedRow; }); } catch (e) { return [{ error: e.message ?? `Evaluating data query failed` }]; diff --git a/packages/sync-rules/src/BucketParameterQuerier.ts b/packages/sync-rules/src/BucketParameterQuerier.ts index 8842c6d7c..5ef89a592 100644 --- a/packages/sync-rules/src/BucketParameterQuerier.ts +++ b/packages/sync-rules/src/BucketParameterQuerier.ts @@ -1,4 +1,5 @@ -import { BucketDescription, ResolvedBucket } from './BucketDescription.js'; +import { ResolvedBucket } from './BucketDescription.js'; +import { ParameterLookupScope } from './HydrationState.js'; import { RequestedStream } from './SqlSyncRules.js'; import { RequestParameters, SqliteJsonRow, SqliteJsonValue } from './types.js'; import { normalizeParameterValue } from './utils.js'; @@ -25,7 +26,7 @@ export interface BucketParameterQuerier { */ readonly hasDynamicBuckets: boolean; - readonly parameterQueryLookups: ParameterLookup[]; + readonly parameterQueryLookups: ScopedParameterLookup[]; /** * These buckets depend on parameter storage, and needs to be retrieved dynamically for each checkpoint. @@ -58,7 +59,7 @@ export interface PendingQueriers { } export interface ParameterLookupSource { - getParameterSets: (lookups: ParameterLookup[]) => Promise; + getParameterSets: (lookups: ScopedParameterLookup[]) => Promise; } export interface QueryBucketDescriptorOptions extends ParameterLookupSource { @@ -88,19 +89,38 @@ export function mergeBucketParameterQueriers(queriers: BucketParameterQuerier[]) * * Other query types are not supported yet. */ -export class ParameterLookup { +export class ScopedParameterLookup { // bucket definition name, parameter query index, ...lookup values readonly values: SqliteJsonValue[]; - static normalized(bucketDefinitionName: string, queryIndex: string, values: SqliteJsonValue[]): ParameterLookup { - return new ParameterLookup([bucketDefinitionName, queryIndex, ...values.map(normalizeParameterValue)]); + static normalized(scope: ParameterLookupScope, lookup: UnscopedParameterLookup): ScopedParameterLookup { + return new ScopedParameterLookup([scope.lookupName, scope.queryId, ...lookup.lookupValues]); + } + + /** + * Primarily for test fixtures. + */ + static direct(scope: ParameterLookupScope, values: SqliteJsonValue[]): ScopedParameterLookup { + return new ScopedParameterLookup([scope.lookupName, scope.queryId, ...values.map(normalizeParameterValue)]); } /** * * @param values must be pre-normalized (any integer converted into bigint) */ - constructor(values: SqliteJsonValue[]) { + private constructor(values: SqliteJsonValue[]) { this.values = values; } } + +export class UnscopedParameterLookup { + readonly lookupValues: SqliteJsonValue[]; + + static normalized(values: SqliteJsonValue[]): UnscopedParameterLookup { + return new UnscopedParameterLookup(values.map(normalizeParameterValue)); + } + + constructor(lookupValues: SqliteJsonValue[]) { + this.lookupValues = lookupValues; + } +} diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts index 6131d78d7..482fb5f5b 100644 --- a/packages/sync-rules/src/BucketSource.ts +++ b/packages/sync-rules/src/BucketSource.ts @@ -1,41 +1,71 @@ -import { BucketParameterQuerier, ParameterLookup, PendingQueriers } from './BucketParameterQuerier.js'; +import { + BucketParameterQuerier, + UnscopedParameterLookup, + PendingQueriers, + ScopedParameterLookup +} from './BucketParameterQuerier.js'; import { ColumnDefinition } from './ExpressionType.js'; +import { DEFAULT_HYDRATION_STATE, HydrationState, ParameterLookupScope } from './HydrationState.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { GetQuerierOptions } from './SqlSyncRules.js'; import { TablePattern } from './TablePattern.js'; -import { EvaluatedParametersResult, EvaluateRowOptions, EvaluationResult, SourceSchema, SqliteRow } from './types.js'; +import { + EvaluatedParametersResult, + EvaluatedRow, + EvaluateRowOptions, + EvaluationResult, + isEvaluationError, + UnscopedEvaluationResult, + SourceSchema, + SqliteRow, + UnscopedEvaluatedParametersResult, + EvaluatedParameters +} from './types.js'; +import { buildBucketName } from './utils.js'; + +export interface CreateSourceParams { + hydrationState: HydrationState; +} /** - * An interface declaring - * - * - which buckets the sync service should create when processing change streams from the database. - * - how data in source tables maps to data in buckets (e.g. when we're not selecting all columns). - * - which buckets a given connection has access to. - * - * There are two ways to define bucket sources: Via sync rules made up of parameter and data queries, and via stream - * definitions that only consist of a single query. + * A BucketSource is a _logical_ bucket or sync stream definition. It is primarily used to group together + * related BucketDataSource, BucketParameterLookupSource and BucketParameterQuerierSource definitions, + * for the purpose of subscribing to specific streams. It does not directly define the implementation + * or replication process. */ export interface BucketSource { readonly name: string; readonly type: BucketSourceType; - readonly subscribedToByDefault: boolean; /** - * Given a row in a source table that affects sync parameters, returns a structure to index which buckets rows should - * be associated with. + * BucketDataSource describing the data in this bucket/stream definition. * - * The returned {@link ParameterLookup} can be referenced by {@link pushBucketParameterQueriers} to allow the storage - * system to find buckets. + * The same data source could in theory be present in multiple stream definitions. + * + * Sources must _only_ be split into multiple ones if they will result in different buckets being created. + * Specifically, bucket definitions would always have a single data source, while stream definitions may have + * one per variant. */ - evaluateParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[]; + readonly dataSources: BucketDataSource[]; /** - * Given a row as it appears in a table that affects sync data, return buckets, logical table names and transformed - * data for rows to add to buckets. + * BucketParameterLookupSource describing the parameter tables used in this bucket/stream definition. + * + * The same source could in theory be present in multiple stream definitions. */ - evaluateRow(options: EvaluateRowOptions): EvaluationResult[]; + readonly parameterIndexLookupCreators: ParameterIndexLookupCreator[]; + + debugRepresentation(): any; + hydrate(params: CreateSourceParams): HydratedBucketSource; +} + +/** + * Internal interface for individual queriers. This is not used on its in the public API directly, apart + * from in HydratedBucketSource. Everywhere else it is just to standardize the internal functions that we re-use. + */ +export interface BucketParameterQuerierSource { /** * Reports {@link BucketParameterQuerier}s resolving buckets that a specific stream request should have access to. * @@ -43,23 +73,46 @@ export interface BucketSource { * @param options Options, including parameters that may affect the buckets loaded by this source. */ pushBucketParameterQueriers(result: PendingQueriers, options: GetQuerierOptions): void; +} + +export interface HydratedBucketSource extends BucketParameterQuerierSource { + readonly definition: BucketSource; +} +export type ScopedEvaluateRow = (options: EvaluateRowOptions) => EvaluationResult[]; +export type ScopedEvaluateParameterRow = ( + sourceTable: SourceTableInterface, + row: SqliteRow +) => EvaluatedParametersResult[]; + +/** + * Encodes a static definition of a bucket source, as parsed from sync rules or stream definitions. + * + * This does not require any "hydration" itself: All results are independent of bucket names. + * The higher-level HydratedSyncRules will use a HydrationState to generate bucket names. + */ +export interface BucketDataSource { /** - * Whether {@link pushBucketParameterQueriers} may include a querier where - * {@link BucketParameterQuerier.hasDynamicBuckets} is true. + * Unique name of the data source within a sync rules version. * - * This is mostly used for testing. + * This may be used as the basis for bucketPrefix (or it could be ignored). */ - hasDynamicBucketQueries(): boolean; - - getSourceTables(): Set; + readonly uniqueName: string; - /** Whether the table possibly affects the buckets resolved by this source. */ - tableSyncsParameters(table: SourceTableInterface): boolean; + /** + * For debug use only. + */ + readonly bucketParameters: string[]; - /** Whether the table possibly affects the contents of buckets resolved by this source. */ + getSourceTables(): Set; tableSyncsData(table: SourceTableInterface): boolean; + /** + * Given a row as it appears in a table that affects sync data, return buckets, logical table names and transformed + * data for rows to add to buckets. + */ + evaluateRow(options: EvaluateRowOptions): UnscopedEvaluationResult[]; + /** * Given a static schema, infer all logical tables and associated columns that appear in buckets defined by this * source. @@ -69,8 +122,39 @@ export interface BucketSource { resolveResultSets(schema: SourceSchema, tables: Record>): void; debugWriteOutputTables(result: Record): void; +} - debugRepresentation(): any; +/** + * This defines how to extract parameter index lookup values from parameter queries or stream subqueries. + * + * This is only relevant for parameter queries and subqueries that query tables. + */ +export interface ParameterIndexLookupCreator { + /** + * lookupName + queryId is used to uniquely identify parameter queries for parameter storage. + * + * This defines the default values if no transformations are applied. + */ + readonly defaultLookupScope: ParameterLookupScope; + + getSourceTables(): Set; + + /** + * Given a row in a source table that affects sync parameters, returns a structure to index which buckets rows should + * be associated with. + * + * The returned {@link UnscopedParameterLookup} can be referenced by {@link pushBucketParameterQueriers} to allow the storage + * system to find buckets. + */ + evaluateParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): UnscopedEvaluatedParametersResult[]; + + /** Whether the table possibly affects the buckets resolved by this source. */ + tableSyncsParameters(table: SourceTableInterface): boolean; +} + +export interface DebugMergedSource extends HydratedBucketSource { + evaluateRow: ScopedEvaluateRow; + evaluateParameterRow: ScopedEvaluateParameterRow; } export enum BucketSourceType { @@ -79,3 +163,83 @@ export enum BucketSourceType { } export type ResultSetDescription = { name: string; columns: ColumnDefinition[] }; + +export function hydrateEvaluateRow(hydrationState: HydrationState, source: BucketDataSource): ScopedEvaluateRow { + const scope = hydrationState.getBucketSourceScope(source); + return (options: EvaluateRowOptions): EvaluationResult[] => { + return source.evaluateRow(options).map((result) => { + if (isEvaluationError(result)) { + return result; + } + return { + bucket: buildBucketName(scope, result.serializedBucketParameters), + id: result.id, + table: result.table, + data: result.data + } satisfies EvaluatedRow; + }); + }; +} + +export function hydrateEvaluateParameterRow( + hydrationState: HydrationState, + source: ParameterIndexLookupCreator +): ScopedEvaluateParameterRow { + const scope = hydrationState.getParameterIndexLookupScope(source); + return (sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] => { + return source.evaluateParameterRow(sourceTable, row).map((result) => { + if (isEvaluationError(result)) { + return result; + } + return { + bucketParameters: result.bucketParameters, + lookup: ScopedParameterLookup.normalized(scope, result.lookup) + } satisfies EvaluatedParameters; + }); + }; +} + +export function mergeDataSources( + hydrationState: HydrationState, + sources: BucketDataSource[] +): { evaluateRow: ScopedEvaluateRow } { + const withScope = sources.map((source) => hydrateEvaluateRow(hydrationState, source)); + return { + evaluateRow(options: EvaluateRowOptions): EvaluationResult[] { + return withScope.flatMap((source) => source(options)); + } + }; +} + +export function mergeParameterIndexLookupCreators( + hydrationState: HydrationState, + sources: ParameterIndexLookupCreator[] +): { evaluateParameterRow: ScopedEvaluateParameterRow } { + const withScope = sources.map((source) => hydrateEvaluateParameterRow(hydrationState, source)); + return { + evaluateParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] { + return withScope.flatMap((source) => source(sourceTable, row)); + } + }; +} + +/** + * For production purposes, we typically need to operate on the different sources separately. However, for debugging, + * it is useful to have a single merged source that can evaluate everything. + */ +export function debugHydratedMergedSource(bucketSource: BucketSource, params?: CreateSourceParams): DebugMergedSource { + const hydrationState = params?.hydrationState ?? DEFAULT_HYDRATION_STATE; + const resolvedParams = { hydrationState }; + const dataSource = mergeDataSources(hydrationState, bucketSource.dataSources); + const parameterLookupSource = mergeParameterIndexLookupCreators( + hydrationState, + bucketSource.parameterIndexLookupCreators + ); + const hydratedBucketSource = bucketSource.hydrate(resolvedParams); + return { + definition: bucketSource, + evaluateParameterRow: parameterLookupSource.evaluateParameterRow.bind(parameterLookupSource), + evaluateRow: dataSource.evaluateRow.bind(dataSource), + pushBucketParameterQueriers: hydratedBucketSource.pushBucketParameterQueriers.bind(hydratedBucketSource) + }; +} diff --git a/packages/sync-rules/src/HydratedSyncRules.ts b/packages/sync-rules/src/HydratedSyncRules.ts new file mode 100644 index 000000000..36ccb8a1f --- /dev/null +++ b/packages/sync-rules/src/HydratedSyncRules.ts @@ -0,0 +1,153 @@ +import { Scope } from 'ajv/dist/compile/codegen/scope.js'; +import { BucketDataSource, CreateSourceParams, HydratedBucketSource } from './BucketSource.js'; +import { BucketDataScope, ParameterLookupScope } from './HydrationState.js'; +import { + ParameterIndexLookupCreator, + BucketParameterQuerier, + buildBucketName, + CompatibilityContext, + EvaluatedParameters, + EvaluatedRow, + EvaluationError, + GetBucketParameterQuerierResult, + GetQuerierOptions, + isEvaluatedParameters, + isEvaluatedRow, + isEvaluationError, + mergeBucketParameterQueriers, + mergeDataSources, + mergeParameterIndexLookupCreators, + QuerierError, + ScopedEvaluateParameterRow, + ScopedEvaluateRow, + SqlEventDescriptor, + SqliteInputValue, + SqliteValue, + SqlSyncRules +} from './index.js'; +import { SourceTableInterface } from './SourceTableInterface.js'; +import { EvaluatedParametersResult, EvaluateRowOptions, EvaluationResult, SqliteRow } from './types.js'; + +/** + * Hydrated sync rules is sync rule definitions along with persisted state. Currently, the persisted state + * specifically affects bucket names. + */ +export class HydratedSyncRules { + bucketSources: HydratedBucketSource[] = []; + eventDescriptors: SqlEventDescriptor[] = []; + compatibility: CompatibilityContext = CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY; + + readonly definition: SqlSyncRules; + + private readonly innerEvaluateRow: ScopedEvaluateRow; + private readonly innerEvaluateParameterRow: ScopedEvaluateParameterRow; + + constructor(params: { + definition: SqlSyncRules; + createParams: CreateSourceParams; + bucketDataSources: BucketDataSource[]; + bucketParameterIndexLookupCreators: ParameterIndexLookupCreator[]; + eventDescriptors?: SqlEventDescriptor[]; + compatibility?: CompatibilityContext; + }) { + const hydrationState = params.createParams.hydrationState; + + this.definition = params.definition; + this.innerEvaluateRow = mergeDataSources(hydrationState, params.bucketDataSources).evaluateRow; + this.innerEvaluateParameterRow = mergeParameterIndexLookupCreators( + hydrationState, + params.bucketParameterIndexLookupCreators + ).evaluateParameterRow; + + if (params.eventDescriptors) { + this.eventDescriptors = params.eventDescriptors; + } + if (params.compatibility) { + this.compatibility = params.compatibility; + } + + this.bucketSources = this.definition.bucketSources.map((source) => source.hydrate(params.createParams)); + } + + // These methods do not depend on hydration, so we can just forward them to the definition. + + getSourceTables() { + return this.definition.getSourceTables(); + } + + tableTriggersEvent(table: SourceTableInterface): boolean { + return this.definition.tableTriggersEvent(table); + } + + tableSyncsData(table: SourceTableInterface): boolean { + return this.definition.tableSyncsData(table); + } + + tableSyncsParameters(table: SourceTableInterface): boolean { + return this.definition.tableSyncsParameters(table); + } + + applyRowContext( + source: SqliteRow + ): SqliteRow { + return this.definition.applyRowContext(source); + } + + /** + * Throws errors. + */ + evaluateRow(options: EvaluateRowOptions): EvaluatedRow[] { + const { results, errors } = this.evaluateRowWithErrors(options); + if (errors.length > 0) { + throw new Error(errors[0].error); + } + return results; + } + + evaluateRowWithErrors(options: EvaluateRowOptions): { results: EvaluatedRow[]; errors: EvaluationError[] } { + const rawResults: EvaluationResult[] = this.innerEvaluateRow(options); + const results = rawResults.filter(isEvaluatedRow) as EvaluatedRow[]; + const errors = rawResults.filter(isEvaluationError) as EvaluationError[]; + + return { results, errors }; + } + + /** + * Throws errors. + */ + evaluateParameterRow(table: SourceTableInterface, row: SqliteRow): EvaluatedParameters[] { + const { results, errors } = this.evaluateParameterRowWithErrors(table, row); + if (errors.length > 0) { + throw new Error(errors[0].error); + } + return results; + } + + evaluateParameterRowWithErrors( + table: SourceTableInterface, + row: SqliteRow + ): { results: EvaluatedParameters[]; errors: EvaluationError[] } { + const rawResults: EvaluatedParametersResult[] = this.innerEvaluateParameterRow(table, row); + const results = rawResults.filter(isEvaluatedParameters) as EvaluatedParameters[]; + const errors = rawResults.filter(isEvaluationError) as EvaluationError[]; + return { results, errors }; + } + + getBucketParameterQuerier(options: GetQuerierOptions): GetBucketParameterQuerierResult { + const queriers: BucketParameterQuerier[] = []; + const errors: QuerierError[] = []; + const pending = { queriers, errors }; + + for (const source of this.bucketSources) { + if ( + (source.definition.subscribedToByDefault && options.hasDefaultStreams) || + source.definition.name in options.streams + ) { + source.pushBucketParameterQueriers(pending, options); + } + } + + const querier = mergeBucketParameterQueriers(queriers); + return { querier, errors }; + } +} diff --git a/packages/sync-rules/src/HydrationState.ts b/packages/sync-rules/src/HydrationState.ts new file mode 100644 index 000000000..f836a62b4 --- /dev/null +++ b/packages/sync-rules/src/HydrationState.ts @@ -0,0 +1,73 @@ +import { BucketDataSource, ParameterIndexLookupCreator } from './BucketSource.js'; + +export interface BucketDataScope { + /** The prefix is the bucket name before the parameters. */ + bucketPrefix: string; +} + +export interface ParameterLookupScope { + /** The lookup name + queryid is used to reference the parameter lookup record. */ + lookupName: string; + queryId: string; +} + +/** + * Hydration state information for a source. + * + * This is what keeps track of bucket name and parameter lookup mappings for hydration. This can be used + * both to re-use mappings across hydrations of different sync rule versions, or to generate new mappings. + */ +export interface HydrationState { + /** + * Given a bucket data source definition, get the bucket prefix to use for it. + */ + getBucketSourceScope(source: BucketDataSource): BucketDataScope; + + /** + * Given a bucket parameter lookup definition, get the persistence name to use. + */ + getParameterIndexLookupScope(source: ParameterIndexLookupCreator): ParameterLookupScope; +} + +/** + * This represents hydration state that performs no transformations. + * + * This is the legacy default behavior with no bucket versioning. + */ +export const DEFAULT_HYDRATION_STATE: HydrationState = { + getBucketSourceScope(source: BucketDataSource) { + return { + bucketPrefix: source.uniqueName + }; + }, + getParameterIndexLookupScope(source) { + return source.defaultLookupScope; + } +}; + +/** + * Transforms bucket ids generated when evaluating the row by e.g. encoding version information. + * + * Because buckets are recreated on a sync rule redeploy, it makes sense to use different bucket ids (otherwise, clients + * may run into checksum errors causing a sync to take longer than necessary or breaking progress). + * + * So, this transformer receives the original bucket id as generated by defined sync rules, and can prepend a version + * identifier. + * + * Note that this transformation has not been present in older versions of the sync service. To preserve backwards + * compatibility, sync rules will not use this without an opt-in. + */ +export function versionedHydrationState(version: number): HydrationState { + return { + getBucketSourceScope(source: BucketDataSource): BucketDataScope { + return { + bucketPrefix: `${version}#${source.uniqueName}` + }; + }, + + getParameterIndexLookupScope(source: ParameterIndexLookupCreator): ParameterLookupScope { + // No transformations applied here + return source.defaultLookupScope; + } + }; +} diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index c121b9038..04eb7f590 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -1,6 +1,11 @@ -import { BucketInclusionReason, ResolvedBucket } from './BucketDescription.js'; -import { BucketParameterQuerier, mergeBucketParameterQueriers, PendingQueriers } from './BucketParameterQuerier.js'; -import { BucketSource, BucketSourceType, ResultSetDescription } from './BucketSource.js'; +import { PendingQueriers } from './BucketParameterQuerier.js'; +import { + BucketDataSource, + BucketSource, + BucketSourceType, + CreateSourceParams, + HydratedBucketSource +} from './BucketSource.js'; import { ColumnDefinition } from './ExpressionType.js'; import { IdSequence } from './IdSequence.js'; import { SourceTableInterface } from './SourceTableInterface.js'; @@ -10,18 +15,9 @@ import { GetQuerierOptions, SyncRulesOptions } from './SqlSyncRules.js'; import { StaticSqlParameterQuery } from './StaticSqlParameterQuery.js'; import { TablePattern } from './TablePattern.js'; import { TableValuedFunctionSqlParameterQuery } from './TableValuedFunctionSqlParameterQuery.js'; -import { SqlRuleError } from './errors.js'; import { CompatibilityContext } from './compatibility.js'; -import { - BucketIdTransformer, - EvaluatedParametersResult, - EvaluateRowOptions, - EvaluationResult, - QueryParseOptions, - RequestParameters, - SourceSchema, - SqliteRow -} from './types.js'; +import { SqlRuleError } from './errors.js'; +import { EvaluateRowOptions, QueryParseOptions, SourceSchema, UnscopedEvaluationResult } from './types.js'; export interface QueryParseResult { /** @@ -34,7 +30,20 @@ export interface QueryParseResult { export class SqlBucketDescriptor implements BucketSource { name: string; - bucketParameters?: string[]; + private bucketParametersInternal: string[] | null = null; + + public readonly subscribedToByDefault: boolean = true; + + private readonly dataSource = new BucketDefinitionDataSource(this); + + /** + * source table -> queries + */ + dataQueries: SqlDataQuery[] = []; + parameterQueries: SqlParameterQuery[] = []; + globalParameterQueries: (StaticSqlParameterQuery | TableValuedFunctionSqlParameterQuery)[] = []; + + parameterIdSequence = new IdSequence(); constructor(name: string) { this.name = name; @@ -44,24 +53,27 @@ export class SqlBucketDescriptor implements BucketSource { return BucketSourceType.SYNC_RULE; } - public get subscribedToByDefault(): boolean { - return true; + public get bucketParameters(): string[] { + return this.bucketParametersInternal ?? []; } - /** - * source table -> queries - */ - dataQueries: SqlDataQuery[] = []; - parameterQueries: SqlParameterQuery[] = []; - globalParameterQueries: (StaticSqlParameterQuery | TableValuedFunctionSqlParameterQuery)[] = []; + get dataSources() { + return [this.dataSource]; + } - parameterIdSequence = new IdSequence(); + get parameterIndexLookupCreators() { + return this.parameterQueries; + } + + get parameterQuerierSources() { + return [...this.parameterQueries, ...this.globalParameterQueries]; + } addDataQuery(sql: string, options: SyncRulesOptions, compatibility: CompatibilityContext): QueryParseResult { - if (this.bucketParameters == null) { + if (this.bucketParametersInternal == null) { throw new Error('Bucket parameters must be defined'); } - const dataRows = SqlDataQuery.fromSql(this.name, this.bucketParameters, sql, options, compatibility); + const dataRows = SqlDataQuery.fromSql(this.bucketParametersInternal, sql, options, compatibility); this.dataQueries.push(dataRows); @@ -72,12 +84,19 @@ export class SqlBucketDescriptor implements BucketSource { } addParameterQuery(sql: string, options: QueryParseOptions): QueryParseResult { - const parameterQuery = SqlParameterQuery.fromSql(this.name, sql, options, this.parameterIdSequence.nextId()); - if (this.bucketParameters == null) { - this.bucketParameters = parameterQuery.bucketParameters; + const parameterQuery = SqlParameterQuery.fromSql( + this.name, + sql, + options, + this.parameterIdSequence.nextId(), + this.dataSource + ); + if (this.bucketParametersInternal == null) { + this.bucketParametersInternal = parameterQuery.bucketParameters; } else { if ( - new Set([...parameterQuery.bucketParameters!, ...this.bucketParameters]).size != this.bucketParameters.length + new Set([...parameterQuery.bucketParameters!, ...this.bucketParametersInternal]).size != + this.bucketParametersInternal.length ) { throw new Error('Bucket parameters must match for each parameter query within a bucket'); } @@ -94,114 +113,93 @@ export class SqlBucketDescriptor implements BucketSource { }; } - evaluateRow(options: EvaluateRowOptions): EvaluationResult[] { - let results: EvaluationResult[] = []; - for (let query of this.dataQueries) { - if (!query.applies(options.sourceTable)) { - continue; - } - - results.push(...query.evaluateRow(options.sourceTable, options.record, options.bucketIdTransformer)); - } - return results; + debugRepresentation() { + let all_parameter_queries = [...this.parameterQueries.values()].flat(); + let all_data_queries = [...this.dataQueries.values()].flat(); + return { + name: this.name, + type: BucketSourceType[this.type], + bucket_parameters: this.bucketParameters, + global_parameter_queries: this.globalParameterQueries.map((q) => { + return { + sql: q.sql + }; + }), + parameter_queries: all_parameter_queries.map((q) => { + return { + sql: q.sql, + table: q.sourceTable, + input_parameters: q.inputParameters + }; + }), + data_queries: all_data_queries.map((q) => { + return { + sql: q.sql, + table: q.sourceTable, + columns: q.columnOutputNames() + }; + }) + }; } - evaluateParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] { - let results: EvaluatedParametersResult[] = []; - for (let query of this.parameterQueries) { - if (query.applies(sourceTable)) { - results.push(...query.evaluateParameterRow(row)); + hydrate(params: CreateSourceParams): HydratedBucketSource { + const hydratedParameterQueriers = this.parameterQueries.map((querier) => + querier.createParameterQuerierSource(params) + ); + const hydratedGlobalParameterQueriers = this.globalParameterQueries.map((querier) => + querier.createParameterQuerierSource(params) + ); + + return { + definition: this, + pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { + for (let querier of hydratedParameterQueriers) { + querier.pushBucketParameterQueriers(result, options); + } + for (let querier of hydratedGlobalParameterQueriers) { + querier.pushBucketParameterQueriers(result, options); + } } - } - return results; + }; } +} + +export class BucketDefinitionDataSource implements BucketDataSource { + constructor(private descriptor: SqlBucketDescriptor) {} /** - * @deprecated Use `pushBucketParameterQueriers` instead and merge at the top-level. + * For debug use only. */ - getBucketParameterQuerier(options: GetQuerierOptions): BucketParameterQuerier { - const queriers: BucketParameterQuerier[] = []; - this.pushBucketParameterQueriers({ queriers, errors: [] }, options); - - return mergeBucketParameterQueriers(queriers); + get bucketParameters() { + return this.descriptor.bucketParameters; } - pushBucketParameterQueriers(result: PendingQueriers, options: GetQuerierOptions) { - const reasons = [this.bucketInclusionReason()]; - const staticBuckets = this.getStaticBucketDescriptions( - options.globalParameters, - reasons, - options.bucketIdTransformer - ); - const staticQuerier = { - staticBuckets, - hasDynamicBuckets: false, - parameterQueryLookups: [], - queryDynamicBucketDescriptions: async () => [] - } satisfies BucketParameterQuerier; - result.queriers.push(staticQuerier); - - if (this.parameterQueries.length == 0) { - return; - } - - const dynamicQueriers = this.parameterQueries.map((query) => - query.getBucketParameterQuerier(options.globalParameters, reasons, options.bucketIdTransformer) - ); - result.queriers.push(...dynamicQueriers); + public get uniqueName(): string { + return this.descriptor.name; } - getStaticBucketDescriptions( - parameters: RequestParameters, - reasons: BucketInclusionReason[], - transformer: BucketIdTransformer - ): ResolvedBucket[] { - let results: ResolvedBucket[] = []; - for (let query of this.globalParameterQueries) { - for (const desc of query.getStaticBucketDescriptions(parameters, transformer)) { - results.push({ - ...desc, - definition: this.name, - inclusion_reasons: reasons - }); + evaluateRow(options: EvaluateRowOptions) { + let results: UnscopedEvaluationResult[] = []; + for (let query of this.descriptor.dataQueries) { + if (!query.applies(options.sourceTable)) { + continue; } + + results.push(...query.evaluateRow(options.sourceTable, options.record)); } return results; } - hasDynamicBucketQueries(): boolean { - return this.parameterQueries.length > 0; - } - getSourceTables(): Set { let result = new Set(); - for (let query of this.parameterQueries) { - result.add(query.sourceTable!); + for (let query of this.descriptor.dataQueries) { + result.add(query.sourceTable); } - for (let query of this.dataQueries) { - result.add(query.sourceTable!); - } - - // Note: No physical tables for global_parameter_queries - return result; } - private bucketInclusionReason(): BucketInclusionReason { - return 'default'; - } - tableSyncsData(table: SourceTableInterface): boolean { - for (let query of this.dataQueries) { - if (query.applies(table)) { - return true; - } - } - return false; - } - - tableSyncsParameters(table: SourceTableInterface): boolean { - for (let query of this.parameterQueries) { + for (let query of this.descriptor.dataQueries) { if (query.applies(table)) { return true; } @@ -210,13 +208,13 @@ export class SqlBucketDescriptor implements BucketSource { } resolveResultSets(schema: SourceSchema, tables: Record>) { - for (let query of this.dataQueries) { + for (let query of this.descriptor.dataQueries) { query.resolveResultSets(schema, tables); } } debugWriteOutputTables(result: Record): void { - for (let q of this.dataQueries) { + for (let q of this.descriptor.dataQueries) { result[q.table!.sqlName] ??= []; const r = { query: q.sql @@ -225,33 +223,4 @@ export class SqlBucketDescriptor implements BucketSource { result[q.table!.sqlName].push(r); } } - - debugRepresentation() { - let all_parameter_queries = [...this.parameterQueries.values()].flat(); - let all_data_queries = [...this.dataQueries.values()].flat(); - return { - name: this.name, - type: BucketSourceType[this.type], - bucket_parameters: this.bucketParameters, - global_parameter_queries: this.globalParameterQueries.map((q) => { - return { - sql: q.sql - }; - }), - parameter_queries: all_parameter_queries.map((q) => { - return { - sql: q.sql, - table: q.sourceTable, - input_parameters: q.inputParameters - }; - }), - data_queries: all_data_queries.map((q) => { - return { - sql: q.sql, - table: q.sourceTable, - columns: q.columnOutputNames() - }; - }) - }; - } } diff --git a/packages/sync-rules/src/SqlDataQuery.ts b/packages/sync-rules/src/SqlDataQuery.ts index 30f0a2802..389189a23 100644 --- a/packages/sync-rules/src/SqlDataQuery.ts +++ b/packages/sync-rules/src/SqlDataQuery.ts @@ -1,6 +1,7 @@ import { JSONBig } from '@powersync/service-jsonbig'; import { parse } from 'pgsql-ast-parser'; import { BaseSqlDataQuery, BaseSqlDataQueryOptions, RowValueExtractor } from './BaseSqlDataQuery.js'; +import { CompatibilityContext } from './compatibility.js'; import { SqlRuleError } from './errors.js'; import { ExpressionType } from './ExpressionType.js'; import { SourceTableInterface } from './SourceTableInterface.js'; @@ -9,9 +10,8 @@ import { checkUnsupportedFeatures, isClauseError } from './sql_support.js'; import { SyncRulesOptions } from './SqlSyncRules.js'; import { TablePattern } from './TablePattern.js'; import { TableQuerySchema } from './TableQuerySchema.js'; -import { BucketIdTransformer, EvaluationResult, ParameterMatchClause, QuerySchema, SqliteRow } from './types.js'; -import { getBucketId, isSelectStatement } from './utils.js'; -import { CompatibilityContext } from './compatibility.js'; +import { ParameterMatchClause, QuerySchema, UnscopedEvaluationResult, SqliteRow } from './types.js'; +import { isSelectStatement, serializeBucketParameters } from './utils.js'; export interface SqlDataQueryOptions extends BaseSqlDataQueryOptions { filter: ParameterMatchClause; @@ -19,7 +19,6 @@ export interface SqlDataQueryOptions extends BaseSqlDataQueryOptions { export class SqlDataQuery extends BaseSqlDataQuery { static fromSql( - descriptorName: string, bucketParameters: string[], sql: string, options: SyncRulesOptions, @@ -170,7 +169,6 @@ export class SqlDataQuery extends BaseSqlDataQuery { sql, filter, columns: q.columns ?? [], - descriptorName, bucketParameters, tools, errors, @@ -192,19 +190,13 @@ export class SqlDataQuery extends BaseSqlDataQuery { this.filter = options.filter; } - evaluateRow( - table: SourceTableInterface, - row: SqliteRow, - bucketIdTransformer: BucketIdTransformer - ): EvaluationResult[] { + evaluateRow(table: SourceTableInterface, row: SqliteRow): UnscopedEvaluationResult[] { return this.evaluateRowWithOptions({ table, row, - bucketIds: (tables) => { + serializedBucketParameters: (tables) => { const bucketParameters = this.filter.filterRow(tables); - return bucketParameters.map((params) => - getBucketId(this.descriptorName, this.bucketParameters, params, bucketIdTransformer) - ); + return bucketParameters.map((params) => serializeBucketParameters(this.bucketParameters, params)); } }); } diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index 76a83a5f0..218724573 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -1,23 +1,35 @@ -import { parse, SelectedColumn } from 'pgsql-ast-parser'; +import { parse } from 'pgsql-ast-parser'; import { BucketDescription, BucketInclusionReason, BucketPriority, DEFAULT_BUCKET_PRIORITY } from './BucketDescription.js'; -import { BucketParameterQuerier, ParameterLookup, ParameterLookupSource } from './BucketParameterQuerier.js'; +import { + BucketParameterQuerier, + ParameterLookupSource, + PendingQueriers, + UnscopedParameterLookup +} from './BucketParameterQuerier.js'; +import { CreateSourceParams, ParameterIndexLookupCreator } from './BucketSource.js'; import { SqlRuleError } from './errors.js'; +import { BucketDataScope, ParameterLookupScope } from './HydrationState.js'; +import { + BucketDataSource, + BucketParameterQuerierSource, + GetQuerierOptions, + ScopedParameterLookup, + UnscopedEvaluatedParameters, + UnscopedEvaluatedParametersResult +} from './index.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; -import { checkUnsupportedFeatures, isClauseError, isParameterValueClause } from './sql_support.js'; +import { checkUnsupportedFeatures, isClauseError } from './sql_support.js'; import { StaticSqlParameterQuery } from './StaticSqlParameterQuery.js'; import { TablePattern } from './TablePattern.js'; import { TableQuerySchema } from './TableQuerySchema.js'; import { TableValuedFunctionSqlParameterQuery } from './TableValuedFunctionSqlParameterQuery.js'; import { - BucketIdTransformer, - EvaluatedParameters, - EvaluatedParametersResult, InputParameter, ParameterMatchClause, ParameterValueClause, @@ -29,7 +41,14 @@ import { SqliteJsonValue, SqliteRow } from './types.js'; -import { filterJsonRow, getBucketId, isJsonValue, isSelectStatement, normalizeParameterValue } from './utils.js'; +import { + buildBucketName, + filterJsonRow, + isJsonValue, + isSelectStatement, + normalizeParameterValue, + serializeBucketParameters +} from './utils.js'; import { DetectRequestParameters } from './validators.js'; export interface SqlParameterQueryOptions { @@ -46,6 +65,7 @@ export interface SqlParameterQueryOptions { bucketParameters: string[]; queryId: string; tools: SqlTools; + querierDataSource: BucketDataSource; errors?: SqlRuleError[]; } @@ -55,12 +75,13 @@ export interface SqlParameterQueryOptions { * SELECT id as user_id FROM users WHERE users.user_id = token_parameters.user_id * SELECT id as user_id, token_parameters.is_admin as is_admin FROM users WHERE users.user_id = token_parameters.user_id */ -export class SqlParameterQuery { +export class SqlParameterQuery implements ParameterIndexLookupCreator { static fromSql( descriptorName: string, sql: string, options: QueryParseOptions, - queryId: string + queryId: string, + querierDataSource: BucketDataSource ): SqlParameterQuery | StaticSqlParameterQuery | TableValuedFunctionSqlParameterQuery { const parsed = parse(sql, { locationTracking: true }); const schema = options?.schema; @@ -76,7 +97,7 @@ export class SqlParameterQuery { if (q.from == null) { // E.g. SELECT token_parameters.user_id as user_id WHERE token_parameters.is_admin - return StaticSqlParameterQuery.fromSql(descriptorName, sql, q, options, queryId); + return StaticSqlParameterQuery.fromSql(descriptorName, sql, q, options, queryId, querierDataSource); } let errors: SqlRuleError[] = []; @@ -87,7 +108,15 @@ export class SqlParameterQuery { throw new SqlRuleError('Must SELECT from a single table', sql, q.from?.[0]._location); } else if (q.from[0].type == 'call') { const from = q.from[0]; - return TableValuedFunctionSqlParameterQuery.fromSql(descriptorName, sql, from, q, options, queryId); + return TableValuedFunctionSqlParameterQuery.fromSql( + descriptorName, + sql, + from, + q, + options, + queryId, + querierDataSource + ); } else if (q.from[0].type == 'statement') { throw new SqlRuleError('Subqueries are not supported yet', sql, q.from?.[0]._location); } @@ -188,6 +217,7 @@ export class SqlParameterQuery { bucketParameters, queryId, tools, + querierDataSource, errors }); @@ -282,6 +312,8 @@ export class SqlParameterQuery { readonly queryId: string; readonly tools: SqlTools; + readonly querierDataSource: BucketDataSource; + readonly errors: SqlRuleError[]; constructor(options: SqlParameterQueryOptions) { @@ -299,25 +331,53 @@ export class SqlParameterQuery { this.queryId = options.queryId; this.tools = options.tools; this.errors = options.errors ?? []; + this.querierDataSource = options.querierDataSource; } - applies(table: SourceTableInterface) { + public get defaultLookupScope(): ParameterLookupScope { + return { + lookupName: this.descriptorName, + queryId: this.queryId + }; + } + + tableSyncsParameters(table: SourceTableInterface): boolean { return this.sourceTable.matches(table); } + getSourceTables(): Set { + return new Set([this.sourceTable]); + } + + createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { + const hydrationState = params.hydrationState; + const bucketScope = hydrationState.getBucketSourceScope(this.querierDataSource); + const lookupState = hydrationState.getParameterIndexLookupScope(this); + + return { + pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { + const q = this.getBucketParameterQuerier(options.globalParameters, ['default'], bucketScope, lookupState); + result.queriers.push(q); + } + }; + } + /** * Given a replicated row, results an array of bucket parameter rows to persist. */ - evaluateParameterRow(row: SqliteRow): EvaluatedParametersResult[] { + evaluateParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): UnscopedEvaluatedParametersResult[] { + if (!this.tableSyncsParameters(sourceTable)) { + return []; + } const tables = { [this.table.nameInSchema]: row }; try { const filterParameters = this.filter.filterRow(tables); - let result: EvaluatedParametersResult[] = []; + let result: UnscopedEvaluatedParametersResult[] = []; for (let filterParamSet of filterParameters) { - let lookup: SqliteJsonValue[] = [this.descriptorName, this.queryId]; - lookup.push( + let lookupValues: SqliteJsonValue[] = []; + lookupValues.push( ...this.inputParameters.map((param) => { return normalizeParameterValue(param.filteredRowToLookupValue(filterParamSet)); }) @@ -325,9 +385,9 @@ export class SqlParameterQuery { const data = this.transformRows(row); - const role: EvaluatedParameters = { + const role: UnscopedEvaluatedParameters = { bucketParameters: data.map((row) => filterJsonRow(row)), - lookup: new ParameterLookup(lookup) + lookup: UnscopedParameterLookup.normalized(lookupValues) }; result.push(role); } @@ -355,7 +415,7 @@ export class SqlParameterQuery { resolveBucketDescriptions( bucketParameters: SqliteJsonRow[], parameters: RequestParameters, - transformer: BucketIdTransformer + bucketScope: BucketDataScope ): BucketDescription[] { // Filters have already been applied and gotten us the set of bucketParameters - don't attempt to filter again. // We _do_ need to evaluate the output columns here, using a combination of precomputed bucketParameters, @@ -379,8 +439,10 @@ export class SqlParameterQuery { } } + const serializedParameters = serializeBucketParameters(this.bucketParameters, result); + return { - bucket: getBucketId(this.descriptorName, this.bucketParameters, result, transformer), + bucket: buildBucketName(bucketScope, serializedParameters), priority: this.priority }; }) @@ -392,12 +454,12 @@ export class SqlParameterQuery { * * Each lookup is [bucket definition name, parameter query index, ...lookup values] */ - getLookups(parameters: RequestParameters): ParameterLookup[] { + getLookups(parameters: RequestParameters): UnscopedParameterLookup[] { if (!this.expandedInputParameter) { - let lookup: SqliteJsonValue[] = [this.descriptorName, this.queryId]; + let lookupValues: SqliteJsonValue[] = []; let valid = true; - lookup.push( + lookupValues.push( ...this.inputParameters.map((param): SqliteJsonValue => { // Scalar value const value = param.parametersToLookupValue(parameters); @@ -413,7 +475,7 @@ export class SqlParameterQuery { if (!valid) { return []; } - return [new ParameterLookup(lookup)]; + return [UnscopedParameterLookup.normalized(lookupValues)]; } else { const arrayString = this.expandedInputParameter.parametersToLookupValue(parameters); @@ -432,10 +494,10 @@ export class SqlParameterQuery { return values .map((expandedValue) => { - let lookup: SqliteJsonValue[] = [this.descriptorName, this.queryId]; + let lookupValues: SqliteJsonValue[] = []; let valid = true; const normalizedExpandedValue = normalizeParameterValue(expandedValue); - lookup.push( + lookupValues.push( ...this.inputParameters.map((param): SqliteJsonValue => { if (param == this.expandedInputParameter) { // Expand array value @@ -457,18 +519,19 @@ export class SqlParameterQuery { return null; } - return new ParameterLookup(lookup); + return UnscopedParameterLookup.normalized(lookupValues); }) - .filter((lookup) => lookup != null) as ParameterLookup[]; + .filter((lookup) => lookup != null) as UnscopedParameterLookup[]; } } getBucketParameterQuerier( requestParameters: RequestParameters, reasons: BucketInclusionReason[], - transformer: BucketIdTransformer + bucketDataScope: BucketDataScope, + scope: ParameterLookupScope ): BucketParameterQuerier { - const lookups = this.getLookups(requestParameters); + const lookups = this.getLookups(requestParameters).map((lookup) => ScopedParameterLookup.normalized(scope, lookup)); if (lookups.length == 0) { // This typically happens when the query is pre-filtered using a where clause // on the parameters, and does not depend on the database state. @@ -486,7 +549,7 @@ export class SqlParameterQuery { parameterQueryLookups: lookups, queryDynamicBucketDescriptions: async (source: ParameterLookupSource) => { const bucketParameters = await source.getParameterSets(lookups); - return this.resolveBucketDescriptions(bucketParameters, requestParameters, transformer).map((bucket) => ({ + return this.resolveBucketDescriptions(bucketParameters, requestParameters, bucketDataScope).map((bucket) => ({ ...bucket, definition: this.descriptorName, inclusion_reasons: reasons diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index 216317702..5a6adfd24 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -1,23 +1,23 @@ import { isScalar, LineCounter, parseDocument, Scalar, YAMLMap, YAMLSeq } from 'yaml'; import { isValidPriority } from './BucketDescription.js'; -import { BucketParameterQuerier, mergeBucketParameterQueriers, QuerierError } from './BucketParameterQuerier.js'; +import { BucketParameterQuerier, QuerierError } from './BucketParameterQuerier.js'; +import { BucketDataSource, BucketSource, CreateSourceParams, ParameterIndexLookupCreator } from './BucketSource.js'; +import { + CompatibilityContext, + CompatibilityEdition, + CompatibilityOption, + TimeValuePrecision +} from './compatibility.js'; import { SqlRuleError, SyncRulesErrors, YamlError } from './errors.js'; import { SqlEventDescriptor } from './events/SqlEventDescriptor.js'; +import { HydratedSyncRules } from './HydratedSyncRules.js'; +import { DEFAULT_HYDRATION_STATE } from './HydrationState.js'; import { validateSyncRulesSchema } from './json_schema.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { QueryParseResult, SqlBucketDescriptor } from './SqlBucketDescriptor.js'; +import { syncStreamFromSql } from './streams/from_sql.js'; import { TablePattern } from './TablePattern.js'; import { - BucketIdTransformer, - EvaluatedParameters, - EvaluatedParametersResult, - EvaluatedRow, - EvaluateRowOptions, - EvaluationError, - EvaluationResult, - isEvaluatedParameters, - isEvaluatedRow, - isEvaluationError, QueryParseOptions, RequestParameters, SourceSchema, @@ -25,17 +25,8 @@ import { SqliteJsonRow, SqliteRow, SqliteValue, - StreamParseOptions, - SyncRules + StreamParseOptions } from './types.js'; -import { BucketSource } from './BucketSource.js'; -import { syncStreamFromSql } from './streams/from_sql.js'; -import { - CompatibilityContext, - CompatibilityEdition, - CompatibilityOption, - TimeValuePrecision -} from './compatibility.js'; import { applyRowContext } from './utils.js'; const ACCEPT_POTENTIALLY_DANGEROUS_QUERIES = Symbol('ACCEPT_POTENTIALLY_DANGEROUS_QUERIES'); @@ -68,13 +59,6 @@ export interface RequestedStream { } export interface GetQuerierOptions { - /** - * A bucket id transformer, compatible to the one used when evaluating rows. - * - * Typically, this transformer only depends on the sync rule id (which is known to both the bucket storage - * implementation responsible for evaluating rows and the sync endpoint). - */ - bucketIdTransformer: BucketIdTransformer; globalParameters: RequestParameters; /** * Whether the client is subscribing to default query streams. @@ -98,8 +82,11 @@ export interface GetBucketParameterQuerierResult { errors: QuerierError[]; } -export class SqlSyncRules implements SyncRules { +export class SqlSyncRules { + bucketDataSources: BucketDataSource[] = []; + bucketParameterLookupSources: ParameterIndexLookupCreator[] = []; bucketSources: BucketSource[] = []; + eventDescriptors: SqlEventDescriptor[] = []; compatibility: CompatibilityContext = CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY; @@ -261,7 +248,10 @@ export class SqlSyncRules implements SyncRules { return descriptor.addDataQuery(q, queryOptions, compatibility); }); } + rules.bucketSources.push(descriptor); + rules.bucketDataSources.push(...descriptor.dataSources); + rules.bucketParameterLookupSources.push(...descriptor.parameterIndexLookupCreators); } for (const entry of streamMap?.items ?? []) { @@ -287,6 +277,8 @@ export class SqlSyncRules implements SyncRules { rules.withScalar(data, (q) => { const [parsed, errors] = syncStreamFromSql(key, q, queryOptions); rules.bucketSources.push(parsed); + rules.bucketDataSources.push(...parsed.dataSources); + rules.bucketParameterLookupSources.push(...parsed.parameterIndexLookupCreators); return { parsed: true, errors @@ -401,101 +393,42 @@ export class SqlSyncRules implements SyncRules { this.content = content; } - applyRowContext( - source: SqliteRow - ): SqliteRow { - return applyRowContext(source, this.compatibility); - } - - /** - * Throws errors. - */ - evaluateRow(options: EvaluateRowOptions): EvaluatedRow[] { - const { results, errors } = this.evaluateRowWithErrors(options); - if (errors.length > 0) { - throw new Error(errors[0].error); - } - return results; - } - - evaluateRowWithErrors(options: EvaluateRowOptions): { results: EvaluatedRow[]; errors: EvaluationError[] } { - const resolvedOptions = this.compatibility.isEnabled(CompatibilityOption.versionedBucketIds) - ? options - : { - ...options, - // Disable bucket id transformer when the option is unused. - bucketIdTransformer: (id: string) => id - }; - - let rawResults: EvaluationResult[] = []; - for (let source of this.bucketSources) { - rawResults.push(...source.evaluateRow(resolvedOptions)); - } - - const results = rawResults.filter(isEvaluatedRow) as EvaluatedRow[]; - const errors = rawResults.filter(isEvaluationError) as EvaluationError[]; - - return { results, errors }; - } - /** - * Throws errors. + * Hydrate the sync rule definitions with persisted state into runnable sync rules. + * + * @param params.hydrationState Transforms bucket ids based on persisted state. May omit for tests. */ - evaluateParameterRow(table: SourceTableInterface, row: SqliteRow): EvaluatedParameters[] { - const { results, errors } = this.evaluateParameterRowWithErrors(table, row); - if (errors.length > 0) { - throw new Error(errors[0].error); - } - return results; - } - - evaluateParameterRowWithErrors( - table: SourceTableInterface, - row: SqliteRow - ): { results: EvaluatedParameters[]; errors: EvaluationError[] } { - let rawResults: EvaluatedParametersResult[] = []; - for (let source of this.bucketSources) { - rawResults.push(...source.evaluateParameterRow(table, row)); - } - - const results = rawResults.filter(isEvaluatedParameters) as EvaluatedParameters[]; - const errors = rawResults.filter(isEvaluationError) as EvaluationError[]; - return { results, errors }; - } - - getBucketParameterQuerier(options: GetQuerierOptions): GetBucketParameterQuerierResult { - const resolvedOptions = this.compatibility.isEnabled(CompatibilityOption.versionedBucketIds) - ? options - : { - ...options, - // Disable bucket id transformer when the option is unused. - bucketIdTransformer: (id: string) => id - }; - - const queriers: BucketParameterQuerier[] = []; - const errors: QuerierError[] = []; - const pending = { queriers, errors }; - - for (const source of this.bucketSources) { - if ( - (source.subscribedToByDefault && resolvedOptions.hasDefaultStreams) || - source.name in resolvedOptions.streams - ) { - source.pushBucketParameterQueriers(pending, resolvedOptions); - } + hydrate(params?: CreateSourceParams): HydratedSyncRules { + let hydrationState = params?.hydrationState; + if (hydrationState == null || !this.compatibility.isEnabled(CompatibilityOption.versionedBucketIds)) { + hydrationState = DEFAULT_HYDRATION_STATE; } - - const querier = mergeBucketParameterQueriers(queriers); - return { querier, errors }; + const resolvedParams = { hydrationState }; + return new HydratedSyncRules({ + definition: this, + createParams: resolvedParams, + bucketDataSources: this.bucketDataSources, + bucketParameterIndexLookupCreators: this.bucketParameterLookupSources, + eventDescriptors: this.eventDescriptors, + compatibility: this.compatibility + }); } - hasDynamicBucketQueries() { - return this.bucketSources.some((s) => s.hasDynamicBucketQueries()); + applyRowContext( + source: SqliteRow + ): SqliteRow { + return applyRowContext(source, this.compatibility); } getSourceTables(): TablePattern[] { const sourceTables = new Map(); - for (const bucket of this.bucketSources) { + for (const bucket of this.bucketDataSources) { + for (const r of bucket.getSourceTables()) { + const key = `${r.connectionTag}.${r.schema}.${r.tablePattern}`; + sourceTables.set(key, r); + } + } + for (const bucket of this.bucketParameterLookupSources) { for (const r of bucket.getSourceTables()) { const key = `${r.connectionTag}.${r.schema}.${r.tablePattern}`; sourceTables.set(key, r); @@ -529,21 +462,25 @@ export class SqlSyncRules implements SyncRules { } tableSyncsData(table: SourceTableInterface): boolean { - return this.bucketSources.some((b) => b.tableSyncsData(table)); + return this.bucketDataSources.some((b) => b.tableSyncsData(table)); } tableSyncsParameters(table: SourceTableInterface): boolean { - return this.bucketSources.some((b) => b.tableSyncsParameters(table)); + return this.bucketParameterLookupSources.some((b) => b.tableSyncsParameters(table)); } debugGetOutputTables() { let result: Record = {}; - for (let bucket of this.bucketSources) { + for (let bucket of this.bucketDataSources) { bucket.debugWriteOutputTables(result); } return result; } + debugRepresentation() { + return this.bucketSources.map((rules) => rules.debugRepresentation()); + } + private parsePriority(value: YAMLMap) { if (value.has('priority')) { const priorityValue = value.get('priority', true)!; @@ -556,8 +493,4 @@ export class SqlSyncRules implements SyncRules { } } } - - static versionedBucketIdTransformer(version: string) { - return (bucketId: string) => `${version}#${bucketId}`; - } } diff --git a/packages/sync-rules/src/StaticSqlParameterQuery.ts b/packages/sync-rules/src/StaticSqlParameterQuery.ts index 9a5d05f66..7a55dbb92 100644 --- a/packages/sync-rules/src/StaticSqlParameterQuery.ts +++ b/packages/sync-rules/src/StaticSqlParameterQuery.ts @@ -1,16 +1,16 @@ -import { SelectedColumn, SelectFromStatement } from 'pgsql-ast-parser'; -import { BucketDescription, BucketPriority, DEFAULT_BUCKET_PRIORITY } from './BucketDescription.js'; +import { SelectFromStatement } from 'pgsql-ast-parser'; +import { BucketDescription, BucketPriority, DEFAULT_BUCKET_PRIORITY, ResolvedBucket } from './BucketDescription.js'; +import { BucketParameterQuerier, PendingQueriers } from './BucketParameterQuerier.js'; +import { CreateSourceParams } from './BucketSource.js'; import { SqlRuleError } from './errors.js'; +import { BucketDataScope } from './HydrationState.js'; +import { BucketDataSource, BucketParameterQuerierSource, GetQuerierOptions } from './index.js'; +import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; -import { checkUnsupportedFeatures, isClauseError, isParameterValueClause, sqliteBool } from './sql_support.js'; -import { - BucketIdTransformer, - ParameterValueClause, - QueryParseOptions, - RequestParameters, - SqliteJsonValue -} from './types.js'; -import { getBucketId, isJsonValue } from './utils.js'; +import { checkUnsupportedFeatures, isClauseError, sqliteBool } from './sql_support.js'; +import { TablePattern } from './TablePattern.js'; +import { ParameterValueClause, QueryParseOptions, RequestParameters, SqliteJsonValue } from './types.js'; +import { buildBucketName, isJsonValue, serializeBucketParameters } from './utils.js'; import { DetectRequestParameters } from './validators.js'; export interface StaticSqlParameterQueryOptions { @@ -21,6 +21,7 @@ export interface StaticSqlParameterQueryOptions { bucketParameters: string[]; queryId: string; filter: ParameterValueClause | undefined; + querierDataSource: BucketDataSource; errors?: SqlRuleError[]; } @@ -36,7 +37,8 @@ export class StaticSqlParameterQuery { sql: string, q: SelectFromStatement, options: QueryParseOptions, - queryId: string + queryId: string, + querierDataSource: BucketDataSource ) { let errors: SqlRuleError[] = []; @@ -93,6 +95,7 @@ export class StaticSqlParameterQuery { priority: priority ?? DEFAULT_BUCKET_PRIORITY, filter: isClauseError(filter) ? undefined : filter, queryId, + querierDataSource, errors }); if (query.usesDangerousRequestParameters && !options?.accept_potentially_dangerous_queries) { @@ -148,6 +151,8 @@ export class StaticSqlParameterQuery { */ readonly filter: ParameterValueClause | undefined; + public readonly querierDataSource: BucketDataSource; + readonly errors: SqlRuleError[]; constructor(options: StaticSqlParameterQueryOptions) { @@ -158,10 +163,46 @@ export class StaticSqlParameterQuery { this.bucketParameters = options.bucketParameters; this.queryId = options.queryId; this.filter = options.filter; + this.querierDataSource = options.querierDataSource; this.errors = options.errors ?? []; } - getStaticBucketDescriptions(parameters: RequestParameters, transformer: BucketIdTransformer): BucketDescription[] { + getSourceTables() { + return new Set(); + } + + tableSyncsParameters(_table: SourceTableInterface): boolean { + return false; + } + + createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { + const hydrationState = params.hydrationState; + const bucketScope = hydrationState.getBucketSourceScope(this.querierDataSource); + return { + pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { + const staticBuckets = this.getStaticBucketDescriptions(options.globalParameters, bucketScope).map((desc) => { + return { + ...desc, + definition: this.descriptorName, + inclusion_reasons: ['default'] + } satisfies ResolvedBucket; + }); + + if (staticBuckets.length == 0) { + return; + } + const staticQuerier = { + staticBuckets, + hasDynamicBuckets: false, + parameterQueryLookups: [], + queryDynamicBucketDescriptions: async () => [] + } satisfies BucketParameterQuerier; + result.queriers.push(staticQuerier); + } + }; + } + + getStaticBucketDescriptions(parameters: RequestParameters, bucketSourceScope: BucketDataScope): BucketDescription[] { if (this.filter == null) { // Error in filter clause return []; @@ -183,9 +224,11 @@ export class StaticSqlParameterQuery { } } + const serializedParamters = serializeBucketParameters(this.bucketParameters, result); + return [ { - bucket: getBucketId(this.descriptorName, this.bucketParameters, result, transformer), + bucket: buildBucketName(bucketSourceScope, serializedParamters), priority: this.priority } ]; diff --git a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts index 8b8da15bf..d4e9bfff7 100644 --- a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts +++ b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts @@ -1,10 +1,21 @@ import { FromCall, SelectFromStatement } from 'pgsql-ast-parser'; +import { BucketDescription, BucketPriority, DEFAULT_BUCKET_PRIORITY, ResolvedBucket } from './BucketDescription.js'; +import { CreateSourceParams } from './BucketSource.js'; import { SqlRuleError } from './errors.js'; +import { BucketDataScope } from './HydrationState.js'; +import { + BucketDataSource, + BucketParameterQuerier, + BucketParameterQuerierSource, + GetQuerierOptions, + PendingQueriers +} from './index.js'; +import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; -import { checkUnsupportedFeatures, isClauseError, isParameterValueClause, sqliteBool } from './sql_support.js'; +import { checkUnsupportedFeatures, isClauseError, sqliteBool } from './sql_support.js'; +import { TablePattern } from './TablePattern.js'; import { generateTableValuedFunctions, TableValuedFunction } from './TableValuedFunctions.js'; import { - BucketIdTransformer, ParameterValueClause, ParameterValueSet, QueryParseOptions, @@ -12,8 +23,7 @@ import { SqliteJsonValue, SqliteRow } from './types.js'; -import { getBucketId, isJsonValue } from './utils.js'; -import { BucketDescription, BucketPriority, DEFAULT_BUCKET_PRIORITY } from './BucketDescription.js'; +import { buildBucketName, isJsonValue, serializeBucketParameters } from './utils.js'; import { DetectRequestParameters } from './validators.js'; export interface TableValuedFunctionSqlParameterQueryOptions { @@ -28,6 +38,7 @@ export interface TableValuedFunctionSqlParameterQueryOptions { callClause: ParameterValueClause | undefined; function: TableValuedFunction; callTable: AvailableTable; + querierDataSource: BucketDataSource; errors: SqlRuleError[]; } @@ -48,7 +59,8 @@ export class TableValuedFunctionSqlParameterQuery { call: FromCall, q: SelectFromStatement, options: QueryParseOptions, - queryId: string + queryId: string, + querierDataSource: BucketDataSource ): TableValuedFunctionSqlParameterQuery { const compatibility = options.compatibility; let errors: SqlRuleError[] = []; @@ -112,6 +124,7 @@ export class TableValuedFunctionSqlParameterQuery { callTable, priority: priority ?? DEFAULT_BUCKET_PRIORITY, queryId, + querierDataSource, errors }); @@ -189,6 +202,8 @@ export class TableValuedFunctionSqlParameterQuery { */ readonly callTable: AvailableTable; + public readonly querierDataSource: BucketDataSource; + readonly errors: SqlRuleError[]; constructor(options: TableValuedFunctionSqlParameterQueryOptions) { @@ -198,6 +213,7 @@ export class TableValuedFunctionSqlParameterQuery { this.descriptorName = options.descriptorName; this.bucketParameters = options.bucketParameters; this.queryId = options.queryId; + this.querierDataSource = options.querierDataSource; this.filter = options.filter; this.callClause = options.callClause; @@ -207,7 +223,42 @@ export class TableValuedFunctionSqlParameterQuery { this.errors = options.errors; } - getStaticBucketDescriptions(parameters: RequestParameters, transformer: BucketIdTransformer): BucketDescription[] { + getSourceTables() { + return new Set(); + } + + tableSyncsParameters(_table: SourceTableInterface): boolean { + return false; + } + + createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { + const hydrationState = params.hydrationState; + const bucketScope = hydrationState.getBucketSourceScope(this.querierDataSource); + return { + pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { + const staticBuckets = this.getStaticBucketDescriptions(options.globalParameters, bucketScope).map((desc) => { + return { + ...desc, + definition: this.descriptorName, + inclusion_reasons: ['default'] + } satisfies ResolvedBucket; + }); + + if (staticBuckets.length == 0) { + return; + } + const staticQuerier = { + staticBuckets, + hasDynamicBuckets: false, + parameterQueryLookups: [], + queryDynamicBucketDescriptions: async () => [] + } satisfies BucketParameterQuerier; + result.queriers.push(staticQuerier); + } + }; + } + + getStaticBucketDescriptions(parameters: RequestParameters, scope: BucketDataScope): BucketDescription[] { if (this.filter == null || this.callClause == null) { // Error in filter clause return []; @@ -217,7 +268,7 @@ export class TableValuedFunctionSqlParameterQuery { const rows = this.function.call([valueString]); let total: BucketDescription[] = []; for (let row of rows) { - const description = this.getIndividualBucketDescription(row, parameters, transformer); + const description = this.getIndividualBucketDescription(row, parameters, scope); if (description !== null) { total.push(description); } @@ -228,7 +279,7 @@ export class TableValuedFunctionSqlParameterQuery { private getIndividualBucketDescription( row: SqliteRow, parameters: RequestParameters, - transformer: BucketIdTransformer + bucketScope: BucketDataScope ): BucketDescription | null { const mergedParams: ParameterValueSet = { ...parameters, @@ -255,8 +306,10 @@ export class TableValuedFunctionSqlParameterQuery { } } + const serializedBucketParameters = serializeBucketParameters(this.bucketParameters, result); + return { - bucket: getBucketId(this.descriptorName, this.bucketParameters, result, transformer), + bucket: buildBucketName(bucketScope, serializedBucketParameters), priority: this.priority }; } diff --git a/packages/sync-rules/src/events/SqlEventDescriptor.ts b/packages/sync-rules/src/events/SqlEventDescriptor.ts index 8ea79f67e..33c34b98e 100644 --- a/packages/sync-rules/src/events/SqlEventDescriptor.ts +++ b/packages/sync-rules/src/events/SqlEventDescriptor.ts @@ -22,7 +22,7 @@ export class SqlEventDescriptor { } addSourceQuery(sql: string, options: SyncRulesOptions): QueryParseResult { - const source = SqlEventSourceQuery.fromSql(this.name, sql, options, this.compatibility); + const source = SqlEventSourceQuery.fromSql(sql, options, this.compatibility); // Each source query should be for a unique table const existingSourceQuery = this.sourceQueries.find((q) => q.table == source.table); diff --git a/packages/sync-rules/src/events/SqlEventSourceQuery.ts b/packages/sync-rules/src/events/SqlEventSourceQuery.ts index 3558eb6d3..17d263e8c 100644 --- a/packages/sync-rules/src/events/SqlEventSourceQuery.ts +++ b/packages/sync-rules/src/events/SqlEventSourceQuery.ts @@ -25,7 +25,7 @@ export type EvaluatedEventRowWithErrors = { * Defines how a Replicated Row is mapped to source parameters for events. */ export class SqlEventSourceQuery extends BaseSqlDataQuery { - static fromSql(descriptor_name: string, sql: string, options: SyncRulesOptions, compatibility: CompatibilityContext) { + static fromSql(sql: string, options: SyncRulesOptions, compatibility: CompatibilityContext) { const parsed = parse(sql, { locationTracking: true }); const schema = options.schema; @@ -121,7 +121,6 @@ export class SqlEventSourceQuery extends BaseSqlDataQuery { sourceTable, table: alias, sql, - descriptorName: descriptor_name, columns: q.columns ?? [], extractors: extractors, tools, diff --git a/packages/sync-rules/src/index.ts b/packages/sync-rules/src/index.ts index 8ebeb4115..4609714fd 100644 --- a/packages/sync-rules/src/index.ts +++ b/packages/sync-rules/src/index.ts @@ -27,3 +27,4 @@ export * from './types.js'; export * from './types/custom_sqlite_value.js'; export * from './types/time.js'; export * from './utils.js'; +export * from './HydratedSyncRules.js'; diff --git a/packages/sync-rules/src/request_functions.ts b/packages/sync-rules/src/request_functions.ts index 585916ec3..33c431a7c 100644 --- a/packages/sync-rules/src/request_functions.ts +++ b/packages/sync-rules/src/request_functions.ts @@ -1,5 +1,5 @@ import { ExpressionType } from './ExpressionType.js'; -import { CompatibilityContext, CompatibilityEdition, CompatibilityOption } from './compatibility.js'; +import { CompatibilityContext, CompatibilityEdition } from './compatibility.js'; import { generateSqlFunctions } from './sql_functions.js'; import { CompiledClause, ParameterValueClause, ParameterValueSet, SqliteValue } from './types.js'; diff --git a/packages/sync-rules/src/schema-generators/SchemaGenerator.ts b/packages/sync-rules/src/schema-generators/SchemaGenerator.ts index e257f5726..84619ec9d 100644 --- a/packages/sync-rules/src/schema-generators/SchemaGenerator.ts +++ b/packages/sync-rules/src/schema-generators/SchemaGenerator.ts @@ -10,7 +10,7 @@ export abstract class SchemaGenerator { protected getAllTables(source: SqlSyncRules, schema: SourceSchema) { let tables: Record> = {}; - for (let descriptor of source.bucketSources) { + for (let descriptor of source.bucketDataSources) { descriptor.resolveResultSets(schema, tables); } diff --git a/packages/sync-rules/src/streams/filter.ts b/packages/sync-rules/src/streams/filter.ts index 05c6e34f1..4ef22eab6 100644 --- a/packages/sync-rules/src/streams/filter.ts +++ b/packages/sync-rules/src/streams/filter.ts @@ -1,15 +1,27 @@ +import { ScopedParameterLookup, UnscopedParameterLookup } from '../BucketParameterQuerier.js'; +import { SqlTools } from '../sql_filters.js'; +import { checkJsonArray, OPERATOR_NOT } from '../sql_functions.js'; import { isParameterValueClause, isRowValueClause, SQLITE_TRUE, sqliteBool } from '../sql_support.js'; import { TablePattern } from '../TablePattern.js'; -import { ParameterMatchClause, ParameterValueClause, RowValueClause, SqliteJsonValue } from '../types.js'; +import { + EvaluatedParametersResult, + ParameterMatchClause, + ParameterValueClause, + RequestParameters, + RowValueClause, + SqliteJsonValue, + SqliteRow, + UnscopedEvaluatedParametersResult +} from '../types.js'; import { isJsonValue, normalizeParameterValue } from '../utils.js'; -import { SqlTools } from '../sql_filters.js'; -import { checkJsonArray, OPERATOR_NOT } from '../sql_functions.js'; -import { ParameterLookup } from '../BucketParameterQuerier.js'; -import { StreamVariant } from './variant.js'; +import { NodeLocation } from 'pgsql-ast-parser'; +import { ParameterIndexLookupCreator, CreateSourceParams } from '../BucketSource.js'; +import { HydrationState, ParameterLookupScope } from '../HydrationState.js'; +import { SourceTableInterface } from '../SourceTableInterface.js'; import { SubqueryEvaluator } from './parameter.js'; import { cartesianProduct } from './utils.js'; -import { NodeLocation } from 'pgsql-ast-parser'; +import { StreamVariant } from './variant.js'; /** * An intermediate representation of a `WHERE` clause for stream queries. @@ -251,33 +263,41 @@ export class Subquery { const column = this.column; - const evaluator: SubqueryEvaluator = { - parameterTable: this.table, - lookupsForParameterRow(sourceTable, row) { - const value = column.evaluate({ [sourceTable.name]: row }); - if (!isJsonValue(value)) { - return null; - } - - const lookups: ParameterLookup[] = []; - for (const [variant, id] of innerVariants) { - for (const instantiation of variant.instantiationsForRow({ sourceTable, record: row })) { - lookups.push(ParameterLookup.normalized(context.streamName, id, instantiation)); - } - } - return { value, lookups }; - }, - lookupsForRequest(parameters) { - const lookups: ParameterLookup[] = []; - - for (const [variant, id] of innerVariants) { + let lookupCreators: ParameterIndexLookupCreator[] = []; + let lookupsForRequest: (( + hydrationState: HydrationState + ) => (parameters: RequestParameters) => ScopedParameterLookup[])[] = []; + + for (let [variant, id] of innerVariants) { + const source = new SubqueryParameterLookupSource(this.table, column, variant, id, context.streamName); + lookupCreators.push(source); + lookupsForRequest.push((hydrationState: HydrationState) => { + const scope = hydrationState.getParameterIndexLookupScope(source); + return (parameters: RequestParameters) => { + const lookups: ScopedParameterLookup[] = []; const instantiations = variant.findStaticInstantiations(parameters); for (const instantiation of instantiations) { - lookups.push(ParameterLookup.normalized(context.streamName, id, instantiation)); + lookups.push(ScopedParameterLookup.normalized(scope, UnscopedParameterLookup.normalized(instantiation))); } - } + return lookups; + }; + }); + } - return lookups; + const evaluator: SubqueryEvaluator = { + parameterTable: this.table, + indexLookupCreators() { + return lookupCreators; + }, + hydrateLookupsForRequest(hydrationState: HydrationState) { + const hydrated = lookupsForRequest.map((fn) => fn(hydrationState)); + return (parameters: RequestParameters) => { + const lookups: ScopedParameterLookup[] = []; + for (const getLookups of hydrated) { + lookups.push(...getLookups(parameters)); + } + return lookups; + }; } }; @@ -510,3 +530,69 @@ export class EvaluateSimpleCondition extends FilterOperator { ); } } + +export class SubqueryParameterLookupSource implements ParameterIndexLookupCreator { + constructor( + private parameterTable: TablePattern, + private column: RowValueClause, + private innerVariant: StreamVariant, + private defaultQueryId: string, + private streamName: string + ) {} + + public get defaultLookupScope() { + return { + lookupName: this.streamName, + queryId: this.defaultQueryId + }; + } + + getSourceTables(): Set { + let result = new Set(); + result.add(this.parameterTable); + return result; + } + + /** + * Creates lookup indices for dynamically-resolved parameters. + * + * Resolving dynamic parameters is a two-step process: First, for tables referenced in subqueries, we create an index + * to resolve which request parameters would match rows in subqueries. Then, when resolving bucket ids for a request, + * we compute subquery results by looking up results in that index. + * + * This implements the first step of that process. + * + * @param result The array into which evaluation results should be written to. + * @param sourceTable A table we depend on in a subquery. + * @param row Row data to index. + */ + evaluateParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): UnscopedEvaluatedParametersResult[] { + if (this.parameterTable.matches(sourceTable)) { + // Theoretically we're doing duplicate work by doing this for each innerVariant in a subquery. + // In practice, we don't have more than one innerVariant per subquery right now, so this is fine. + const value = this.column.evaluate({ [sourceTable.name]: row }); + if (!isJsonValue(value)) { + return []; + } + + const lookups: UnscopedParameterLookup[] = []; + for (const instantiation of this.innerVariant.instantiationsForRow({ sourceTable, record: row })) { + lookups.push(UnscopedParameterLookup.normalized(instantiation)); + } + + // The row of the subquery. Since we only support subqueries with a single column, we unconditionally name the + // column `result` for simplicity. + const resultRow = { result: value }; + + return lookups.map((l) => ({ + lookup: l, + bucketParameters: [resultRow] + })); + } + return []; + } + + tableSyncsParameters(table: SourceTableInterface): boolean { + return this.parameterTable.matches(table); + } +} diff --git a/packages/sync-rules/src/streams/from_sql.ts b/packages/sync-rules/src/streams/from_sql.ts index ba325167a..e11c594b2 100644 --- a/packages/sync-rules/src/streams/from_sql.ts +++ b/packages/sync-rules/src/streams/from_sql.ts @@ -100,14 +100,13 @@ class SyncStreamCompiler { let filter = this.whereClauseToFilters(tools, query.where); filter = filter.toDisjunctiveNormalForm(tools); + const variants = filter.isValid(tools) ? filter.compileVariants(this.descriptorName) : []; const stream = new SyncStream( this.descriptorName, - new BaseSqlDataQuery(this.compileDataQuery(tools, query, alias, sourceTable)) + new BaseSqlDataQuery(this.compileDataQuery(tools, query, alias, sourceTable)), + variants ); stream.subscribedToByDefault = this.options.auto_subscribe ?? false; - if (filter.isValid(tools)) { - stream.variants = filter.compileVariants(this.descriptorName); - } this.errors.push(...tools.errors); if (this.parameterDetector.usesStreamParameters && stream.subscribedToByDefault) { @@ -202,7 +201,6 @@ class SyncStreamCompiler { table: alias, sql: this.sql, columns: query.columns ?? [], - descriptorName: this.descriptorName, tools, extractors, // Streams don't have traditional parameters, and parameters aren't used in the rest of the stream implementation. diff --git a/packages/sync-rules/src/streams/parameter.ts b/packages/sync-rules/src/streams/parameter.ts index 873216385..e92936374 100644 --- a/packages/sync-rules/src/streams/parameter.ts +++ b/packages/sync-rules/src/streams/parameter.ts @@ -1,15 +1,8 @@ -import { ParameterLookup } from '../BucketParameterQuerier.js'; -import { SourceTableInterface } from '../SourceTableInterface.js'; +import { ScopedParameterLookup, UnscopedParameterLookup } from '../BucketParameterQuerier.js'; +import { ParameterIndexLookupCreator } from '../BucketSource.js'; +import { HydrationState } from '../HydrationState.js'; import { TablePattern } from '../TablePattern.js'; -import { - EvaluateRowOptions, - ParameterValueSet, - RequestParameters, - SqliteJsonValue, - SqliteRow, - SqliteValue, - TableRow -} from '../types.js'; +import { ParameterValueSet, RequestParameters, SqliteJsonValue, SqliteValue, TableRow } from '../types.js'; /** * A source of parameterization, causing data from the source table to be distributed into multiple buckets instead of @@ -38,12 +31,17 @@ export interface BucketParameter { export interface SubqueryEvaluator { parameterTable: TablePattern; - lookupsForParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): SubqueryLookups | null; - lookupsForRequest(params: RequestParameters): ParameterLookup[]; + indexLookupCreators(): ParameterIndexLookupCreator[]; + // TODO: Is there a better design here? + // This is used for parameter _queries_. But the queries need to know which lookup scopes to + // use, and each querier may use multiple lookup sources, each with their own scope. + // This implementation here does "hydration" on each subquery, which gives us hydrated function call. + // Should this maybe be part of a higher-level class instead of just a function, i.e. a hydrated subquery? + hydrateLookupsForRequest(hydrationState: HydrationState): (params: RequestParameters) => ScopedParameterLookup[]; } export interface SubqueryLookups { - lookups: ParameterLookup[]; + lookups: UnscopedParameterLookup[]; /** * The value that the single column in the subquery evaluated to. */ diff --git a/packages/sync-rules/src/streams/stream.ts b/packages/sync-rules/src/streams/stream.ts index 01070b74d..3394cfa51 100644 --- a/packages/sync-rules/src/streams/stream.ts +++ b/packages/sync-rules/src/streams/stream.ts @@ -1,144 +1,111 @@ import { BaseSqlDataQuery } from '../BaseSqlDataQuery.js'; -import { BucketInclusionReason, BucketPriority, DEFAULT_BUCKET_PRIORITY } from '../BucketDescription.js'; -import { BucketParameterQuerier, PendingQueriers } from '../BucketParameterQuerier.js'; -import { BucketSource, BucketSourceType, ResultSetDescription } from '../BucketSource.js'; +import { BucketPriority, DEFAULT_BUCKET_PRIORITY } from '../BucketDescription.js'; +import { + BucketDataSource, + ParameterIndexLookupCreator, + BucketSource, + BucketSourceType, + CreateSourceParams, + HydratedBucketSource, + BucketParameterQuerierSource +} from '../BucketSource.js'; import { ColumnDefinition } from '../ExpressionType.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; -import { GetQuerierOptions, RequestedStream } from '../SqlSyncRules.js'; import { TablePattern } from '../TablePattern.js'; -import { - BucketIdTransformer, - EvaluatedParametersResult, - EvaluateRowOptions, - EvaluationResult, - RequestParameters, - SourceSchema, - SqliteRow, - TableRow -} from '../types.js'; +import { EvaluateRowOptions, UnscopedEvaluationResult, SourceSchema, TableRow } from '../types.js'; import { StreamVariant } from './variant.js'; export class SyncStream implements BucketSource { name: string; subscribedToByDefault: boolean; priority: BucketPriority; - variants: StreamVariant[]; + variants: { variant: StreamVariant; dataSource: SyncStreamDataSource }[]; data: BaseSqlDataQuery; - constructor(name: string, data: BaseSqlDataQuery) { + public readonly dataSources: BucketDataSource[]; + public readonly parameterIndexLookupCreators: ParameterIndexLookupCreator[]; + + constructor(name: string, data: BaseSqlDataQuery, variants: StreamVariant[]) { this.name = name; this.subscribedToByDefault = false; this.priority = DEFAULT_BUCKET_PRIORITY; this.variants = []; this.data = data; + + this.dataSources = []; + this.parameterIndexLookupCreators = []; + + for (let variant of variants) { + const dataSource = new SyncStreamDataSource(this, data, variant); + this.dataSources.push(dataSource); + this.variants.push({ variant, dataSource }); + const lookupCreators = variant.indexLookupCreators(); + this.parameterIndexLookupCreators.push(...lookupCreators); + } } public get type(): BucketSourceType { return BucketSourceType.SYNC_STREAM; } - pushBucketParameterQueriers(result: PendingQueriers, options: GetQuerierOptions): void { - const subscriptions = options.streams[this.name] ?? []; - - if (!this.subscribedToByDefault && !subscriptions.length) { - // The client is not subscribing to this stream, so don't query buckets related to it. - return; - } - - let hasExplicitDefaultSubscription = false; - for (const subscription of subscriptions) { - let subscriptionParams = options.globalParameters; - if (subscription.parameters != null) { - subscriptionParams = subscriptionParams.withAddedStreamParameters(subscription.parameters); - } else { - hasExplicitDefaultSubscription = true; + debugRepresentation() { + return { + name: this.name, + type: BucketSourceType[BucketSourceType.SYNC_STREAM], + variants: this.variants.map(({ variant }) => variant.debugRepresentation()), + data: { + table: this.data.sourceTable, + columns: this.data.columnOutputNames() } + }; + } - this.queriersForSubscription(result, subscription, subscriptionParams, options.bucketIdTransformer); - } - - // If the stream is subscribed to by default and there is no explicit subscription that would match the default - // subscription, also include the default querier. - if (this.subscribedToByDefault && !hasExplicitDefaultSubscription) { - this.queriersForSubscription(result, null, options.globalParameters, options.bucketIdTransformer); + hydrate(params: CreateSourceParams): HydratedBucketSource { + let queriers: BucketParameterQuerierSource[] = []; + for (let { variant, dataSource } of this.variants) { + const querier = variant.createParameterQuerierSource(params, this, dataSource); + queriers.push(querier); } - } - private queriersForSubscription( - result: PendingQueriers, - subscription: RequestedStream | null, - params: RequestParameters, - bucketIdTransformer: BucketIdTransformer - ) { - const reason: BucketInclusionReason = subscription != null ? { subscription: subscription.opaque_id } : 'default'; - const queriers: BucketParameterQuerier[] = []; - - try { - for (const variant of this.variants) { - const querier = variant.querier(this, reason, params, bucketIdTransformer); - if (querier) { - queriers.push(querier); + return { + definition: this, + pushBucketParameterQueriers(result, options) { + for (let querier of queriers) { + querier.pushBucketParameterQueriers(result, options); } } - - result.queriers.push(...queriers); - } catch (e) { - result.errors.push({ - descriptor: this.name, - message: `Error evaluating bucket ids: ${e.message}`, - subscription: subscription ?? undefined - }); - } - } - - hasDynamicBucketQueries(): boolean { - return this.variants.some((v) => v.hasDynamicBucketQueries); + }; } +} - tableSyncsData(table: SourceTableInterface): boolean { - return this.data.applies(table); +export class SyncStreamDataSource implements BucketDataSource { + constructor( + private stream: SyncStream, + private data: BaseSqlDataQuery, + private variant: StreamVariant + ) {} + + /** + * Not relevant for sync streams. + */ + get bucketParameters() { + return []; } - tableSyncsParameters(table: SourceTableInterface): boolean { - for (const variant of this.variants) { - for (const subquery of variant.subqueries) { - if (subquery.parameterTable.matches(table)) { - return true; - } - } - } - - return false; + public get uniqueName(): string { + return this.variant.defaultBucketPrefix(this.stream.name); } getSourceTables(): Set { - let result = new Set(); - result.add(this.data.sourceTable); - for (let variant of this.variants) { - for (const subquery of variant.subqueries) { - result.add(subquery.parameterTable); - } - } - - // Note: No physical tables for global_parameter_queries - - return result; + return new Set([this.data.sourceTable]); } - resolveResultSets(schema: SourceSchema, tables: Record>) { - this.data.resolveResultSets(schema, tables); + tableSyncsData(table: SourceTableInterface): boolean { + return this.data.applies(table); } - debugRepresentation() { - return { - name: this.name, - type: BucketSourceType[this.type], - variants: this.variants.map((v) => v.debugRepresentation()), - data: { - table: this.data.sourceTable, - columns: this.data.columnOutputNames() - } - }; + resolveResultSets(schema: SourceSchema, tables: Record>): void { + return this.data.resolveResultSets(schema, tables); } debugWriteOutputTables(result: Record): void { @@ -150,37 +117,24 @@ export class SyncStream implements BucketSource { result[this.data.table!.sqlName].push(r); } - evaluateParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] { - const result: EvaluatedParametersResult[] = []; - - for (const variant of this.variants) { - variant.pushParameterRowEvaluation(result, sourceTable, row); - } - - return result; - } - - evaluateRow(options: EvaluateRowOptions): EvaluationResult[] { + evaluateRow(options: EvaluateRowOptions): UnscopedEvaluationResult[] { if (!this.data.applies(options.sourceTable)) { return []; } - const stream = this; const row: TableRow = { sourceTable: options.sourceTable, record: options.record }; + // There is some duplication in work here when there are multiple variants on a stream: + // Each variant does the same row transformation (only the filters / bucket ids differ). + // However, architecturally we do need to be able to evaluate each variant separately. return this.data.evaluateRowWithOptions({ table: options.sourceTable, row: options.record, - bucketIds() { - const bucketIds: string[] = []; - for (const variant of stream.variants) { - bucketIds.push(...variant.bucketIdsForRow(stream.name, row, options.bucketIdTransformer)); - } - - return bucketIds; + serializedBucketParameters: () => { + return this.variant.bucketParametersForRow(row); } }); } diff --git a/packages/sync-rules/src/streams/variant.ts b/packages/sync-rules/src/streams/variant.ts index 1e9c7097e..faa448f6f 100644 --- a/packages/sync-rules/src/streams/variant.ts +++ b/packages/sync-rules/src/streams/variant.ts @@ -1,18 +1,12 @@ import { BucketInclusionReason, ResolvedBucket } from '../BucketDescription.js'; -import { BucketParameterQuerier, ParameterLookup } from '../BucketParameterQuerier.js'; -import { SourceTableInterface } from '../SourceTableInterface.js'; -import { - BucketIdTransformer, - EvaluatedParametersResult, - EvaluateRowOptions, - RequestParameters, - SqliteJsonValue, - SqliteRow, - TableRow -} from '../types.js'; -import { isJsonValue, JSONBucketNameSerialize, normalizeParameterValue } from '../utils.js'; +import { BucketParameterQuerier, PendingQueriers } from '../BucketParameterQuerier.js'; +import { BucketDataSource, BucketParameterQuerierSource, ParameterIndexLookupCreator } from '../BucketSource.js'; +import { BucketDataScope } from '../HydrationState.js'; +import { CreateSourceParams, GetQuerierOptions, RequestedStream, ScopedParameterLookup } from '../index.js'; +import { RequestParameters, SqliteJsonValue, TableRow } from '../types.js'; +import { buildBucketName, isJsonValue, JSONBucketNameSerialize } from '../utils.js'; import { BucketParameter, SubqueryEvaluator } from './parameter.js'; -import { SyncStream } from './stream.js'; +import { SyncStream, SyncStreamDataSource } from './stream.js'; import { cartesianProduct } from './utils.js'; /** @@ -65,11 +59,19 @@ export class StreamVariant { this.requestFilters = []; } + defaultBucketPrefix(streamName: string): string { + return `${streamName}|${this.id}`; + } + + indexLookupCreators(): ParameterIndexLookupCreator[] { + return this.subqueries.flatMap((subquery) => subquery.indexLookupCreators()); + } + /** * Given a row in the table this stream selects from, returns all ids of buckets to which that row belongs to. */ - bucketIdsForRow(streamName: string, options: TableRow, transformer: BucketIdTransformer): string[] { - return this.instantiationsForRow(options).map((values) => this.buildBucketId(streamName, values, transformer)); + bucketParametersForRow(options: TableRow): string[] { + return this.instantiationsForRow(options).map((values) => this.serializeBucketParameters(values)); } /** @@ -114,15 +116,12 @@ export class StreamVariant { return [...cartesianProduct(...instantiations)]; } - get hasDynamicBucketQueries(): boolean { - return this.requestFilters.some((f) => f.type == 'dynamic'); - } - querier( stream: SyncStream, reason: BucketInclusionReason, params: RequestParameters, - bucketIdTransformer: BucketIdTransformer + bucketScope: BucketDataScope, + hydratedSubqueries: HydratedSubqueries ): BucketParameterQuerier | null { const instantiation = this.partiallyEvaluateParameters(params); if (instantiation == null) { @@ -136,7 +135,7 @@ export class StreamVariant { const dynamicRequestFilters: SubqueryRequestFilter[] = this.requestFilters.filter((f) => f.type == 'dynamic'); const dynamicParameters: ResolvedDynamicParameter[] = []; - const subqueryToLookups = new Map(); + const subqueryToLookups = new Map(); for (let i = 0; i < this.parameters.length; i++) { const parameter = this.parameters[i]; @@ -151,7 +150,11 @@ export class StreamVariant { } for (const subquery of this.subqueries) { - subqueryToLookups.set(subquery, subquery.lookupsForRequest(params)); + const subqueryLookup = hydratedSubqueries.get(subquery); + if (subqueryLookup == null) { + throw new Error('Internal error, missing subquery lookup'); + } + subqueryToLookups.set(subquery, subqueryLookup(params)); } const staticBuckets: ResolvedBucket[] = []; @@ -159,7 +162,7 @@ export class StreamVariant { // When we have no dynamic parameters, the partial evaluation is a full instantiation. const instantiations = this.cartesianProductOfParameterInstantiations(instantiation as SqliteJsonValue[][]); for (const instantiation of instantiations) { - staticBuckets.push(this.resolveBucket(stream, instantiation, reason, bucketIdTransformer)); + staticBuckets.push(this.resolveBucket(stream, instantiation, reason, bucketScope)); } } @@ -204,7 +207,7 @@ export class StreamVariant { perParameterInstantiation as SqliteJsonValue[][] ); - return Promise.resolve(product.map((e) => variant.resolveBucket(stream, e, reason, bucketIdTransformer))); + return Promise.resolve(product.map((e) => variant.resolveBucket(stream, e, reason, bucketScope))); } }; } @@ -220,41 +223,6 @@ export class StreamVariant { ); } - /** - * Creates lookup indices for dynamically-resolved parameters. - * - * Resolving dynamic parameters is a two-step process: First, for tables referenced in subqueries, we create an index - * to resolve which request parameters would match rows in subqueries. Then, when resolving bucket ids for a request, - * we compute subquery results by looking up results in that index. - * - * This implements the first step of that process. - * - * @param result The array into which evaluation results should be written to. - * @param sourceTable A table we depend on in a subquery. - * @param row Row data to index. - */ - pushParameterRowEvaluation(result: EvaluatedParametersResult[], sourceTable: SourceTableInterface, row: SqliteRow) { - for (const subquery of this.subqueries) { - if (subquery.parameterTable.matches(sourceTable)) { - const lookups = subquery.lookupsForParameterRow(sourceTable, row); - if (lookups == null) { - continue; - } - - // The row of the subquery. Since we only support subqueries with a single column, we unconditionally name the - // column `result` for simplicity. - const resultRow = { result: lookups.value }; - - result.push( - ...lookups.lookups.map((l) => ({ - lookup: l, - bucketParameters: [resultRow] - })) - ); - } - } - } - debugRepresentation(): any { return { id: this.id, @@ -305,32 +273,105 @@ export class StreamVariant { /** * Builds a bucket id for an instantiation, like `stream|0[1,2,"foo"]`. * - * @param streamName The name of the stream, included in the bucket id + * @param bucketPrefix The name of the the bucket, excluding parameters * @param instantiation An instantiation for all parameters in this variant. * @param transformer A transformer adding version information to the inner id. * @returns The generated bucket id */ - private buildBucketId(streamName: string, instantiation: SqliteJsonValue[], transformer: BucketIdTransformer) { + private serializeBucketParameters(instantiation: SqliteJsonValue[]) { if (instantiation.length != this.parameters.length) { throw Error('Internal error, instantiation length mismatch'); } - return transformer(`${streamName}|${this.id}${JSONBucketNameSerialize.stringify(instantiation)}`); + return JSONBucketNameSerialize.stringify(instantiation); } private resolveBucket( stream: SyncStream, instantiation: SqliteJsonValue[], reason: BucketInclusionReason, - bucketIdTransformer: BucketIdTransformer + bucketScope: BucketDataScope ): ResolvedBucket { return { definition: stream.name, inclusion_reasons: [reason], - bucket: this.buildBucketId(stream.name, instantiation, bucketIdTransformer), + bucket: buildBucketName(bucketScope, this.serializeBucketParameters(instantiation)), priority: stream.priority }; } + + createParameterQuerierSource( + params: CreateSourceParams, + stream: SyncStream, + querierDataSource: BucketDataSource + ): BucketParameterQuerierSource { + const hydrationState = params.hydrationState; + const bucketScope = hydrationState.getBucketSourceScope(querierDataSource); + + const hydratedSubqueries: HydratedSubqueries = new Map( + this.subqueries.map((s) => [s, s.hydrateLookupsForRequest(hydrationState)]) + ); + + return { + pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions): void => { + const subscriptions = options.streams[stream.name] ?? []; + + if (!stream.subscribedToByDefault && !subscriptions.length) { + // The client is not subscribing to this stream, so don't query buckets related to it. + return; + } + + let hasExplicitDefaultSubscription = false; + for (const subscription of subscriptions) { + let subscriptionParams = options.globalParameters; + if (subscription.parameters != null) { + subscriptionParams = subscriptionParams.withAddedStreamParameters(subscription.parameters); + } else { + hasExplicitDefaultSubscription = true; + } + + this.queriersForSubscription( + stream, + result, + subscription, + subscriptionParams, + bucketScope, + hydratedSubqueries + ); + } + + // If the stream is subscribed to by default and there is no explicit subscription that would match the default + // subscription, also include the default querier. + if (stream.subscribedToByDefault && !hasExplicitDefaultSubscription) { + this.queriersForSubscription(stream, result, null, options.globalParameters, bucketScope, hydratedSubqueries); + } + } + }; + } + + private queriersForSubscription( + stream: SyncStream, + result: PendingQueriers, + subscription: RequestedStream | null, + params: RequestParameters, + bucketScope: BucketDataScope, + hydratedSubqueries: HydratedSubqueries + ) { + const reason: BucketInclusionReason = subscription != null ? { subscription: subscription.opaque_id } : 'default'; + + try { + const querier = this.querier(stream, reason, params, bucketScope, hydratedSubqueries); + if (querier) { + result.queriers.push(querier); + } + } catch (e) { + result.errors.push({ + descriptor: stream.name, + message: `Error evaluating bucket ids: ${e.message}`, + subscription: subscription ?? undefined + }); + } + } } /** * A stateless filter condition that only depends on the request itself, e.g. `WHERE token_parameters.is_admin`. @@ -356,3 +397,5 @@ export interface SubqueryRequestFilter { matches(params: RequestParameters, results: SqliteJsonValue[]): boolean; } export type RequestFilter = StaticRequestFilter | SubqueryRequestFilter; + +type HydratedSubqueries = Map ScopedParameterLookup[]>; diff --git a/packages/sync-rules/src/types.ts b/packages/sync-rules/src/types.ts index 8a607428c..44bc7ee30 100644 --- a/packages/sync-rules/src/types.ts +++ b/packages/sync-rules/src/types.ts @@ -1,20 +1,14 @@ import { JSONBig, JsonContainer } from '@powersync/service-jsonbig'; +import { BucketPriority } from './BucketDescription.js'; +import { ScopedParameterLookup, UnscopedParameterLookup } from './BucketParameterQuerier.js'; +import { CompatibilityContext } from './compatibility.js'; import { ColumnDefinition } from './ExpressionType.js'; +import { RequestFunctionCall } from './request_functions.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { SyncRulesOptions } from './SqlSyncRules.js'; import { TablePattern } from './TablePattern.js'; -import { toSyncRulesParameters } from './utils.js'; -import { BucketPriority } from './BucketDescription.js'; -import { ParameterLookup } from './BucketParameterQuerier.js'; import { CustomSqliteValue } from './types/custom_sqlite_value.js'; -import { CompatibilityContext } from './compatibility.js'; -import { RequestFunctionCall } from './request_functions.js'; - -export interface SyncRules { - evaluateRow(options: EvaluateRowOptions): EvaluationResult[]; - - evaluateParameterRow(table: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[]; -} +import { toSyncRulesParameters } from './utils.js'; export interface QueryParseOptions extends SyncRulesOptions { accept_potentially_dangerous_queries?: boolean; @@ -27,7 +21,18 @@ export interface StreamParseOptions extends QueryParseOptions { } export interface EvaluatedParameters { - lookup: ParameterLookup; + lookup: ScopedParameterLookup; + + /** + * Parameters used to generate bucket id. May be incomplete. + * + * JSON-serializable. + */ + bucketParameters: Record[]; +} + +export interface UnscopedEvaluatedParameters { + lookup: UnscopedParameterLookup; /** * Parameters used to generate bucket id. May be incomplete. @@ -38,6 +43,7 @@ export interface EvaluatedParameters { } export type EvaluatedParametersResult = EvaluatedParameters | EvaluationError; +export type UnscopedEvaluatedParametersResult = UnscopedEvaluatedParameters | EvaluationError; export interface EvaluatedRow { bucket: string; @@ -54,23 +60,60 @@ export interface EvaluatedRow { data: SqliteJsonRow; } +/** + * Bucket data as evaluated by the BucketDataSource. + * + * The bucket name must still be resolved, external to this. + */ +export interface UnscopedEvaluatedRow { + /** + * Serialized evaluated parameters used to generate the bucket id. Serialized as a JSON array. + * + * Examples: + * [] // no bucket parameters + * [1] // single numeric parameter + * [1,"foo"] // multiple parameters + * + * The bucket name is derived by using concetenating these parameters with the generated bucket name. + */ + serializedBucketParameters: string; + + /** Output table - may be different from input table. */ + table: string; + + /** + * Convenience attribute. Must match data.id. + */ + id: string; + + /** Must be JSON-serializable. */ + data: SqliteJsonRow; +} + export interface EvaluationError { error: string; } -export function isEvaluationError(e: any): e is EvaluationError { - return typeof e.error == 'string'; +export function isEvaluationError( + e: EvaluationResult | UnscopedEvaluationResult | EvaluatedParametersResult | UnscopedEvaluatedParametersResult +): e is EvaluationError { + return typeof (e as EvaluationError).error == 'string'; } export function isEvaluatedRow(e: EvaluationResult): e is EvaluatedRow { return typeof (e as EvaluatedRow).bucket == 'string'; } +export function isSourceEvaluatedRow(e: UnscopedEvaluationResult): e is UnscopedEvaluatedRow { + return typeof (e as UnscopedEvaluatedRow).serializedBucketParameters == 'string'; +} + export function isEvaluatedParameters(e: EvaluatedParametersResult): e is EvaluatedParameters { return 'lookup' in e; } export type EvaluationResult = EvaluatedRow | EvaluationError; +export type UnscopedEvaluationResult = UnscopedEvaluatedRow | EvaluationError; export interface RequestJwtPayload { /** @@ -291,23 +334,7 @@ export interface InputParameter { parametersToLookupValue(parameters: ParameterValueSet): SqliteValue; } -/** - * Transforms bucket ids generated when evaluating the row by e.g. encoding version information. - * - * Because buckets are recreated on a sync rule redeploy, it makes sense to use different bucket ids (otherwise, clients - * may run into checksum errors causing a sync to take longer than necessary or breaking progress). - * - * So, this transformer receives the original bucket id as generated by defined sync rules, and can prepend a version - * identifier. - * - * Note that this transformation has not been present in older versions of the sync service. To preserve backwards - * compatibility, sync rules will not use this function without an opt-in. - */ -export type BucketIdTransformer = (regularId: string) => string; - -export interface EvaluateRowOptions extends TableRow { - bucketIdTransformer: BucketIdTransformer; -} +export interface EvaluateRowOptions extends TableRow {} /** * A row associated with the table it's coming from. diff --git a/packages/sync-rules/src/types/custom_sqlite_value.ts b/packages/sync-rules/src/types/custom_sqlite_value.ts index d26af5390..53ef1cc44 100644 --- a/packages/sync-rules/src/types/custom_sqlite_value.ts +++ b/packages/sync-rules/src/types/custom_sqlite_value.ts @@ -1,7 +1,7 @@ import { JSONBig } from '@powersync/service-jsonbig'; import { CompatibilityContext } from '../compatibility.js'; -import { SqliteValue, EvaluatedRow, SqliteInputValue, DatabaseInputValue } from '../types.js'; import { SqliteValueType } from '../ExpressionType.js'; +import { EvaluatedRow, SqliteValue } from '../types.js'; /** * A value that decays into a {@link SqliteValue} in a context-specific way. diff --git a/packages/sync-rules/src/utils.ts b/packages/sync-rules/src/utils.ts index 04f849e08..cd5ed1566 100644 --- a/packages/sync-rules/src/utils.ts +++ b/packages/sync-rules/src/utils.ts @@ -2,9 +2,9 @@ import { JSONBig, JsonContainer, Replacer, stringifyRaw } from '@powersync/servi import { SelectFromStatement, Statement } from 'pgsql-ast-parser'; import { CompatibilityContext } from './compatibility.js'; import { SyncRuleProcessingError as SyncRulesProcessingError } from './errors.js'; +import { BucketDataScope } from './HydrationState.js'; import { SQLITE_FALSE, SQLITE_TRUE } from './sql_support.js'; import { - BucketIdTransformer, DatabaseInputRow, DatabaseInputValue, SqliteInputRow, @@ -20,15 +20,14 @@ export function isSelectStatement(q: Statement): q is SelectFromStatement { return q.type == 'select'; } -export function getBucketId( - descriptor_id: string, - bucket_parameters: string[], - params: Record, - transformer: BucketIdTransformer -): string { +export function buildBucketName(scope: BucketDataScope, serializedParameters: string): string { + return scope.bucketPrefix + serializedParameters; +} + +export function serializeBucketParameters(bucketParameters: string[], params: Record): string { // Important: REAL and INTEGER values matching the same number needs the same representation in the bucket name. - const paramArray = bucket_parameters.map((name) => params[`bucket.${name}`]); - return transformer(`${descriptor_id}${JSONBucketNameSerialize.stringify(paramArray)}`); + const paramArray = bucketParameters.map((name) => params[`bucket.${name}`]); + return JSONBucketNameSerialize.stringify(paramArray); } const DEPTH_LIMIT = 10; diff --git a/packages/sync-rules/test/src/compatibility.test.ts b/packages/sync-rules/test/src/compatibility.test.ts index 67a248b69..11068de01 100644 --- a/packages/sync-rules/test/src/compatibility.test.ts +++ b/packages/sync-rules/test/src/compatibility.test.ts @@ -1,7 +1,8 @@ import { describe, expect, test } from 'vitest'; -import { SqlSyncRules, DateTimeValue, toSyncRulesValue, TimeValuePrecision } from '../../src/index.js'; +import { DateTimeValue, SqlSyncRules, TimeValuePrecision, toSyncRulesValue } from '../../src/index.js'; -import { ASSETS, identityBucketTransformer, normalizeQuerierOptions, PARSE_OPTIONS } from './util.js'; +import { versionedHydrationState } from '../../src/HydrationState.js'; +import { ASSETS, normalizeQuerierOptions, PARSE_OPTIONS } from './util.js'; describe('compatibility options', () => { describe('timestamps', () => { @@ -19,12 +20,11 @@ bucket_definitions: - SELECT id, description FROM assets `, PARSE_OPTIONS - ); + ).hydrate(); expect( rules.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer(''), record: rules.applyRowContext({ id: 'id', description: value @@ -47,12 +47,11 @@ config: timestamps_iso8601: true `, PARSE_OPTIONS - ); + ).hydrate(); expect( rules.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer(''), record: rules.applyRowContext({ id: 'id', description: value @@ -75,11 +74,10 @@ config: edition: 2 `, PARSE_OPTIONS - ); + ).hydrate({ hydrationState: versionedHydrationState(1) }); expect( rules.evaluateRow({ - bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1'), sourceTable: ASSETS, record: rules.applyRowContext({ id: 'id', @@ -90,11 +88,7 @@ config: { bucket: '1#stream|0[]', data: { description: '2025-08-19T09:21:00Z', id: 'id' }, id: 'id', table: 'assets' } ]); - expect( - rules.getBucketParameterQuerier( - normalizeQuerierOptions({}, {}, {}, SqlSyncRules.versionedBucketIdTransformer('1')) - ).querier.staticBuckets - ).toStrictEqual([ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({}, {}, {})).querier.staticBuckets).toStrictEqual([ { bucket: '1#stream|0[]', definition: 'stream', @@ -118,11 +112,10 @@ config: versioned_bucket_ids: false `, PARSE_OPTIONS - ); + ).hydrate({ hydrationState: versionedHydrationState(1) }); expect( rules.evaluateRow({ - bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1'), sourceTable: ASSETS, record: rules.applyRowContext({ id: 'id', @@ -132,11 +125,7 @@ config: ).toStrictEqual([ { bucket: 'stream|0[]', data: { description: '2025-08-19 09:21:00Z', id: 'id' }, id: 'id', table: 'assets' } ]); - expect( - rules.getBucketParameterQuerier( - normalizeQuerierOptions({}, {}, {}, SqlSyncRules.versionedBucketIdTransformer('1')) - ).querier.staticBuckets - ).toStrictEqual([ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({}, {}, {})).querier.staticBuckets).toStrictEqual([ { bucket: 'stream|0[]', definition: 'stream', @@ -160,12 +149,11 @@ config: versioned_bucket_ids: true `, PARSE_OPTIONS - ); + ).hydrate({ hydrationState: versionedHydrationState(1) }); expect( rules.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1'), record: { id: 'id', description: 'desc' @@ -185,12 +173,11 @@ config: edition: 2 `, PARSE_OPTIONS - ); + ).hydrate({ hydrationState: versionedHydrationState(1) }); expect( rules.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1'), record: rules.applyRowContext({ id: 'id', description: new DateTimeValue('2025-08-19T09:21:00Z', undefined, { @@ -216,7 +203,7 @@ bucket_definitions: - SELECT id, description ->> 'foo.bar' AS "desc" FROM assets `, PARSE_OPTIONS - ); + ).hydrate(); expect( rules.evaluateRow({ @@ -224,8 +211,7 @@ bucket_definitions: record: { id: 'id', description: description - }, - bucketIdTransformer: identityBucketTransformer + } }) ).toStrictEqual([{ bucket: 'a[]', data: { desc: 'baz', id: 'id' }, id: 'id', table: 'assets' }]); }); @@ -241,7 +227,7 @@ config: fixed_json_extract: true `, PARSE_OPTIONS - ); + ).hydrate(); expect( rules.evaluateRow({ @@ -249,8 +235,7 @@ config: record: { id: 'id', description: description - }, - bucketIdTransformer: identityBucketTransformer + } }) ).toStrictEqual([{ bucket: 'a[]', data: { desc: null, id: 'id' }, id: 'id', table: 'assets' }]); }); @@ -297,11 +282,12 @@ config: `; } - const rules = SqlSyncRules.fromYaml(syncRules, PARSE_OPTIONS); + const rules = SqlSyncRules.fromYaml(syncRules, PARSE_OPTIONS).hydrate({ + hydrationState: versionedHydrationState(1) + }); expect( rules.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1'), record: rules.applyRowContext({ id: 'id', description: data @@ -321,11 +307,7 @@ config: } ]); - expect( - rules.getBucketParameterQuerier( - normalizeQuerierOptions({}, {}, {}, SqlSyncRules.versionedBucketIdTransformer('1')) - ).querier.staticBuckets - ).toStrictEqual([ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({}, {}, {})).querier.staticBuckets).toStrictEqual([ { bucket: withFixedQuirk ? '1#mybucket[]' : 'mybucket[]', definition: 'mybucket', diff --git a/packages/sync-rules/test/src/data_queries.test.ts b/packages/sync-rules/test/src/data_queries.test.ts index 34fba0db3..09e615095 100644 --- a/packages/sync-rules/test/src/data_queries.test.ts +++ b/packages/sync-rules/test/src/data_queries.test.ts @@ -1,121 +1,86 @@ import { describe, expect, test } from 'vitest'; -import { CompatibilityContext, ExpressionType, SqlDataQuery, SqlSyncRules } from '../../src/index.js'; -import { ASSETS, BASIC_SCHEMA, identityBucketTransformer, PARSE_OPTIONS } from './util.js'; +import { CompatibilityContext, ExpressionType, SqlDataQuery } from '../../src/index.js'; +import { ASSETS, BASIC_SCHEMA, PARSE_OPTIONS } from './util.js'; describe('data queries', () => { - test('uses bucket id transformer', function () { - const sql = 'SELECT * FROM assets WHERE assets.org_id = bucket.org_id'; - const query = SqlDataQuery.fromSql('mybucket', ['org_id'], sql, PARSE_OPTIONS, compatibility); - expect(query.errors).toEqual([]); - - expect( - query.evaluateRow(ASSETS, { id: 'asset1', org_id: 'org1' }, SqlSyncRules.versionedBucketIdTransformer('1')) - ).toEqual([ - { - bucket: '1#mybucket["org1"]', - table: 'assets', - id: 'asset1', - data: { id: 'asset1', org_id: 'org1' } - } - ]); - }); - test('bucket parameters = query', function () { const sql = 'SELECT * FROM assets WHERE assets.org_id = bucket.org_id'; - const query = SqlDataQuery.fromSql('mybucket', ['org_id'], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql(['org_id'], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toEqual([]); - expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: 'org1' }, identityBucketTransformer)).toEqual([ + expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: 'org1' })).toEqual([ { - bucket: 'mybucket["org1"]', + serializedBucketParameters: '["org1"]', table: 'assets', id: 'asset1', data: { id: 'asset1', org_id: 'org1' } } ]); - expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: null }, identityBucketTransformer)).toEqual([]); + expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: null })).toEqual([]); }); test('bucket parameters IN query', function () { const sql = 'SELECT * FROM assets WHERE bucket.category IN assets.categories'; - const query = SqlDataQuery.fromSql('mybucket', ['category'], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql(['category'], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toEqual([]); - expect( - query.evaluateRow( - ASSETS, - { id: 'asset1', categories: JSON.stringify(['red', 'green']) }, - identityBucketTransformer - ) - ).toMatchObject([ + expect(query.evaluateRow(ASSETS, { id: 'asset1', categories: JSON.stringify(['red', 'green']) })).toMatchObject([ { - bucket: 'mybucket["red"]', + serializedBucketParameters: '["red"]', table: 'assets', id: 'asset1' }, { - bucket: 'mybucket["green"]', + serializedBucketParameters: '["green"]', table: 'assets', id: 'asset1' } ]); - expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: null }, identityBucketTransformer)).toEqual([]); + expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: null })).toEqual([]); }); test('static IN data query', function () { const sql = `SELECT * FROM assets WHERE 'green' IN assets.categories`; - const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toEqual([]); - expect( - query.evaluateRow( - ASSETS, - { id: 'asset1', categories: JSON.stringify(['red', 'green']) }, - identityBucketTransformer - ) - ).toMatchObject([ + expect(query.evaluateRow(ASSETS, { id: 'asset1', categories: JSON.stringify(['red', 'green']) })).toMatchObject([ { - bucket: 'mybucket[]', + serializedBucketParameters: '[]', table: 'assets', id: 'asset1' } ]); - expect( - query.evaluateRow( - ASSETS, - { id: 'asset1', categories: JSON.stringify(['red', 'blue']) }, - identityBucketTransformer - ) - ).toEqual([]); + expect(query.evaluateRow(ASSETS, { id: 'asset1', categories: JSON.stringify(['red', 'blue']) })).toEqual([]); }); test('data IN static query', function () { const sql = `SELECT * FROM assets WHERE assets.condition IN '["good","great"]'`; - const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toEqual([]); - expect(query.evaluateRow(ASSETS, { id: 'asset1', condition: 'good' }, identityBucketTransformer)).toMatchObject([ + expect(query.evaluateRow(ASSETS, { id: 'asset1', condition: 'good' })).toMatchObject([ { - bucket: 'mybucket[]', + serializedBucketParameters: '[]', table: 'assets', id: 'asset1' } ]); - expect(query.evaluateRow(ASSETS, { id: 'asset1', condition: 'bad' }, identityBucketTransformer)).toEqual([]); + expect(query.evaluateRow(ASSETS, { id: 'asset1', condition: 'bad' })).toEqual([]); }); test('table alias', function () { const sql = 'SELECT * FROM assets as others WHERE others.org_id = bucket.org_id'; - const query = SqlDataQuery.fromSql('mybucket', ['org_id'], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql(['org_id'], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toEqual([]); - expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: 'org1' }, identityBucketTransformer)).toEqual([ + expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: 'org1' })).toEqual([ { - bucket: 'mybucket["org1"]', + serializedBucketParameters: '["org1"]', table: 'others', id: 'asset1', data: { id: 'asset1', org_id: 'org1' } @@ -127,7 +92,6 @@ describe('data queries', () => { const schema = BASIC_SCHEMA; const q1 = SqlDataQuery.fromSql( - 'q1', ['user_id'], `SELECT * FROM assets WHERE owner_id = bucket.user_id`, PARSE_OPTIONS, @@ -146,7 +110,6 @@ describe('data queries', () => { ]); const q2 = SqlDataQuery.fromSql( - 'q1', ['user_id'], ` SELECT id :: integer as id, @@ -181,7 +144,6 @@ describe('data queries', () => { test('validate columns', () => { const schema = BASIC_SCHEMA; const q1 = SqlDataQuery.fromSql( - 'q1', ['user_id'], 'SELECT id, name, count FROM assets WHERE owner_id = bucket.user_id', { ...PARSE_OPTIONS, schema }, @@ -190,7 +152,6 @@ describe('data queries', () => { expect(q1.errors).toEqual([]); const q2 = SqlDataQuery.fromSql( - 'q2', ['user_id'], 'SELECT id, upper(description) as d FROM assets WHERE other_id = bucket.user_id', { ...PARSE_OPTIONS, schema }, @@ -208,7 +169,6 @@ describe('data queries', () => { ]); const q3 = SqlDataQuery.fromSql( - 'q3', ['user_id'], 'SELECT id, description, * FROM nope WHERE other_id = bucket.user_id', { ...PARSE_OPTIONS, schema }, @@ -221,7 +181,7 @@ describe('data queries', () => { } ]); - const q4 = SqlDataQuery.fromSql('q4', [], 'SELECT * FROM other', { ...PARSE_OPTIONS, schema }, compatibility); + const q4 = SqlDataQuery.fromSql([], 'SELECT * FROM other', { ...PARSE_OPTIONS, schema }, compatibility); expect(q4.errors).toMatchObject([ { message: `Query must return an "id" column`, @@ -230,7 +190,6 @@ describe('data queries', () => { ]); const q5 = SqlDataQuery.fromSql( - 'q5', [], 'SELECT other_id as id, * FROM other', { ...PARSE_OPTIONS, schema }, @@ -241,7 +200,7 @@ describe('data queries', () => { test('invalid query - invalid IN', function () { const sql = 'SELECT * FROM assets WHERE assets.category IN bucket.categories'; - const query = SqlDataQuery.fromSql('mybucket', ['categories'], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql(['categories'], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([ { type: 'fatal', message: 'Cannot use bucket parameters on the right side of IN operators' } ]); @@ -249,7 +208,7 @@ describe('data queries', () => { test('invalid query - not all parameters used', function () { const sql = 'SELECT * FROM assets WHERE 1'; - const query = SqlDataQuery.fromSql('mybucket', ['org_id'], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql(['org_id'], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([ { type: 'fatal', message: 'Query must cover all bucket parameters. Expected: ["bucket.org_id"] Got: []' } ]); @@ -257,7 +216,7 @@ describe('data queries', () => { test('invalid query - parameter not defined', function () { const sql = 'SELECT * FROM assets WHERE assets.org_id = bucket.org_id'; - const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([ { type: 'fatal', message: 'Query must cover all bucket parameters. Expected: [] Got: ["bucket.org_id"]' } ]); @@ -265,25 +224,25 @@ describe('data queries', () => { test('invalid query - function on parameter (1)', function () { const sql = 'SELECT * FROM assets WHERE assets.org_id = upper(bucket.org_id)'; - const query = SqlDataQuery.fromSql('mybucket', ['org_id'], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql(['org_id'], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([{ type: 'fatal', message: 'Cannot use bucket parameters in expressions' }]); }); test('invalid query - function on parameter (2)', function () { const sql = 'SELECT * FROM assets WHERE assets.org_id = upper(bucket.org_id)'; - const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([{ type: 'fatal', message: 'Cannot use bucket parameters in expressions' }]); }); test('invalid query - match clause in select', () => { const sql = 'SELECT id, (bucket.org_id = assets.org_id) as org_matches FROM assets where org_id = bucket.org_id'; - const query = SqlDataQuery.fromSql('mybucket', ['org_id'], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql(['org_id'], sql, PARSE_OPTIONS, compatibility); expect(query.errors[0].message).toMatch(/Parameter match expression is not allowed here/); }); test('case-sensitive queries (1)', () => { const sql = 'SELECT * FROM Assets'; - const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "Assets" instead.` } ]); @@ -291,7 +250,7 @@ describe('data queries', () => { test('case-sensitive queries (2)', () => { const sql = 'SELECT *, Name FROM assets'; - const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "Name" instead.` } ]); @@ -299,7 +258,7 @@ describe('data queries', () => { test('case-sensitive queries (3)', () => { const sql = 'SELECT * FROM assets WHERE Archived = False'; - const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "Archived" instead.` } ]); @@ -308,7 +267,7 @@ describe('data queries', () => { test.skip('case-sensitive queries (4)', () => { // Cannot validate table alias yet const sql = 'SELECT * FROM assets as myAssets'; - const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "myAssets" instead.` } ]); @@ -317,7 +276,7 @@ describe('data queries', () => { test.skip('case-sensitive queries (5)', () => { // Cannot validate table alias yet const sql = 'SELECT * FROM assets myAssets'; - const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "myAssets" instead.` } ]); @@ -326,7 +285,7 @@ describe('data queries', () => { test.skip('case-sensitive queries (6)', () => { // Cannot validate anything with a schema yet const sql = 'SELECT * FROM public.ASSETS'; - const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "ASSETS" instead.` } ]); @@ -335,7 +294,7 @@ describe('data queries', () => { test.skip('case-sensitive queries (7)', () => { // Cannot validate schema yet const sql = 'SELECT * FROM PUBLIC.assets'; - const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "PUBLIC" instead.` } ]); diff --git a/packages/sync-rules/test/src/parameter_queries.test.ts b/packages/sync-rules/test/src/parameter_queries.test.ts index acabec993..81c5f690a 100644 --- a/packages/sync-rules/test/src/parameter_queries.test.ts +++ b/packages/sync-rules/test/src/parameter_queries.test.ts @@ -1,16 +1,48 @@ -import { describe, expect, test } from 'vitest'; -import { ParameterLookup, SqlParameterQuery } from '../../src/index.js'; -import { BASIC_SCHEMA, identityBucketTransformer, normalizeTokenParameters, PARSE_OPTIONS } from './util.js'; +import { beforeEach, describe, expect, test } from 'vitest'; +import { + UnscopedParameterLookup, + SqlParameterQuery, + SourceTableInterface, + debugHydratedMergedSource, + BucketParameterQuerier, + QuerierError, + GetQuerierOptions, + RequestParameters, + ScopedParameterLookup, + mergeParameterIndexLookupCreators +} from '../../src/index.js'; import { StaticSqlParameterQuery } from '../../src/StaticSqlParameterQuery.js'; +import { BASIC_SCHEMA, EMPTY_DATA_SOURCE, normalizeTokenParameters, PARSE_OPTIONS } from './util.js'; +import { HydrationState } from '../../src/HydrationState.js'; describe('parameter queries', () => { + const table = (name: string): SourceTableInterface => ({ + connectionTag: 'default', + name, + schema: PARSE_OPTIONS.defaultSchema + }); + + const TABLE_USERS = table('users'); + const TABLE_REGIONS = table('regions'); + const TABLE_WORKSPACES = table('workspaces'); + const TABLE_POSTS = table('posts'); + const TABLE_GROUPS = table('groups'); + test('token_parameters IN query', function () { const sql = 'SELECT id as group_id FROM groups WHERE token_parameters.user_id IN groups.user_ids'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'group1', user_ids: JSON.stringify(['user1', 'user2']) })).toEqual([ + expect( + query.evaluateParameterRow(TABLE_GROUPS, { id: 'group1', user_ids: JSON.stringify(['user1', 'user2']) }) + ).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['user1']), + lookup: UnscopedParameterLookup.normalized(['user1']), bucketParameters: [ { group_id: 'group1' @@ -18,7 +50,7 @@ describe('parameter queries', () => { ] }, { - lookup: ParameterLookup.normalized('mybucket', '1', ['user2']), + lookup: UnscopedParameterLookup.normalized(['user2']), bucketParameters: [ { group_id: 'group1' @@ -32,16 +64,22 @@ describe('parameter queries', () => { user_id: 'user1' }) ) - ).toEqual([ParameterLookup.normalized('mybucket', '1', ['user1'])]); + ).toEqual([UnscopedParameterLookup.normalized(['user1'])]); }); test('IN token_parameters query', function () { const sql = 'SELECT id as region_id FROM regions WHERE name IN token_parameters.region_names'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'region1', name: 'colorado' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_REGIONS, { id: 'region1', name: 'colorado' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['colorado']), + lookup: UnscopedParameterLookup.normalized(['colorado']), bucketParameters: [ { region_id: 'region1' @@ -55,21 +93,24 @@ describe('parameter queries', () => { region_names: JSON.stringify(['colorado', 'texas']) }) ) - ).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['colorado']), - ParameterLookup.normalized('mybucket', '1', ['texas']) - ]); + ).toEqual([UnscopedParameterLookup.normalized(['colorado']), UnscopedParameterLookup.normalized(['texas'])]); }); test('queried numeric parameters', () => { const sql = 'SELECT users.int1, users.float1, users.float2 FROM users WHERE users.int1 = token_parameters.int1 AND users.float1 = token_parameters.float1 AND users.float2 = token_parameters.float2'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); // Note: We don't need to worry about numeric vs decimal types in the lookup - JSONB handles normalization for us. - expect(query.evaluateParameterRow({ int1: 314n, float1: 3.14, float2: 314 })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { int1: 314n, float1: 3.14, float2: 314 })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', [314n, 3.14, 314]), + lookup: UnscopedParameterLookup.normalized([314n, 3.14, 314]), bucketParameters: [{ int1: 314n, float1: 3.14, float2: 314 }] } @@ -78,251 +119,325 @@ describe('parameter queries', () => { // Similarly, we don't need to worry about the types here. // This test just checks the current behavior. expect(query.getLookups(normalizeTokenParameters({ int1: 314n, float1: 3.14, float2: 314 }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [314n, 3.14, 314n]) + UnscopedParameterLookup.normalized([314n, 3.14, 314n]) ]); // We _do_ need to care about the bucket string representation. expect( - query.resolveBucketDescriptions( - [{ int1: 314, float1: 3.14, float2: 314 }], - normalizeTokenParameters({}), - identityBucketTransformer - ) + query.resolveBucketDescriptions([{ int1: 314, float1: 3.14, float2: 314 }], normalizeTokenParameters({}), { + bucketPrefix: 'mybucket' + }) ).toEqual([{ bucket: 'mybucket[314,3.14,314]', priority: 3 }]); expect( - query.resolveBucketDescriptions( - [{ int1: 314n, float1: 3.14, float2: 314 }], - normalizeTokenParameters({}), - identityBucketTransformer - ) + query.resolveBucketDescriptions([{ int1: 314n, float1: 3.14, float2: 314 }], normalizeTokenParameters({}), { + bucketPrefix: 'mybucket' + }) ).toEqual([{ bucket: 'mybucket[314,3.14,314]', priority: 3 }]); }); test('plain token_parameter (baseline)', () => { const sql = 'SELECT id from users WHERE filter_param = token_parameters.user_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'test_id', filter_param: 'test_param' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'test_id', filter_param: 'test_param' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['test_param']), + lookup: UnscopedParameterLookup.normalized(['test_param']), bucketParameters: [{ id: 'test_id' }] } ]); expect(query.getLookups(normalizeTokenParameters({ user_id: 'test' }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['test']) + UnscopedParameterLookup.normalized(['test']) ]); }); test('function on token_parameter', () => { const sql = 'SELECT id from users WHERE filter_param = upper(token_parameters.user_id)'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'test_id', filter_param: 'test_param' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'test_id', filter_param: 'test_param' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['test_param']), + lookup: UnscopedParameterLookup.normalized(['test_param']), bucketParameters: [{ id: 'test_id' }] } ]); expect(query.getLookups(normalizeTokenParameters({ user_id: 'test' }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['TEST']) + UnscopedParameterLookup.normalized(['TEST']) ]); }); test('token parameter member operator', () => { const sql = "SELECT id from users WHERE filter_param = token_parameters.some_param ->> 'description'"; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'test_id', filter_param: 'test_param' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'test_id', filter_param: 'test_param' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['test_param']), + lookup: UnscopedParameterLookup.normalized(['test_param']), bucketParameters: [{ id: 'test_id' }] } ]); expect(query.getLookups(normalizeTokenParameters({ some_param: { description: 'test_description' } }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['test_description']) + UnscopedParameterLookup.normalized(['test_description']) ]); }); test('token parameter and binary operator', () => { const sql = 'SELECT id from users WHERE filter_param = token_parameters.some_param + 2'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getLookups(normalizeTokenParameters({ some_param: 3 }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [5n]) + UnscopedParameterLookup.normalized([5n]) ]); }); test('token parameter IS NULL as filter', () => { const sql = 'SELECT id from users WHERE filter_param = (token_parameters.some_param IS NULL)'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getLookups(normalizeTokenParameters({ some_param: null }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [1n]) + UnscopedParameterLookup.normalized([1n]) ]); expect(query.getLookups(normalizeTokenParameters({ some_param: 'test' }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [0n]) + UnscopedParameterLookup.normalized([0n]) ]); }); test('direct token parameter', () => { const sql = 'SELECT FROM users WHERE token_parameters.some_param'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', [1n]), + lookup: UnscopedParameterLookup.normalized([1n]), bucketParameters: [{}] } ]); expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [0n]) + UnscopedParameterLookup.normalized([0n]) ]); expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [1n]) + UnscopedParameterLookup.normalized([1n]) ]); }); test('token parameter IS NULL', () => { const sql = 'SELECT FROM users WHERE token_parameters.some_param IS NULL'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', [1n]), + lookup: UnscopedParameterLookup.normalized([1n]), bucketParameters: [{}] } ]); expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [1n]) + UnscopedParameterLookup.normalized([1n]) ]); expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [0n]) + UnscopedParameterLookup.normalized([0n]) ]); }); test('token parameter IS NOT NULL', () => { const sql = 'SELECT FROM users WHERE token_parameters.some_param IS NOT NULL'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', [1n]), + lookup: UnscopedParameterLookup.normalized([1n]), bucketParameters: [{}] } ]); expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [0n]) + UnscopedParameterLookup.normalized([0n]) ]); expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [1n]) + UnscopedParameterLookup.normalized([1n]) ]); }); test('token parameter NOT', () => { const sql = 'SELECT FROM users WHERE NOT token_parameters.is_admin'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', [1n]), + lookup: UnscopedParameterLookup.normalized([1n]), bucketParameters: [{}] } ]); expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: false }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [1n]) + UnscopedParameterLookup.normalized([1n]) ]); expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: 123 }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [0n]) + UnscopedParameterLookup.normalized([0n]) ]); }); test('row filter and token parameter IS NULL', () => { const sql = 'SELECT FROM users WHERE users.id = token_parameters.user_id AND token_parameters.some_param IS NULL'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['user1', 1n]), + lookup: UnscopedParameterLookup.normalized(['user1', 1n]), bucketParameters: [{}] } ]); expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['user1', 1n]) + UnscopedParameterLookup.normalized(['user1', 1n]) ]); expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['user1', 0n]) + UnscopedParameterLookup.normalized(['user1', 0n]) ]); }); test('row filter and direct token parameter', () => { const sql = 'SELECT FROM users WHERE users.id = token_parameters.user_id AND token_parameters.some_param'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['user1', 1n]), + lookup: UnscopedParameterLookup.normalized(['user1', 1n]), bucketParameters: [{}] } ]); expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['user1', 1n]) + UnscopedParameterLookup.normalized(['user1', 1n]) ]); expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['user1', 0n]) + UnscopedParameterLookup.normalized(['user1', 0n]) ]); }); test('cast', () => { const sql = 'SELECT FROM users WHERE users.id = cast(token_parameters.user_id as text)'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['user1']) + UnscopedParameterLookup.normalized(['user1']) ]); expect(query.getLookups(normalizeTokenParameters({ user_id: 123 }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['123']) + UnscopedParameterLookup.normalized(['123']) ]); }); test('IS NULL row filter', () => { const sql = 'SELECT id FROM users WHERE role IS NULL'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'user1', role: null })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'user1', role: null })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', []), + lookup: UnscopedParameterLookup.normalized([]), bucketParameters: [{ id: 'user1' }] } ]); expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', []) + UnscopedParameterLookup.normalized([]) ]); }); @@ -331,60 +446,78 @@ describe('parameter queries', () => { // Not supported: token_parameters.is_admin != false // Support could be added later. const sql = 'SELECT FROM users WHERE users.id = token_parameters.user_id AND token_parameters.is_admin'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['user1', 1n]), + lookup: UnscopedParameterLookup.normalized(['user1', 1n]), bucketParameters: [{}] } ]); expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: true }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['user1', 1n]) + UnscopedParameterLookup.normalized(['user1', 1n]) ]); // Would not match any actual lookups expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: false }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['user1', 0n]) + UnscopedParameterLookup.normalized(['user1', 0n]) ]); }); test('token filter (2)', () => { const sql = 'SELECT users.id AS user_id, token_parameters.is_admin as is_admin FROM users WHERE users.id = token_parameters.user_id AND token_parameters.is_admin'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['user1', 1n]), + lookup: UnscopedParameterLookup.normalized(['user1', 1n]), bucketParameters: [{ user_id: 'user1' }] } ]); expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: true }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['user1', 1n]) + UnscopedParameterLookup.normalized(['user1', 1n]) ]); expect( query.resolveBucketDescriptions( [{ user_id: 'user1' }], normalizeTokenParameters({ user_id: 'user1', is_admin: true }), - identityBucketTransformer + { bucketPrefix: 'mybucket' } ) ).toEqual([{ bucket: 'mybucket["user1",1]', priority: 3 }]); }); test('case-sensitive parameter queries (1)', () => { const sql = 'SELECT users."userId" AS user_id FROM users WHERE users."userId" = token_parameters.user_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ userId: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { userId: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['user1']), + lookup: UnscopedParameterLookup.normalized(['user1']), bucketParameters: [{ user_id: 'user1' }] } @@ -396,16 +529,22 @@ describe('parameter queries', () => { // This may change in the future - we should check against expected behavior for // Postgres and/or SQLite. const sql = 'SELECT users.userId AS user_id FROM users WHERE users.userId = token_parameters.user_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "userId" instead.` }, { message: `Unquoted identifiers are converted to lower-case. Use "userId" instead.` } ]); - expect(query.evaluateParameterRow({ userId: 'user1' })).toEqual([]); - expect(query.evaluateParameterRow({ userid: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { userId: 'user1' })).toEqual([]); + expect(query.evaluateParameterRow(TABLE_USERS, { userid: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['user1']), + lookup: UnscopedParameterLookup.normalized(['user1']), bucketParameters: [{ user_id: 'user1' }] } @@ -414,7 +553,13 @@ describe('parameter queries', () => { test('case-sensitive parameter queries (3)', () => { const sql = 'SELECT user_id FROM users WHERE Users.user_id = token_parameters.user_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "Users" instead.` } ]); @@ -422,7 +567,13 @@ describe('parameter queries', () => { test('case-sensitive parameter queries (4)', () => { const sql = 'SELECT Users.user_id FROM users WHERE user_id = token_parameters.user_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "Users" instead.` } ]); @@ -430,7 +581,13 @@ describe('parameter queries', () => { test('case-sensitive parameter queries (5)', () => { const sql = 'SELECT user_id FROM Users WHERE user_id = token_parameters.user_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "Users" instead.` } ]); @@ -438,7 +595,13 @@ describe('parameter queries', () => { test('case-sensitive parameter queries (6)', () => { const sql = 'SELECT userId FROM users'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "userId" instead.` } ]); @@ -446,7 +609,13 @@ describe('parameter queries', () => { test('case-sensitive parameter queries (7)', () => { const sql = 'SELECT user_id as userId FROM users'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "userId" instead.` } ]); @@ -454,37 +623,49 @@ describe('parameter queries', () => { test('dynamic global parameter query', () => { const sql = "SELECT workspaces.id AS workspace_id FROM workspaces WHERE visibility = 'public'"; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'workspace1', visibility: 'public' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_WORKSPACES, { id: 'workspace1', visibility: 'public' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', []), + lookup: UnscopedParameterLookup.normalized([]), bucketParameters: [{ workspace_id: 'workspace1' }] } ]); - expect(query.evaluateParameterRow({ id: 'workspace1', visibility: 'private' })).toEqual([]); + expect(query.evaluateParameterRow(TABLE_WORKSPACES, { id: 'workspace1', visibility: 'private' })).toEqual([]); }); test('multiple different functions on token_parameter with AND', () => { // This is treated as two separate lookup index values const sql = 'SELECT id from users WHERE filter_param = upper(token_parameters.user_id) AND filter_param = lower(token_parameters.user_id)'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'test_id', filter_param: 'test_param' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'test_id', filter_param: 'test_param' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['test_param', 'test_param']), + lookup: UnscopedParameterLookup.normalized(['test_param', 'test_param']), bucketParameters: [{ id: 'test_id' }] } ]); expect(query.getLookups(normalizeTokenParameters({ user_id: 'test' }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['TEST', 'test']) + UnscopedParameterLookup.normalized(['TEST', 'test']) ]); }); @@ -492,22 +673,30 @@ describe('parameter queries', () => { // This is treated as the same index lookup value, can use OR with the two clauses const sql = 'SELECT id from users WHERE filter_param1 = upper(token_parameters.user_id) OR filter_param2 = upper(token_parameters.user_id)'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'test_id', filter_param1: 'test1', filter_param2: 'test2' })).toEqual([ + expect( + query.evaluateParameterRow(TABLE_USERS, { id: 'test_id', filter_param1: 'test1', filter_param2: 'test2' }) + ).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['test1']), + lookup: UnscopedParameterLookup.normalized(['test1']), bucketParameters: [{ id: 'test_id' }] }, { - lookup: ParameterLookup.normalized('mybucket', '1', ['test2']), + lookup: UnscopedParameterLookup.normalized(['test2']), bucketParameters: [{ id: 'test_id' }] } ]); expect(query.getLookups(normalizeTokenParameters({ user_id: 'test' }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['TEST']) + UnscopedParameterLookup.normalized(['TEST']) ]); }); @@ -520,17 +709,18 @@ describe('parameter queries', () => { accept_potentially_dangerous_queries: true, ...PARSE_OPTIONS }, - '1' + '1', + EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'group1', category: 'red' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_POSTS, { id: 'group1', category: 'red' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['red']), + lookup: UnscopedParameterLookup.normalized(['red']), bucketParameters: [{}] } ]); expect(query.getLookups(normalizeTokenParameters({}, { category_id: 'red' }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['red']) + UnscopedParameterLookup.normalized(['red']) ]); }); @@ -543,11 +733,12 @@ describe('parameter queries', () => { accept_potentially_dangerous_queries: true, ...PARSE_OPTIONS }, - '1' + '1', + EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getLookups(normalizeTokenParameters({}, { details: { category: 'red' } }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['red']) + UnscopedParameterLookup.normalized(['red']) ]); }); @@ -560,11 +751,12 @@ describe('parameter queries', () => { accept_potentially_dangerous_queries: true, ...PARSE_OPTIONS }, - '1' + '1', + EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getLookups(normalizeTokenParameters({}, { details: { category: 'red' } }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['red']) + UnscopedParameterLookup.normalized(['red']) ]); }); @@ -578,12 +770,13 @@ describe('parameter queries', () => { accept_potentially_dangerous_queries: true, ...PARSE_OPTIONS }, - '1' + '1', + EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'region1', name: 'colorado' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_REGIONS, { id: 'region1', name: 'colorado' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['colorado']), + lookup: UnscopedParameterLookup.normalized(['colorado']), bucketParameters: [ { region_id: 'region1' @@ -600,57 +793,157 @@ describe('parameter queries', () => { } ) ) - ).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['colorado']), - ParameterLookup.normalized('mybucket', '1', ['texas']) - ]); + ).toEqual([UnscopedParameterLookup.normalized(['colorado']), UnscopedParameterLookup.normalized(['texas'])]); }); test('user_parameters in SELECT', function () { const sql = 'SELECT id, user_parameters.other_id as other_id FROM users WHERE id = token_parameters.user_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['user1']), + lookup: UnscopedParameterLookup.normalized(['user1']), bucketParameters: [{ id: 'user1' }] } ]); const requestParams = normalizeTokenParameters({ user_id: 'user1' }, { other_id: 'red' }); - expect(query.getLookups(requestParams)).toEqual([ParameterLookup.normalized('mybucket', '1', ['user1'])]); + expect(query.getLookups(requestParams)).toEqual([UnscopedParameterLookup.normalized(['user1'])]); }); test('request.parameters() in SELECT', function () { const sql = "SELECT id, request.parameters() ->> 'other_id' as other_id FROM users WHERE id = token_parameters.user_id"; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['user1']), + lookup: UnscopedParameterLookup.normalized(['user1']), bucketParameters: [{ id: 'user1' }] } ]); const requestParams = normalizeTokenParameters({ user_id: 'user1' }, { other_id: 'red' }); - expect(query.getLookups(requestParams)).toEqual([ParameterLookup.normalized('mybucket', '1', ['user1'])]); + expect(query.getLookups(requestParams)).toEqual([UnscopedParameterLookup.normalized(['user1'])]); }); test('request.jwt()', function () { const sql = "SELECT FROM users WHERE id = request.jwt() ->> 'sub'"; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); const requestParams = normalizeTokenParameters({ user_id: 'user1' }); - expect(query.getLookups(requestParams)).toEqual([ParameterLookup.normalized('mybucket', '1', ['user1'])]); + expect(query.getLookups(requestParams)).toEqual([UnscopedParameterLookup.normalized(['user1'])]); }); test('request.user_id()', function () { const sql = 'SELECT FROM users WHERE id = request.user_id()'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); const requestParams = normalizeTokenParameters({ user_id: 'user1' }); - expect(query.getLookups(requestParams)).toEqual([ParameterLookup.normalized('mybucket', '1', ['user1'])]); + expect(query.getLookups(requestParams)).toEqual([UnscopedParameterLookup.normalized(['user1'])]); + }); + + describe('custom hydrationState', function () { + const hydrationState: HydrationState = { + getBucketSourceScope(source) { + return { bucketPrefix: `${source.uniqueName}-test` }; + }, + getParameterIndexLookupScope(source) { + return { + lookupName: `${source.defaultLookupScope.lookupName}.test`, + queryId: `${source.defaultLookupScope.queryId}.test` + }; + } + }; + + let query: SqlParameterQuery; + + beforeEach(() => { + const sql = 'SELECT id as group_id FROM groups WHERE token_parameters.user_id IN groups.user_ids'; + query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + 'myquery', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; + + expect(query.errors).toEqual([]); + }); + + test('for lookups', function () { + const merged = mergeParameterIndexLookupCreators(hydrationState, [query]); + const result = merged.evaluateParameterRow(TABLE_GROUPS, { + id: 'group1', + user_ids: JSON.stringify(['test-user', 'other-user']) + }); + expect(result).toEqual([ + { + lookup: ScopedParameterLookup.direct({ lookupName: 'mybucket.test', queryId: 'myquery.test' }, ['test-user']), + bucketParameters: [{ group_id: 'group1' }] + }, + { + lookup: ScopedParameterLookup.direct({ lookupName: 'mybucket.test', queryId: 'myquery.test' }, [ + 'other-user' + ]), + bucketParameters: [{ group_id: 'group1' }] + } + ]); + }); + + test('for queries', function () { + const hydrated = query.createParameterQuerierSource({ hydrationState }); + + const queriers: BucketParameterQuerier[] = []; + const errors: QuerierError[] = []; + const pending = { queriers, errors }; + + const querierOptions: GetQuerierOptions = { + hasDefaultStreams: true, + globalParameters: new RequestParameters( + { + sub: 'test-user' + }, + {} + ), + streams: {} + }; + + hydrated.pushBucketParameterQueriers(pending, querierOptions); + + expect(errors).toEqual([]); + expect(queriers.length).toBe(1); + + const querier = queriers[0]; + + expect(querier.parameterQueryLookups).toEqual([ + ScopedParameterLookup.direct({ lookupName: 'mybucket.test', queryId: 'myquery.test' }, ['test-user']) + ]); + }); }); test('invalid OR in parameter queries', () => { @@ -658,51 +951,99 @@ describe('parameter queries', () => { // into separate queries, but it's a significant change. For now, developers should do that manually. const sql = "SELECT workspaces.id AS workspace_id FROM workspaces WHERE workspaces.user_id = token_parameters.user_id OR visibility = 'public'"; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors[0].message).toMatch(/must use the same parameters/); }); test('invalid OR in parameter queries (2)', () => { const sql = 'SELECT id from users WHERE filter_param = upper(token_parameters.user_id) OR filter_param = lower(token_parameters.user_id)'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors[0].message).toMatch(/must use the same parameters/); }); test('invalid parameter match clause (1)', () => { const sql = 'SELECT FROM users WHERE (id = token_parameters.user_id) = false'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors[0].message).toMatch(/Parameter match clauses cannot be used here/); }); test('invalid parameter match clause (2)', () => { const sql = 'SELECT FROM users WHERE NOT (id = token_parameters.user_id)'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors[0].message).toMatch(/Parameter match clauses cannot be used here/); }); test('invalid parameter match clause (3)', () => { // May be supported in the future const sql = 'SELECT FROM users WHERE token_parameters.start_at < users.created_at'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors[0].message).toMatch(/Cannot use table values and parameters in the same clauses/); }); test('invalid parameter match clause (4)', () => { const sql = 'SELECT FROM users WHERE json_extract(users.description, token_parameters.path)'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors[0].message).toMatch(/Cannot use table values and parameters in the same clauses/); }); test('invalid parameter match clause (5)', () => { const sql = 'SELECT (user_parameters.role = posts.roles) as r FROM posts'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors[0].message).toMatch(/Parameter match expression is not allowed here/); }); test('invalid function schema', () => { const sql = 'SELECT FROM users WHERE something.length(users.id) = 0'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors[0].message).toMatch(/Function 'something.length' is not defined/); }); @@ -716,7 +1057,8 @@ describe('parameter queries', () => { ...PARSE_OPTIONS, schema }, - '1' + '1', + EMPTY_DATA_SOURCE ); expect(q1.errors).toMatchObject([]); @@ -724,7 +1066,8 @@ describe('parameter queries', () => { 'q5', 'SELECT id as asset_id FROM assets WHERE other_id = token_parameters.user_id', { ...PARSE_OPTIONS, schema }, - '1' + '1', + EMPTY_DATA_SOURCE ); expect(q2.errors).toMatchObject([ @@ -738,7 +1081,13 @@ describe('parameter queries', () => { describe('dangerous queries', function () { function testDangerousQuery(sql: string) { test(sql, function () { - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toMatchObject([ { message: @@ -750,7 +1099,13 @@ describe('parameter queries', () => { } function testSafeQuery(sql: string) { test(sql, function () { - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.usesDangerousRequestParameters).toEqual(false); }); @@ -779,7 +1134,13 @@ describe('parameter queries', () => { describe('bucket priorities', () => { test('valid definition', function () { const sql = 'SELECT id as group_id, 1 AS _priority FROM groups WHERE token_parameters.user_id IN groups.user_ids'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(Object.entries(query.lookupExtractors)).toHaveLength(1); expect(Object.entries(query.parameterExtractors)).toHaveLength(0); @@ -789,7 +1150,13 @@ describe('parameter queries', () => { test('valid definition, static query', function () { const sql = 'SELECT token_parameters.user_id, 0 AS _priority'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(Object.entries(query.parameterExtractors)).toHaveLength(1); expect(query.bucketParameters).toEqual(['user_id']); @@ -798,21 +1165,39 @@ describe('parameter queries', () => { test('invalid dynamic query', function () { const sql = 'SELECT LENGTH(assets.name) AS _priority FROM assets'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toMatchObject([{ message: 'Priority must be a simple integer literal' }]); }); test('invalid literal type', function () { const sql = "SELECT 'very fast please' AS _priority"; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toMatchObject([{ message: 'Priority must be a simple integer literal' }]); }); test('invalid literal value', function () { const sql = 'SELECT 15 AS _priority'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toMatchObject([ { message: 'Invalid value for priority, must be between 0 and 3 (inclusive).' } diff --git a/packages/sync-rules/test/src/static_parameter_queries.test.ts b/packages/sync-rules/test/src/static_parameter_queries.test.ts index 3e2861307..62566d4eb 100644 --- a/packages/sync-rules/test/src/static_parameter_queries.test.ts +++ b/packages/sync-rules/test/src/static_parameter_queries.test.ts @@ -1,91 +1,136 @@ import { describe, expect, test } from 'vitest'; -import { RequestParameters, SqlParameterQuery, SqlSyncRules } from '../../src/index.js'; +import { BucketDataScope, HydrationState } from '../../src/HydrationState.js'; +import { + BucketParameterQuerier, + GetQuerierOptions, + QuerierError, + RequestParameters, + ScopedParameterLookup, + SqlParameterQuery +} from '../../src/index.js'; import { StaticSqlParameterQuery } from '../../src/StaticSqlParameterQuery.js'; -import { identityBucketTransformer, normalizeTokenParameters, PARSE_OPTIONS } from './util.js'; +import { EMPTY_DATA_SOURCE, normalizeTokenParameters, PARSE_OPTIONS } from './util.js'; describe('static parameter queries', () => { + const MYBUCKET_SCOPE: BucketDataScope = { + bucketPrefix: 'mybucket' + }; + test('basic query', function () { const sql = 'SELECT token_parameters.user_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters!).toEqual(['user_id']); - expect( - query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), identityBucketTransformer) - ).toEqual([{ bucket: 'mybucket["user1"]', priority: 3 }]); + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), MYBUCKET_SCOPE)).toEqual([ + { bucket: 'mybucket["user1"]', priority: 3 } + ]); }); - test('uses bucket id transformer', function () { + test('uses bucketPrefix', function () { const sql = 'SELECT token_parameters.user_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters!).toEqual(['user_id']); expect( - query.getStaticBucketDescriptions( - normalizeTokenParameters({ user_id: 'user1' }), - SqlSyncRules.versionedBucketIdTransformer('1') - ) + query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), { + bucketPrefix: '1#mybucket' + }) ).toEqual([{ bucket: '1#mybucket["user1"]', priority: 3 }]); }); test('global query', function () { const sql = 'SELECT'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters!).toEqual([]); - expect( - query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), identityBucketTransformer) - ).toEqual([{ bucket: 'mybucket[]', priority: 3 }]); + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), MYBUCKET_SCOPE)).toEqual([ + { bucket: 'mybucket[]', priority: 3 } + ]); }); test('query with filter', function () { const sql = 'SELECT token_parameters.user_id WHERE token_parameters.is_admin'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect( - query.getStaticBucketDescriptions( - normalizeTokenParameters({ user_id: 'user1', is_admin: true }), - identityBucketTransformer - ) + query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1', is_admin: true }), MYBUCKET_SCOPE) ).toEqual([{ bucket: 'mybucket["user1"]', priority: 3 }]); expect( - query.getStaticBucketDescriptions( - normalizeTokenParameters({ user_id: 'user1', is_admin: false }), - identityBucketTransformer - ) + query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1', is_admin: false }), MYBUCKET_SCOPE) ).toEqual([]); }); test('function in select clause', function () { const sql = 'SELECT upper(token_parameters.user_id) as upper_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect( - query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), identityBucketTransformer) - ).toEqual([{ bucket: 'mybucket["USER1"]', priority: 3 }]); + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), MYBUCKET_SCOPE)).toEqual([ + { bucket: 'mybucket["USER1"]', priority: 3 } + ]); expect(query.bucketParameters!).toEqual(['upper_id']); }); test('function in filter clause', function () { const sql = "SELECT WHERE upper(token_parameters.role) = 'ADMIN'"; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect( - query.getStaticBucketDescriptions(normalizeTokenParameters({ role: 'admin' }), identityBucketTransformer) - ).toEqual([{ bucket: 'mybucket[]', priority: 3 }]); - expect( - query.getStaticBucketDescriptions(normalizeTokenParameters({ role: 'user' }), identityBucketTransformer) - ).toEqual([]); + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ role: 'admin' }), MYBUCKET_SCOPE)).toEqual([ + { bucket: 'mybucket[]', priority: 3 } + ]); + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ role: 'user' }), MYBUCKET_SCOPE)).toEqual([]); }); test('comparison in filter clause', function () { const sql = 'SELECT WHERE token_parameters.id1 = token_parameters.id2'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect( - query.getStaticBucketDescriptions(normalizeTokenParameters({ id1: 't1', id2: 't1' }), identityBucketTransformer) + query.getStaticBucketDescriptions(normalizeTokenParameters({ id1: 't1', id2: 't1' }), MYBUCKET_SCOPE) ).toEqual([{ bucket: 'mybucket[]', priority: 3 }]); expect( - query.getStaticBucketDescriptions(normalizeTokenParameters({ id1: 't1', id2: 't2' }), identityBucketTransformer) + query.getStaticBucketDescriptions(normalizeTokenParameters({ id1: 't1', id2: 't2' }), MYBUCKET_SCOPE) ).toEqual([]); }); @@ -98,88 +143,129 @@ describe('static parameter queries', () => { ...PARSE_OPTIONS, accept_potentially_dangerous_queries: true }, - '1' + '1', + EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect( - query.getStaticBucketDescriptions(normalizeTokenParameters({}, { org_id: 'test' }), identityBucketTransformer) - ).toEqual([{ bucket: 'mybucket["test"]', priority: 3 }]); + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({}, { org_id: 'test' }), MYBUCKET_SCOPE)).toEqual( + [{ bucket: 'mybucket["test"]', priority: 3 }] + ); }); test('request.jwt()', function () { const sql = "SELECT request.jwt() ->> 'sub' as user_id"; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['user_id']); - expect( - query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), identityBucketTransformer) - ).toEqual([{ bucket: 'mybucket["user1"]', priority: 3 }]); + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), MYBUCKET_SCOPE)).toEqual([ + { bucket: 'mybucket["user1"]', priority: 3 } + ]); }); test('request.user_id()', function () { const sql = 'SELECT request.user_id() as user_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['user_id']); - expect( - query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), identityBucketTransformer) - ).toEqual([{ bucket: 'mybucket["user1"]', priority: 3 }]); + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), MYBUCKET_SCOPE)).toEqual([ + { bucket: 'mybucket["user1"]', priority: 3 } + ]); }); test('static value', function () { const sql = `SELECT WHERE 1`; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), identityBucketTransformer) - ).toEqual([{ bucket: 'mybucket[]', priority: 3 }]); + expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), MYBUCKET_SCOPE)).toEqual([ + { bucket: 'mybucket[]', priority: 3 } + ]); }); test('static expression (1)', function () { const sql = `SELECT WHERE 1 = 1`; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), identityBucketTransformer) - ).toEqual([{ bucket: 'mybucket[]', priority: 3 }]); + expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), MYBUCKET_SCOPE)).toEqual([ + { bucket: 'mybucket[]', priority: 3 } + ]); }); test('static expression (2)', function () { const sql = `SELECT WHERE 1 != 1`; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), identityBucketTransformer) - ).toEqual([]); + expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), MYBUCKET_SCOPE)).toEqual([]); }); test('static IN expression', function () { const sql = `SELECT WHERE 'admin' IN '["admin", "superuser"]'`; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), identityBucketTransformer) - ).toEqual([{ bucket: 'mybucket[]', priority: 3 }]); + expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), MYBUCKET_SCOPE)).toEqual([ + { bucket: 'mybucket[]', priority: 3 } + ]); }); test('IN for permissions in request.jwt() (1)', function () { // Can use -> or ->> here const sql = `SELECT 'read:users' IN (request.jwt() ->> 'permissions') as access_granted`; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect( query.getStaticBucketDescriptions( new RequestParameters({ sub: '', permissions: ['write', 'read:users'] }, {}), - identityBucketTransformer + MYBUCKET_SCOPE ) ).toEqual([{ bucket: 'mybucket[1]', priority: 3 }]); expect( query.getStaticBucketDescriptions( new RequestParameters({ sub: '', permissions: ['write', 'write:users'] }, {}), - identityBucketTransformer + MYBUCKET_SCOPE ) ).toEqual([{ bucket: 'mybucket[0]', priority: 3 }]); }); @@ -187,43 +273,55 @@ describe('static parameter queries', () => { test('IN for permissions in request.jwt() (2)', function () { // Can use -> or ->> here const sql = `SELECT WHERE 'read:users' IN (request.jwt() ->> 'permissions')`; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect( query.getStaticBucketDescriptions( new RequestParameters({ sub: '', permissions: ['write', 'read:users'] }, {}), - identityBucketTransformer + MYBUCKET_SCOPE ) ).toEqual([{ bucket: 'mybucket[]', priority: 3 }]); expect( query.getStaticBucketDescriptions( new RequestParameters({ sub: '', permissions: ['write', 'write:users'] }, {}), - identityBucketTransformer + MYBUCKET_SCOPE ) ).toEqual([]); }); test('IN for permissions in request.jwt() (3)', function () { const sql = `SELECT WHERE request.jwt() ->> 'role' IN '["admin", "superuser"]'`; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect( - query.getStaticBucketDescriptions( - new RequestParameters({ sub: '', role: 'superuser' }, {}), - identityBucketTransformer - ) + query.getStaticBucketDescriptions(new RequestParameters({ sub: '', role: 'superuser' }, {}), MYBUCKET_SCOPE) ).toEqual([{ bucket: 'mybucket[]', priority: 3 }]); expect( - query.getStaticBucketDescriptions( - new RequestParameters({ sub: '', role: 'superadmin' }, {}), - identityBucketTransformer - ) + query.getStaticBucketDescriptions(new RequestParameters({ sub: '', role: 'superadmin' }, {}), MYBUCKET_SCOPE) ).toEqual([]); }); test('case-sensitive queries (1)', () => { const sql = 'SELECT request.user_id() as USER_ID'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "USER_ID" instead.` } ]); @@ -232,7 +330,13 @@ describe('static parameter queries', () => { describe('dangerous queries', function () { function testDangerousQuery(sql: string) { test(sql, function () { - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toMatchObject([ { message: @@ -244,7 +348,13 @@ describe('static parameter queries', () => { } function testSafeQuery(sql: string) { test(sql, function () { - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.usesDangerousRequestParameters).toEqual(false); }); @@ -262,4 +372,63 @@ describe('static parameter queries', () => { "select request.parameters() ->> 'project_id' as project_id where request.jwt() ->> 'role' = 'authenticated'" ); }); + + test('custom hydrationState for buckets', function () { + const sql = 'SELECT token_parameters.user_id'; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; + + expect(query.errors).toEqual([]); + + const hydrationState: HydrationState = { + getBucketSourceScope(source) { + return { bucketPrefix: `${source.uniqueName}-test` }; + }, + getParameterIndexLookupScope(source) { + return { + lookupName: `${source.defaultLookupScope.lookupName}.test`, + queryId: `${source.defaultLookupScope.queryId}.test` + }; + } + }; + + // Internal API + const hydrated = query.createParameterQuerierSource({ hydrationState }); + + const queriers: BucketParameterQuerier[] = []; + const errors: QuerierError[] = []; + const pending = { queriers, errors }; + + const querierOptions: GetQuerierOptions = { + hasDefaultStreams: true, + globalParameters: new RequestParameters( + { + sub: 'test-user' + }, + {} + ), + streams: {} + }; + + hydrated.pushBucketParameterQueriers(pending, querierOptions); + + expect(errors).toEqual([]); + expect(queriers).toMatchObject([ + { + staticBuckets: [ + { + bucket: 'mybucket-test["test-user"]', + definition: 'mybucket', + inclusion_reasons: ['default'], + priority: 3 + } + ] + } + ]); + }); }); diff --git a/packages/sync-rules/test/src/streams.test.ts b/packages/sync-rules/test/src/streams.test.ts index 519a8cd28..dc8b59b85 100644 --- a/packages/sync-rules/test/src/streams.test.ts +++ b/packages/sync-rules/test/src/streams.test.ts @@ -1,28 +1,41 @@ /// import { describe, expect, test } from 'vitest'; +import { HydrationState, ParameterLookupScope, versionedHydrationState } from '../../src/HydrationState.js'; import { BucketParameterQuerier, CompatibilityContext, CompatibilityEdition, + CreateSourceParams, + debugHydratedMergedSource, DEFAULT_TAG, + EvaluationResult, GetBucketParameterQuerierResult, GetQuerierOptions, mergeBucketParameterQueriers, - ParameterLookup, + UnscopedParameterLookup, QuerierError, RequestParameters, SourceTableInterface, SqliteJsonRow, SqliteRow, - SqlSyncRules, StaticSchema, StreamParseOptions, SyncStream, - syncStreamFromSql + syncStreamFromSql, + ScopedParameterLookup } from '../../src/index.js'; import { normalizeQuerierOptions, PARSE_OPTIONS, TestSourceTable } from './util.js'; describe('streams', () => { + const STREAM_0: ParameterLookupScope = { + lookupName: 'stream', + queryId: '0' + }; + const STREAM_1: ParameterLookupScope = { + lookupName: 'stream', + queryId: '1' + }; + test('refuses edition: 1', () => { expect(() => syncStreamFromSql('stream', 'SELECT * FROM comments', { @@ -37,7 +50,7 @@ describe('streams', () => { expect(desc.variants).toHaveLength(1); expect(evaluateBucketIds(desc, COMMENTS, { id: 'foo' })).toStrictEqual(['1#stream|0[]']); - expect(desc.evaluateRow({ sourceTable: USERS, bucketIdTransformer, record: { id: 'foo' } })).toHaveLength(0); + expect(desc.dataSources[0].evaluateRow({ sourceTable: USERS, record: { id: 'foo' } })).toHaveLength(0); }); test('row condition', () => { @@ -67,18 +80,14 @@ describe('streams', () => { test('legacy token parameter', async () => { const desc = parseStream(`SELECT * FROM issues WHERE owner_id = auth.parameter('$.parameters.test')`); + const source = debugHydratedMergedSource(desc, hydrationParams); const queriers: BucketParameterQuerier[] = []; const errors: QuerierError[] = []; const pending = { queriers, errors }; - desc.pushBucketParameterQueriers( + source.pushBucketParameterQueriers( pending, - normalizeQuerierOptions( - { test: 'foo' }, - {}, - { stream: [{ opaque_id: 0, parameters: null }] }, - bucketIdTransformer - ) + normalizeQuerierOptions({ test: 'foo' }, {}, { stream: [{ opaque_id: 0, parameters: null }] }) ); expect(mergeBucketParameterQueriers(queriers).staticBuckets).toEqual([ @@ -220,9 +229,14 @@ describe('streams', () => { '1#stream|1[]' ]); - expect(desc.evaluateParameterRow(ISSUES, { id: 'i1', owner_id: 'u1' })).toStrictEqual([ + expect( + debugHydratedMergedSource(desc, hydrationParams).evaluateParameterRow(ISSUES, { + id: 'i1', + owner_id: 'u1' + }) + ).toStrictEqual([ { - lookup: ParameterLookup.normalized('stream', '0', ['u1']), + lookup: ScopedParameterLookup.direct(STREAM_0, ['u1']), bucketParameters: [ { result: 'i1' @@ -231,8 +245,8 @@ describe('streams', () => { } ]); - function getParameterSets(lookups: ParameterLookup[]) { - expect(lookups).toStrictEqual([ParameterLookup.normalized('stream', '0', ['u1'])]); + function getParameterSets(lookups: ScopedParameterLookup[]) { + expect(lookups).toStrictEqual([ScopedParameterLookup.direct(STREAM_0, ['u1'])]); return [{ result: 'i1' }]; } @@ -251,11 +265,12 @@ describe('streams', () => { const desc = parseStream( 'SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id())' ); + const lookup = desc.parameterIndexLookupCreators[0]; - expect(desc.tableSyncsParameters(ISSUES)).toBe(true); - expect(desc.evaluateParameterRow(ISSUES, { id: 'issue_id', owner_id: 'user1', name: 'name' })).toStrictEqual([ + expect(lookup.tableSyncsParameters(ISSUES)).toBe(true); + expect(lookup.evaluateParameterRow(ISSUES, { id: 'issue_id', owner_id: 'user1', name: 'name' })).toStrictEqual([ { - lookup: ParameterLookup.normalized('stream', '0', ['user1']), + lookup: UnscopedParameterLookup.normalized(['user1']), bucketParameters: [ { result: 'issue_id' @@ -271,7 +286,7 @@ describe('streams', () => { await queryBucketIds(desc, { token: { sub: 'user1' }, getParameterSets(lookups) { - expect(lookups).toStrictEqual([ParameterLookup.normalized('stream', '0', ['user1'])]); + expect(lookups).toStrictEqual([ScopedParameterLookup.direct(STREAM_0, ['user1'])]); return [{ result: 'issue_id' }]; } @@ -281,13 +296,14 @@ describe('streams', () => { test('parameter value in subquery', async () => { const desc = parseStream('SELECT * FROM issues WHERE auth.user_id() IN (SELECT id FROM users WHERE is_admin)'); + const lookup = desc.parameterIndexLookupCreators[0]; - expect(desc.tableSyncsParameters(ISSUES)).toBe(false); - expect(desc.tableSyncsParameters(USERS)).toBe(true); + expect(lookup.tableSyncsParameters(ISSUES)).toBe(false); + expect(lookup.tableSyncsParameters(USERS)).toBe(true); - expect(desc.evaluateParameterRow(USERS, { id: 'u', is_admin: 1n })).toStrictEqual([ + expect(lookup.evaluateParameterRow(USERS, { id: 'u', is_admin: 1n })).toStrictEqual([ { - lookup: ParameterLookup.normalized('stream', '0', ['u']), + lookup: UnscopedParameterLookup.normalized(['u']), bucketParameters: [ { result: 'u' @@ -295,14 +311,14 @@ describe('streams', () => { ] } ]); - expect(desc.evaluateParameterRow(USERS, { id: 'u', is_admin: 0n })).toStrictEqual([]); + expect(lookup.evaluateParameterRow(USERS, { id: 'u', is_admin: 0n })).toStrictEqual([]); // Should return bucket id for admin users expect( await queryBucketIds(desc, { token: { sub: 'u' }, - getParameterSets: (lookups: ParameterLookup[]) => { - expect(lookups).toStrictEqual([ParameterLookup.normalized('stream', '0', ['u'])]); + getParameterSets: (lookups: ScopedParameterLookup[]) => { + expect(lookups).toStrictEqual([ScopedParameterLookup.direct(STREAM_0, ['u'])]); return [{ result: 'u' }]; } }) @@ -312,8 +328,8 @@ describe('streams', () => { expect( await queryBucketIds(desc, { token: { sub: 'u2' }, - getParameterSets: (lookups: ParameterLookup[]) => { - expect(lookups).toStrictEqual([ParameterLookup.normalized('stream', '0', ['u2'])]); + getParameterSets: (lookups: ScopedParameterLookup[]) => { + expect(lookups).toStrictEqual([ScopedParameterLookup.direct(STREAM_0, ['u2'])]); return []; } }) @@ -331,9 +347,11 @@ describe('streams', () => { '1#stream|1["a"]' ]); - expect(desc.evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' })).toStrictEqual([ + const source = debugHydratedMergedSource(desc, hydrationParams); + + expect(source.evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' })).toStrictEqual([ { - lookup: ParameterLookup.normalized('stream', '0', ['b']), + lookup: ScopedParameterLookup.direct(STREAM_0, ['b']), bucketParameters: [ { result: 'a' @@ -341,7 +359,7 @@ describe('streams', () => { ] }, { - lookup: ParameterLookup.normalized('stream', '1', ['a']), + lookup: ScopedParameterLookup.direct(STREAM_1, ['a']), bucketParameters: [ { result: 'b' @@ -350,14 +368,14 @@ describe('streams', () => { } ]); - function getParameterSets(lookups: ParameterLookup[]) { + function getParameterSets(lookups: ScopedParameterLookup[]) { expect(lookups).toHaveLength(1); const [lookup] = lookups; if (lookup.values[1] == '0') { - expect(lookup).toStrictEqual(ParameterLookup.normalized('stream', '0', ['a'])); + expect(lookup).toStrictEqual(ScopedParameterLookup.direct(STREAM_0, ['a'])); return []; } else { - expect(lookup).toStrictEqual(ParameterLookup.normalized('stream', '1', ['a'])); + expect(lookup).toStrictEqual(ScopedParameterLookup.direct(STREAM_1, ['a'])); return [{ result: 'b' }]; } } @@ -397,7 +415,7 @@ describe('streams', () => { getParameterSets(lookups) { expect(lookups).toHaveLength(1); const [lookup] = lookups; - expect(lookup).toStrictEqual(ParameterLookup.normalized('stream', '0', ['a'])); + expect(lookup).toStrictEqual(ScopedParameterLookup.direct(STREAM_0, ['a'])); return [{ result: 'i1' }, { result: 'i2' }]; } }) @@ -435,11 +453,12 @@ describe('streams', () => { const desc = parseStream( 'SELECT * FROM comments WHERE tagged_users && (SELECT user_a FROM friends WHERE user_b = auth.user_id())' ); + const lookup = desc.parameterIndexLookupCreators[0]; - expect(desc.tableSyncsParameters(FRIENDS)).toBe(true); - expect(desc.evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' })).toStrictEqual([ + expect(lookup.tableSyncsParameters(FRIENDS)).toBe(true); + expect(lookup.evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' })).toStrictEqual([ { - lookup: ParameterLookup.normalized('stream', '0', ['b']), + lookup: UnscopedParameterLookup.normalized(['b']), bucketParameters: [ { result: 'a' @@ -456,7 +475,7 @@ describe('streams', () => { await queryBucketIds(desc, { token: { sub: 'user1' }, getParameterSets(lookups) { - expect(lookups).toStrictEqual([ParameterLookup.normalized('stream', '0', ['user1'])]); + expect(lookups).toStrictEqual([ScopedParameterLookup.direct(STREAM_0, ['user1'])]); return [{ result: 'issue_id' }]; } @@ -592,9 +611,15 @@ describe('streams', () => { 'select * from comments where NOT (issue_id not in (select id from issues where owner_id = auth.user_id()))' ); - expect(desc.evaluateParameterRow(ISSUES, { id: 'issue_id', owner_id: 'user1', name: 'name' })).toStrictEqual([ + expect( + desc.parameterIndexLookupCreators[0].evaluateParameterRow(ISSUES, { + id: 'issue_id', + owner_id: 'user1', + name: 'name' + }) + ).toStrictEqual([ { - lookup: ParameterLookup.normalized('stream', '0', ['user1']), + lookup: UnscopedParameterLookup.normalized(['user1']), bucketParameters: [ { result: 'issue_id' @@ -610,7 +635,7 @@ describe('streams', () => { await queryBucketIds(desc, { token: { sub: 'user1' }, getParameterSets(lookups) { - expect(lookups).toStrictEqual([ParameterLookup.normalized('stream', '0', ['user1'])]); + expect(lookups).toStrictEqual([ScopedParameterLookup.direct(STREAM_0, ['user1'])]); return [{ result: 'issue_id' }]; } @@ -665,7 +690,7 @@ describe('streams', () => { await queryBucketIds(desc, { token: { sub: 'user1' }, getParameterSets(lookups) { - expect(lookups).toStrictEqual([ParameterLookup.normalized('stream', '0', ['user1'])]); + expect(lookups).toStrictEqual([ScopedParameterLookup.direct(STREAM_0, ['user1'])]); return [{ result: 'issue_id' }]; } @@ -675,7 +700,7 @@ describe('streams', () => { await queryBucketIds(desc, { token: { sub: 'user1', is_admin: true }, getParameterSets(lookups) { - expect(lookups).toStrictEqual([ParameterLookup.normalized('stream', '0', ['user1'])]); + expect(lookups).toStrictEqual([ScopedParameterLookup.direct(STREAM_0, ['user1'])]); return [{ result: 'issue_id' }]; } @@ -715,13 +740,13 @@ describe('streams', () => { ); const row = { id: 'id', account_id: 'account_id' }; - expect(stream.tableSyncsData(accountMember)).toBeTruthy(); - expect(stream.tableSyncsParameters(accountMember)).toBeTruthy(); + expect(stream.dataSources[0].tableSyncsData(accountMember)).toBeTruthy(); + expect(stream.parameterIndexLookupCreators[0].tableSyncsParameters(accountMember)).toBeTruthy(); // Ensure lookup steps work. - expect(stream.evaluateParameterRow(accountMember, row)).toStrictEqual([ + expect(stream.parameterIndexLookupCreators[0].evaluateParameterRow(accountMember, row)).toStrictEqual([ { - lookup: ParameterLookup.normalized('account_member', '0', ['id']), + lookup: UnscopedParameterLookup.normalized(['id']), bucketParameters: [ { result: 'account_id' @@ -735,7 +760,9 @@ describe('streams', () => { token: { sub: 'id' }, parameters: {}, getParameterSets(lookups) { - expect(lookups).toStrictEqual([ParameterLookup.normalized('account_member', '0', ['id'])]); + expect(lookups).toStrictEqual([ + ScopedParameterLookup.direct({ lookupName: 'account_member', queryId: '0' }, ['id']) + ]); return [{ result: 'account_id' }]; } }) @@ -743,7 +770,7 @@ describe('streams', () => { // And that the data alias is respected for generated schemas. const outputSchema = {}; - stream.resolveResultSets(schema, outputSchema); + stream.dataSources[0].resolveResultSets(schema, outputSchema); expect(Object.keys(outputSchema)).toStrictEqual(['outer']); }); @@ -800,14 +827,14 @@ WHERE expect(evaluateBucketIds(desc, scene, { _id: 'scene', project: 'foo' })).toStrictEqual(['1#stream|0["foo"]']); expect( - desc.evaluateParameterRow(projectInvitation, { + desc.parameterIndexLookupCreators[0].evaluateParameterRow(projectInvitation, { project: 'foo', appliedTo: '[1,2]', status: 'CLAIMED' }) ).toStrictEqual([ { - lookup: ParameterLookup.normalized('stream', '0', [1n, 'foo']), + lookup: UnscopedParameterLookup.normalized([1n, 'foo']), bucketParameters: [ { result: 'foo' @@ -815,7 +842,7 @@ WHERE ] }, { - lookup: ParameterLookup.normalized('stream', '0', [2n, 'foo']), + lookup: UnscopedParameterLookup.normalized([2n, 'foo']), bucketParameters: [ { result: 'foo' @@ -829,13 +856,115 @@ WHERE token: { sub: 'user1', haystack_id: 1 }, parameters: { project: 'foo' }, getParameterSets(lookups) { - expect(lookups).toStrictEqual([ParameterLookup.normalized('stream', '0', [1n, 'foo'])]); + expect(lookups).toStrictEqual([ScopedParameterLookup.direct(STREAM_0, [1n, 'foo'])]); return [{ result: 'foo' }]; } }) ).toStrictEqual(['1#stream|0["foo"]']); }); }); + + test('variants with custom hydrationState', async () => { + // Convoluted example, but want to test specific variant usage. + // This test that bucket prefix and lookup scope mappings are correctly applied for each variant. + const desc = parseStream(` + SELECT * FROM comments WHERE + issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id()) OR -- stream|0 + issue_id IN (SELECT id FROM issues WHERE name = subscription.parameter('issue_name')) OR -- stream|1 + label = subscription.parameter('comment_label') OR -- stream|2 + auth.parameter('is_admin') -- stream|3 + `); + + const hydrationState: HydrationState = { + getBucketSourceScope(source) { + return { bucketPrefix: `${source.uniqueName}.test` }; + }, + getParameterIndexLookupScope(source) { + return { + lookupName: `${source.defaultLookupScope.lookupName}.test`, + queryId: `${source.defaultLookupScope.queryId}.test` + }; + } + }; + + const hydrated = debugHydratedMergedSource(desc, { hydrationState }); + + expect( + bucketIds(hydrated.evaluateRow({ sourceTable: COMMENTS, record: { id: 'c', issue_id: 'i1', label: 'l1' } })) + ).toStrictEqual(['stream|0.test["i1"]', 'stream|1.test["i1"]', 'stream|2.test["l1"]', 'stream|3.test[]']); + + expect( + hydrated.evaluateParameterRow(ISSUES, { + id: 'i1', + owner_id: 'u1', + name: 'myname' + }) + ).toStrictEqual([ + { + lookup: ScopedParameterLookup.direct({ lookupName: 'stream.test', queryId: '0.test' }, ['u1']), + bucketParameters: [ + { + result: 'i1' + } + ] + }, + + { + lookup: ScopedParameterLookup.direct({ lookupName: 'stream.test', queryId: '1.test' }, ['myname']), + bucketParameters: [ + { + result: 'i1' + } + ] + } + ]); + + expect( + hydrated.evaluateParameterRow(ISSUES, { + id: 'i1', + owner_id: 'u1' + }) + ).toStrictEqual([ + { + lookup: ScopedParameterLookup.direct({ lookupName: 'stream.test', queryId: '0.test' }, ['u1']), + bucketParameters: [ + { + result: 'i1' + } + ] + } + ]); + + function getParameterSets(lookups: ScopedParameterLookup[]) { + return lookups.flatMap((lookup) => { + if (JSON.stringify(lookup.values) == JSON.stringify(['stream.test', '1.test', null])) { + return []; + } else if (JSON.stringify(lookup.values) == JSON.stringify(['stream.test', '0.test', 'u1'])) { + return [{ result: 'i1' }]; + } else if (JSON.stringify(lookup.values) == JSON.stringify(['stream.test', '1.test', 'myname'])) { + return [{ result: 'i2' }]; + } else { + throw new Error(`Unexpected lookup: ${JSON.stringify(lookup.values)}`); + } + }); + } + + expect( + await queryBucketIds(desc, { + hydrationState, + token: { sub: 'u1', is_admin: false }, + getParameterSets + }) + ).toStrictEqual(['stream|2.test[null]', 'stream|0.test["i1"]']); + expect( + await queryBucketIds(desc, { + hydrationState, + token: { sub: 'u1', is_admin: true }, + parameters: { comment_label: 'l1', issue_name: 'myname' }, + getParameterSets + }) + ).toStrictEqual(['stream|2.test["l1"]', 'stream|3.test[]', 'stream|0.test["i1"]', 'stream|1.test["i2"]']); + }); }); const USERS = new TestSourceTable('users'); @@ -896,10 +1025,14 @@ const options: StreamParseOptions = { compatibility: new CompatibilityContext({ edition: CompatibilityEdition.SYNC_STREAMS }) }; -const bucketIdTransformer = SqlSyncRules.versionedBucketIdTransformer('1'); +const hydrationParams: CreateSourceParams = { hydrationState: versionedHydrationState(1) }; function evaluateBucketIds(stream: SyncStream, sourceTable: SourceTableInterface, record: SqliteRow) { - return stream.evaluateRow({ sourceTable, record, bucketIdTransformer }).map((r) => { + return bucketIds(debugHydratedMergedSource(stream, hydrationParams).evaluateRow({ sourceTable, record })); +} + +function bucketIds(result: EvaluationResult[]): string[] { + return result.map((r) => { if ('error' in r) { throw new Error(`Unexpected error evaluating row: ${r.error}`); } @@ -908,13 +1041,15 @@ function evaluateBucketIds(stream: SyncStream, sourceTable: SourceTableInterface }); } +interface TestQuerierOptions { + token?: Record; + parameters?: Record; + getParameterSets?: (lookups: ScopedParameterLookup[]) => SqliteJsonRow[]; + hydrationState?: HydrationState; +} async function createQueriers( stream: SyncStream, - options?: { - token?: Record; - parameters?: Record; - getParameterSets?: (lookups: ParameterLookup[]) => SqliteJsonRow[]; - } + options?: TestQuerierOptions ): Promise { const queriers: BucketParameterQuerier[] = []; const errors: QuerierError[] = []; @@ -929,27 +1064,22 @@ async function createQueriers( }, {} ), - streams: { [stream.name]: [{ opaque_id: 0, parameters: options?.parameters ?? null }] }, - bucketIdTransformer + streams: { [stream.name]: [{ opaque_id: 0, parameters: options?.parameters ?? null }] } }; - stream.pushBucketParameterQueriers(pending, querierOptions); + const hydrated = stream.hydrate( + options?.hydrationState ? { hydrationState: options.hydrationState } : hydrationParams + ); + hydrated.pushBucketParameterQueriers(pending, querierOptions); return { querier: mergeBucketParameterQueriers(queriers), errors }; } -async function queryBucketIds( - stream: SyncStream, - options?: { - token?: Record; - parameters?: Record; - getParameterSets?: (lookups: ParameterLookup[]) => SqliteJsonRow[]; - } -) { +async function queryBucketIds(stream: SyncStream, options?: TestQuerierOptions) { const { querier, errors } = await createQueriers(stream, options); expect(errors).toHaveLength(0); - async function getParameterSets(lookups: ParameterLookup[]): Promise { + async function getParameterSets(lookups: ScopedParameterLookup[]): Promise { const provided = options?.getParameterSets; if (provided) { return provided(lookups); diff --git a/packages/sync-rules/test/src/sync_rules.test.ts b/packages/sync-rules/test/src/sync_rules.test.ts index 9332418d5..80a73bce8 100644 --- a/packages/sync-rules/test/src/sync_rules.test.ts +++ b/packages/sync-rules/test/src/sync_rules.test.ts @@ -1,6 +1,9 @@ import { describe, expect, test } from 'vitest'; -import { ParameterLookup, SqlSyncRules } from '../../src/index.js'; +import { CreateSourceParams, ScopedParameterLookup, SqlSyncRules } from '../../src/index.js'; +import { DEFAULT_HYDRATION_STATE, HydrationState } from '../../src/HydrationState.js'; +import { SqlBucketDescriptor } from '../../src/SqlBucketDescriptor.js'; +import { StaticSqlParameterQuery } from '../../src/StaticSqlParameterQuery.js'; import { ASSETS, BASIC_SCHEMA, @@ -10,14 +13,14 @@ import { normalizeQuerierOptions, normalizeTokenParameters } from './util.js'; -import { SqlBucketDescriptor } from '../../src/SqlBucketDescriptor.js'; describe('sync rules', () => { - const bucketIdTransformer = SqlSyncRules.versionedBucketIdTransformer(''); + const hydrationParams: CreateSourceParams = { hydrationState: DEFAULT_HYDRATION_STATE }; test('parse empty sync rules', () => { const rules = SqlSyncRules.fromYaml('bucket_definitions: {}', PARSE_OPTIONS); - expect(rules.bucketSources).toEqual([]); + expect(rules.bucketParameterLookupSources).toEqual([]); + expect(rules.bucketDataSources).toEqual([]); }); test('parse global sync rules', () => { @@ -30,6 +33,7 @@ bucket_definitions: `, PARSE_OPTIONS ); + const hydrated = rules.hydrate(hydrationParams); const bucket = rules.bucketSources[0] as SqlBucketDescriptor; expect(bucket.name).toEqual('mybucket'); expect(bucket.bucketParameters).toEqual([]); @@ -37,9 +41,8 @@ bucket_definitions: expect(dataQuery.bucketParameters).toEqual([]); expect(dataQuery.columnOutputNames()).toEqual(['id', 'description']); expect( - rules.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', description: 'test' } }) ).toEqual([ @@ -53,8 +56,7 @@ bucket_definitions: bucket: 'mybucket[]' } ]); - expect(rules.hasDynamicBucketQueries()).toBe(false); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ + expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket[]', priority: 3 }], hasDynamicBuckets: false }); @@ -70,23 +72,24 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketSources[0] as SqlBucketDescriptor; - expect(bucket.bucketParameters).toEqual([]); - const param_query = bucket.globalParameterQueries[0]; + const hydrated = rules.hydrate(hydrationParams); + expect(rules.bucketParameterLookupSources).toEqual([]); // Internal API, subject to change - expect(param_query.filter!.lookupParameterValue(normalizeTokenParameters({ is_admin: 1n }))).toEqual(1n); - expect(param_query.filter!.lookupParameterValue(normalizeTokenParameters({ is_admin: 0n }))).toEqual(0n); + const parameterQuery = (rules.bucketSources[0] as SqlBucketDescriptor) + .globalParameterQueries[0] as StaticSqlParameterQuery; + expect(parameterQuery.filter!.lookupParameterValue(normalizeTokenParameters({ is_admin: 1n }))).toEqual(1n); + expect(parameterQuery.filter!.lookupParameterValue(normalizeTokenParameters({ is_admin: 0n }))).toEqual(0n); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: true })).querier).toMatchObject({ + expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: true })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket[]', priority: 3 }], hasDynamicBuckets: false }); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: false })).querier).toMatchObject({ + expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: false })).querier).toMatchObject({ staticBuckets: [], hasDynamicBuckets: false }); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ + expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ staticBuckets: [], hasDynamicBuckets: false }); @@ -102,17 +105,14 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketSources[0] as SqlBucketDescriptor; - expect(bucket.bucketParameters).toEqual([]); - const param_query = bucket.parameterQueries[0]; - expect(param_query.bucketParameters).toEqual([]); - expect(rules.evaluateParameterRow(USERS, { id: 'user1', is_admin: 1 })).toEqual([ + const hydrated = rules.hydrate(hydrationParams); + expect(hydrated.evaluateParameterRow(USERS, { id: 'user1', is_admin: 1 })).toEqual([ { bucketParameters: [{}], - lookup: ParameterLookup.normalized('mybucket', '1', ['user1']) + lookup: ScopedParameterLookup.direct({ lookupName: 'mybucket', queryId: '1' }, ['user1']) } ]); - expect(rules.evaluateParameterRow(USERS, { id: 'user1', is_admin: 0 })).toEqual([]); + expect(hydrated.evaluateParameterRow(USERS, { id: 'user1', is_admin: 0 })).toEqual([]); }); test('parse bucket with parameters', () => { @@ -126,23 +126,19 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketSources[0] as SqlBucketDescriptor; - expect(bucket.bucketParameters).toEqual(['user_id', 'device_id']); - const param_query = bucket.globalParameterQueries[0]; - expect(param_query.bucketParameters).toEqual(['user_id', 'device_id']); + const hydrated = rules.hydrate(hydrationParams); + const bucketData = rules.bucketDataSources[0]; + expect(bucketData.bucketParameters).toEqual(['user_id', 'device_id']); expect( - rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' }, { device_id: 'device1' })).querier - .staticBuckets + hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' }, { device_id: 'device1' })) + .querier.staticBuckets ).toEqual([ { bucket: 'mybucket["user1","device1"]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 } ]); - const data_query = bucket.dataQueries[0]; - expect(data_query.bucketParameters).toEqual(['user_id', 'device_id']); expect( - rules.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', description: 'test', user_id: 'user1', device_id: 'device1' } }) ).toEqual([ @@ -157,14 +153,83 @@ bucket_definitions: } ]); expect( - rules.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', description: 'test', user_id: 'user1', archived: 1, device_id: 'device1' } }) ).toEqual([]); }); + test('bucket with parameters with custom hydrationState', () => { + // "end-to-end" test with custom hydrationState. + // We don't test complex details here, but do cover bucket names and parameter lookup scope. + const rules = SqlSyncRules.fromYaml( + ` +config: + edition: 2 +bucket_definitions: + mybucket: + parameters: + - SELECT token_parameters.user_id as user_id + - SELECT users.id as user_id FROM users WHERE users.id = token_parameters.user_id AND users.is_admin + data: + - SELECT id, description FROM assets WHERE assets.user_id = bucket.user_id AND NOT assets.archived + `, + PARSE_OPTIONS + ); + const hydrationState: HydrationState = { + getBucketSourceScope(source) { + return { bucketPrefix: `${source.uniqueName}-test` }; + }, + getParameterIndexLookupScope(source) { + return { + lookupName: `${source.defaultLookupScope.lookupName}.test`, + queryId: `${source.defaultLookupScope.queryId}.test` + }; + } + }; + const hydrated = rules.hydrate({ hydrationState }); + const querier = hydrated.getBucketParameterQuerier( + normalizeQuerierOptions({ user_id: 'user1' }, { device_id: 'device1' }) + ); + expect(querier.errors).toEqual([]); + expect(querier.querier.staticBuckets).toEqual([ + { + bucket: 'mybucket-test["user1"]', + definition: 'mybucket', + inclusion_reasons: ['default'], + priority: 3 + } + ]); + expect(querier.querier.parameterQueryLookups).toEqual([ + ScopedParameterLookup.direct({ lookupName: 'mybucket.test', queryId: '2.test' }, ['user1']) + ]); + + expect(hydrated.evaluateParameterRow(USERS, { id: 'user1', is_admin: 1 })).toEqual([ + { + bucketParameters: [{ user_id: 'user1' }], + lookup: ScopedParameterLookup.direct({ lookupName: 'mybucket.test', queryId: '2.test' }, ['user1']) + } + ]); + + expect( + hydrated.evaluateRow({ + sourceTable: ASSETS, + record: { id: 'asset1', description: 'test', user_id: 'user1', device_id: 'device1' } + }) + ).toEqual([ + { + bucket: 'mybucket-test["user1"]', + id: 'asset1', + data: { + id: 'asset1', + description: 'test' + }, + table: 'assets' + } + ]); + }); + test('parse bucket with parameters and OR condition', () => { const rules = SqlSyncRules.fromYaml( ` @@ -176,20 +241,16 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketSources[0] as SqlBucketDescriptor; - expect(bucket.bucketParameters).toEqual(['user_id']); - const param_query = bucket.globalParameterQueries[0]; - expect(param_query.bucketParameters).toEqual(['user_id']); + const hydrated = rules.hydrate(hydrationParams); + const bucketData = rules.bucketDataSources[0]; + expect(bucketData.bucketParameters).toEqual(['user_id']); expect( - rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier.staticBuckets + hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier.staticBuckets ).toEqual([{ bucket: 'mybucket["user1"]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 }]); - const data_query = bucket.dataQueries[0]; - expect(data_query.bucketParameters).toEqual(['user_id']); expect( - rules.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', description: 'test', user_id: 'user1' } }) ).toEqual([ @@ -204,9 +265,8 @@ bucket_definitions: } ]); expect( - rules.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', description: 'test', owner_id: 'user1' } }) ).toEqual([ @@ -322,17 +382,17 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketSources[0] as SqlBucketDescriptor; - expect(bucket.bucketParameters).toEqual(['user_id']); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ + const hydrated = rules.hydrate(hydrationParams); + const bucketData = rules.bucketDataSources[0]; + expect(bucketData.bucketParameters).toEqual(['user_id']); + expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["USER1"]', priority: 3 }], hasDynamicBuckets: false }); expect( - rules.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', description: 'test', user_id: 'user1' } }) ).toEqual([ @@ -360,17 +420,17 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketSources[0] as SqlBucketDescriptor; - expect(bucket.bucketParameters).toEqual(['user_id']); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ + const hydrated = rules.hydrate(hydrationParams); + const bucketData = rules.bucketDataSources[0]; + expect(bucketData.bucketParameters).toEqual(['user_id']); + expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["USER1"]', priority: 3 }], hasDynamicBuckets: false }); expect( - rules.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', description: 'test', user_id: 'user1' } }) ).toEqual([ @@ -396,10 +456,10 @@ bucket_definitions: `, PARSE_OPTIONS ); + const hydrated = rules.hydrate(hydrationParams); expect( - rules.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', data: JSON.stringify({ count: 5, bool: true }) } }) ).toEqual([ @@ -430,11 +490,11 @@ bucket_definitions: `, PARSE_OPTIONS ); + const hydrated = rules.hydrate(hydrationParams); expect( - rules.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', description: 'test', @@ -475,11 +535,11 @@ bucket_definitions: `, PARSE_OPTIONS ); + const hydrated = rules.hydrate(hydrationParams); expect( - rules.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', description: 'test', role: 'admin' } }) ).toEqual([ @@ -497,9 +557,8 @@ bucket_definitions: ]); expect( - rules.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset2', description: 'test', role: 'normal' } }) ).toEqual([ @@ -538,9 +597,9 @@ bucket_definitions: } ]); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: true })).querier.staticBuckets).toEqual([ - { bucket: 'mybucket[1]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 } - ]); + expect( + hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: true })).querier.staticBuckets + ).toEqual([{ bucket: 'mybucket[1]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 }]); }); test('some math', () => { @@ -553,8 +612,9 @@ bucket_definitions: `, PARSE_OPTIONS ); + const hydrated = rules.hydrate(hydrationParams); - expect(rules.evaluateRow({ sourceTable: ASSETS, bucketIdTransformer, record: { id: 'asset1' } })).toEqual([ + expect(hydrated.evaluateRow({ sourceTable: ASSETS, record: { id: 'asset1' } })).toEqual([ { bucket: 'mybucket[]', id: 'asset1', @@ -580,14 +640,14 @@ bucket_definitions: `, PARSE_OPTIONS ); + const hydrated = rules.hydrate(hydrationParams); expect( - rules.getBucketParameterQuerier(normalizeQuerierOptions({ int1: 314, float1: 3.14, float2: 314 })).querier + hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ int1: 314, float1: 3.14, float2: 314 })).querier ).toMatchObject({ staticBuckets: [{ bucket: 'mybucket[314,3.14,314]', priority: 3 }] }); expect( - rules.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', int1: 314n, float1: 3.14, float2: 314 } }) ).toEqual([ @@ -613,7 +673,8 @@ bucket_definitions: PARSE_OPTIONS ); expect(rules.errors).toEqual([]); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'test' })).querier).toMatchObject({ + const hydrated = rules.hydrate(hydrationParams); + expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'test' })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["TEST"]', priority: 3 }], hasDynamicBuckets: false }); @@ -630,11 +691,11 @@ bucket_definitions: `, PARSE_OPTIONS ); + const hydrated = rules.hydrate(hydrationParams); expect( - rules.evaluateRow({ + hydrated.evaluateRow({ sourceTable: new TestSourceTable('assets_123'), - bucketIdTransformer, record: { client_id: 'asset1', description: 'test', archived: 0n, other_id: 'other1' } }) ).toEqual([ @@ -671,11 +732,11 @@ bucket_definitions: `, PARSE_OPTIONS ); + const hydrated = rules.hydrate(hydrationParams); expect( - rules.evaluateRow({ + hydrated.evaluateRow({ sourceTable: new TestSourceTable('assets_123'), - bucketIdTransformer, record: { client_id: 'asset1', description: 'test', archived: 0n, other_id: 'other1' } }) ).toEqual([ @@ -705,11 +766,11 @@ bucket_definitions: `, PARSE_OPTIONS ); + const hydrated = rules.hydrate(hydrationParams); expect( - rules.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', description: 'test', archived: 0n } }) ).toEqual([ @@ -741,11 +802,11 @@ bucket_definitions: `, PARSE_OPTIONS ); + const hydrated = rules.hydrate(hydrationParams); expect( - rules.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1' } }) ).toEqual([ @@ -870,7 +931,8 @@ bucket_definitions: expect(rules.errors).toEqual([]); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ + const hydrated = rules.hydrate(hydrationParams); + expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ staticBuckets: [ { bucket: 'highprio[]', priority: 0 }, { bucket: 'defaultprio[]', priority: 3 } @@ -895,7 +957,8 @@ bucket_definitions: expect(rules.errors).toEqual([]); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ + const hydrated = rules.hydrate(hydrationParams); + expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ staticBuckets: [ { bucket: 'highprio[]', priority: 0 }, { bucket: 'defaultprio[]', priority: 3 } @@ -956,17 +1019,18 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketSources[0] as SqlBucketDescriptor; - expect(bucket.bucketParameters).toEqual(['user_id']); - expect(rules.hasDynamicBucketQueries()).toBe(true); + const bucket1data = rules.bucketDataSources[0]; + expect(bucket1data.bucketParameters).toEqual(['user_id']); + + const hydrated = rules.hydrate(hydrationParams); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ + expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ hasDynamicBuckets: true, parameterQueryLookups: [ - ParameterLookup.normalized('mybucket', '2', ['user1']), - ParameterLookup.normalized('by_list', '1', ['user1']), + ScopedParameterLookup.direct({ lookupName: 'mybucket', queryId: '2' }, ['user1']), + ScopedParameterLookup.direct({ lookupName: 'by_list', queryId: '1' }, ['user1']), // These are not filtered out yet, due to how the lookups are structured internally - ParameterLookup.normalized('admin_only', '1', [1]) + ScopedParameterLookup.direct({ lookupName: 'admin_only', queryId: '1' }, [1]) ], staticBuckets: [ { diff --git a/packages/sync-rules/test/src/table_valued_function_queries.test.ts b/packages/sync-rules/test/src/table_valued_function_queries.test.ts index 2ead7b2be..306fb6935 100644 --- a/packages/sync-rules/test/src/table_valued_function_queries.test.ts +++ b/packages/sync-rules/test/src/table_valued_function_queries.test.ts @@ -7,7 +7,7 @@ import { SqlParameterQuery } from '../../src/index.js'; import { StaticSqlParameterQuery } from '../../src/StaticSqlParameterQuery.js'; -import { identityBucketTransformer, PARSE_OPTIONS } from './util.js'; +import { EMPTY_DATA_SOURCE, PARSE_OPTIONS } from './util.js'; describe('table-valued function queries', () => { test('json_each(array param)', function () { @@ -19,16 +19,16 @@ describe('table-valued function queries', () => { ...PARSE_OPTIONS, accept_potentially_dangerous_queries: true }, - '1' + '1', + EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['v']); expect( - query.getStaticBucketDescriptions( - new RequestParameters({ sub: '' }, { array: [1, 2, 3, null] }), - identityBucketTransformer - ) + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3, null] }), { + bucketPrefix: 'mybucket' + }) ).toEqual([ { bucket: 'mybucket[1]', priority: 3 }, { bucket: 'mybucket[2]', priority: 3 }, @@ -50,16 +50,16 @@ describe('table-valued function queries', () => { overrides: new Map([[CompatibilityOption.fixedJsonExtract, true]]) }) }, - '1' + '1', + EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['v']); expect( - query.getStaticBucketDescriptions( - new RequestParameters({ sub: '' }, { array: [1, 2, 3, null] }), - identityBucketTransformer - ) + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3, null] }), { + bucketPrefix: 'mybucket' + }) ).toEqual([ { bucket: 'mybucket[1]', priority: 3 }, { bucket: 'mybucket[2]', priority: 3 }, @@ -70,12 +70,20 @@ describe('table-valued function queries', () => { test('json_each(static string)', function () { const sql = `SELECT json_each.value as v FROM json_each('[1,2,3]')`; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['v']); expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), identityBucketTransformer) + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), { + bucketPrefix: 'mybucket' + }) ).toEqual([ { bucket: 'mybucket[1]', priority: 3 }, { bucket: 'mybucket[2]', priority: 3 }, @@ -85,12 +93,20 @@ describe('table-valued function queries', () => { test('json_each(null)', function () { const sql = `SELECT json_each.value as v FROM json_each(null)`; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['v']); expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), identityBucketTransformer) + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), { + bucketPrefix: 'mybucket' + }) ).toEqual([]); }); @@ -103,13 +119,16 @@ describe('table-valued function queries', () => { ...PARSE_OPTIONS, accept_potentially_dangerous_queries: true }, - '1' + '1', + EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['v']); expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), identityBucketTransformer) + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), { + bucketPrefix: 'mybucket' + }) ).toEqual([]); }); @@ -122,24 +141,35 @@ describe('table-valued function queries', () => { ...PARSE_OPTIONS, accept_potentially_dangerous_queries: true }, - '1' + '1', + EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['v']); expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), identityBucketTransformer) + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), { + bucketPrefix: 'mybucket' + }) ).toEqual([]); }); test('json_each on json_keys', function () { const sql = `SELECT value FROM json_each(json_keys('{"a": [], "b": 2, "c": null}'))`; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['value']); expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), identityBucketTransformer) + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), { + bucketPrefix: 'mybucket' + }) ).toEqual([ { bucket: 'mybucket["a"]', priority: 3 }, { bucket: 'mybucket["b"]', priority: 3 }, @@ -156,16 +186,16 @@ describe('table-valued function queries', () => { ...PARSE_OPTIONS, accept_potentially_dangerous_queries: true }, - '1' + '1', + EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['value']); expect( - query.getStaticBucketDescriptions( - new RequestParameters({ sub: '' }, { array: [1, 2, 3] }), - identityBucketTransformer - ) + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3] }), { + bucketPrefix: 'mybucket' + }) ).toEqual([ { bucket: 'mybucket[1]', priority: 3 }, { bucket: 'mybucket[2]', priority: 3 }, @@ -182,16 +212,16 @@ describe('table-valued function queries', () => { ...PARSE_OPTIONS, accept_potentially_dangerous_queries: true }, - '1' + '1', + EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['value']); expect( - query.getStaticBucketDescriptions( - new RequestParameters({ sub: '' }, { array: [1, 2, 3] }), - identityBucketTransformer - ) + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3] }), { + bucketPrefix: 'mybucket' + }) ).toEqual([ { bucket: 'mybucket[1]', priority: 3 }, { bucket: 'mybucket[2]', priority: 3 }, @@ -208,16 +238,16 @@ describe('table-valued function queries', () => { ...PARSE_OPTIONS, accept_potentially_dangerous_queries: true }, - '1' + '1', + EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['v']); expect( - query.getStaticBucketDescriptions( - new RequestParameters({ sub: '' }, { array: [1, 2, 3] }), - identityBucketTransformer - ) + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3] }), { + bucketPrefix: 'mybucket' + }) ).toEqual([ { bucket: 'mybucket[2]', priority: 3 }, { bucket: 'mybucket[3]', priority: 3 } @@ -234,7 +264,8 @@ describe('table-valued function queries', () => { ...PARSE_OPTIONS, accept_potentially_dangerous_queries: true }, - '1' + '1', + EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['project_id']); @@ -251,7 +282,9 @@ describe('table-valued function queries', () => { }, {} ), - identityBucketTransformer + { + bucketPrefix: 'mybucket' + } ) ).toEqual([{ bucket: 'mybucket[1]', priority: 3 }]); }); @@ -259,7 +292,13 @@ describe('table-valued function queries', () => { describe('dangerous queries', function () { function testDangerousQuery(sql: string) { test(sql, function () { - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toMatchObject([ { message: @@ -271,7 +310,13 @@ describe('table-valued function queries', () => { } function testSafeQuery(sql: string) { test(sql, function () { - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.usesDangerousRequestParameters).toEqual(false); }); diff --git a/packages/sync-rules/test/src/util.ts b/packages/sync-rules/test/src/util.ts index c86667e70..61478218a 100644 --- a/packages/sync-rules/test/src/util.ts +++ b/packages/sync-rules/test/src/util.ts @@ -1,13 +1,17 @@ import { + BucketDataSource, + ColumnDefinition, CompatibilityContext, - BucketIdTransformer, + CreateSourceParams, DEFAULT_TAG, GetQuerierOptions, RequestedStream, RequestJwtPayload, RequestParameters, + SourceSchema, SourceTableInterface, - StaticSchema + StaticSchema, + TablePattern } from '../../src/index.js'; export class TestSourceTable implements SourceTableInterface { @@ -69,18 +73,40 @@ export function normalizeTokenParameters( export function normalizeQuerierOptions( token_parameters: Record, user_parameters?: Record, - streams?: Record, - bucketIdTransformer?: BucketIdTransformer + streams?: Record ): GetQuerierOptions { const globalParameters = normalizeTokenParameters(token_parameters, user_parameters); return { globalParameters, hasDefaultStreams: true, - streams: streams ?? {}, - bucketIdTransformer: bucketIdTransformer ?? identityBucketTransformer + streams: streams ?? {} }; } export function identityBucketTransformer(id: string) { return id; } + +/** + * Empty data source that can be used for testing parameter queries, where most of the functionality here is not used. + */ +export const EMPTY_DATA_SOURCE: BucketDataSource = { + uniqueName: 'mybucket', + bucketParameters: [], + // These are not used in the tests. + getSourceTables: function (): Set { + return new Set(); + }, + evaluateRow(options) { + throw new Error('Function not implemented.'); + }, + tableSyncsData: function (table: SourceTableInterface): boolean { + throw new Error('Function not implemented.'); + }, + resolveResultSets: function (schema: SourceSchema, tables: Record>): void { + throw new Error('Function not implemented.'); + }, + debugWriteOutputTables: function (result: Record): void { + throw new Error('Function not implemented.'); + } +};