From 2c651ddabc306dd0785118126d57dd3176864fe9 Mon Sep 17 00:00:00 2001 From: Sean Oliver <882952+seanoliver@users.noreply.github.com> Date: Sat, 23 May 2026 09:30:25 -0700 Subject: [PATCH] fix(experiment): make dataApiRevokeOnCreateDefault flag reads shape-agnostic (#46289) --- .../useDataApiRevokeOnCreateDefault.test.ts | 89 +++++++++++++++++++ .../misc/useDataApiRevokeOnCreateDefault.ts | 29 +++++- .../[slug]/deploy-button/new-project.tsx | 7 +- apps/studio/pages/new/[slug].tsx | 22 +++-- packages/common/telemetry-constants.ts | 18 ++-- 5 files changed, 147 insertions(+), 18 deletions(-) diff --git a/apps/studio/hooks/misc/__tests__/useDataApiRevokeOnCreateDefault.test.ts b/apps/studio/hooks/misc/__tests__/useDataApiRevokeOnCreateDefault.test.ts index e39dc7a0937d5..2d74b58dda2bd 100644 --- a/apps/studio/hooks/misc/__tests__/useDataApiRevokeOnCreateDefault.test.ts +++ b/apps/studio/hooks/misc/__tests__/useDataApiRevokeOnCreateDefault.test.ts @@ -2,6 +2,7 @@ import { renderHook } from '@testing-library/react' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { + isInDataApiRevokeTreatment, useDataApiRevokeOnCreateDefaultEnabled, useTrackDefaultPrivilegesExposure, } from '../useDataApiRevokeOnCreateDefault' @@ -25,6 +26,33 @@ vi.mock('@/lib/telemetry/track', () => ({ useTrack: vi.fn(), })) +describe('isInDataApiRevokeTreatment', () => { + it('returns true for boolean true (current rollout shape)', () => { + expect(isInDataApiRevokeTreatment(true)).toBe(true) + }) + + it("returns true for the 'test' variant (future multivariate shape)", () => { + expect(isInDataApiRevokeTreatment('test')).toBe(true) + }) + + it('returns false for boolean false', () => { + expect(isInDataApiRevokeTreatment(false)).toBe(false) + }) + + it("returns false for the 'control' variant", () => { + expect(isInDataApiRevokeTreatment('control')).toBe(false) + }) + + it('returns false for undefined (flag not resolved)', () => { + expect(isInDataApiRevokeTreatment(undefined)).toBe(false) + }) + + it('returns false for unrelated string values', () => { + expect(isInDataApiRevokeTreatment('something-else')).toBe(false) + expect(isInDataApiRevokeTreatment('')).toBe(false) + }) +}) + describe('useDataApiRevokeOnCreateDefaultEnabled', () => { afterEach(() => { vi.restoreAllMocks() @@ -49,6 +77,18 @@ describe('useDataApiRevokeOnCreateDefaultEnabled', () => { expect(result.current).toBe(true) }) + it("returns true when the PostHog flag is the 'test' variant string", () => { + vi.mocked(usePHFlag).mockReturnValue('test') + const { result } = renderHook(() => useDataApiRevokeOnCreateDefaultEnabled()) + expect(result.current).toBe(true) + }) + + it("returns false when the PostHog flag is the 'control' variant string", () => { + vi.mocked(usePHFlag).mockReturnValue('control') + const { result } = renderHook(() => useDataApiRevokeOnCreateDefaultEnabled()) + expect(result.current).toBe(false) + }) + it('returns false in test env regardless of flag value', () => { vi.mocked(constants, { partial: true }).IS_TEST_ENV = true vi.mocked(usePHFlag).mockReturnValue(true) @@ -252,4 +292,53 @@ describe('useTrackDefaultPrivilegesExposure', () => { undefined ) }) + + // The next two tests cover the future multivariate flag shape (GROWTH-877). + // Today the flag returns boolean true/false; post-migration it returns the + // variant string. The convergence gate must derive the expected default from + // the variant, not from `!flag` directly — `!'control'` is false (truthy + // string negation), which would have set the wrong expected default and + // either skipped or mis-fired the exposure for control-arm users. + + it("fires for the 'test' variant with the correct convergence default (treatment)", () => { + vi.mocked(usePHFlag).mockReturnValue('test') + renderHook(() => + useTrackDefaultPrivilegesExposure({ + surface: 'main', + dataApiDefaultPrivileges: false, // expected default for treatment + hasUserModified: false, + }) + ) + expect(track).toHaveBeenCalledTimes(1) + expect(track).toHaveBeenCalledWith( + 'project_creation_default_privileges_exposed', + { + surface: 'main', + dataApiDefaultPrivileges: false, + dataApiRevokeOnCreateDefaultEnabled: 'test', + }, + undefined + ) + }) + + it("fires for the 'control' variant with the correct convergence default (legacy)", () => { + vi.mocked(usePHFlag).mockReturnValue('control') + renderHook(() => + useTrackDefaultPrivilegesExposure({ + surface: 'main', + dataApiDefaultPrivileges: true, // expected default for non-treatment + hasUserModified: false, + }) + ) + expect(track).toHaveBeenCalledTimes(1) + expect(track).toHaveBeenCalledWith( + 'project_creation_default_privileges_exposed', + { + surface: 'main', + dataApiDefaultPrivileges: true, + dataApiRevokeOnCreateDefaultEnabled: 'control', + }, + undefined + ) + }) }) diff --git a/apps/studio/hooks/misc/useDataApiRevokeOnCreateDefault.ts b/apps/studio/hooks/misc/useDataApiRevokeOnCreateDefault.ts index d180a13f7c090..8ebe9174e0c08 100644 --- a/apps/studio/hooks/misc/useDataApiRevokeOnCreateDefault.ts +++ b/apps/studio/hooks/misc/useDataApiRevokeOnCreateDefault.ts @@ -4,6 +4,26 @@ import { usePHFlag } from '../ui/useFlag' import { IS_TEST_ENV } from '@/lib/constants' import { useTrack } from '@/lib/telemetry/track' +/** + * Returns true iff the user is assigned to the treatment arm of the + * dataApiRevokeOnCreateDefault experiment. Shape-agnostic across the current + * boolean rollout config and a future multivariate experiment with named + * variants. See GROWTH-877 for the migration plan. + * + * Accepts: + * - `true` → treatment (current boolean shape) + * - `'test'` → treatment (future multivariate shape) + * - anything else (`false`, `'control'`, `null`, `undefined`) → not treatment + * + * Use this everywhere the flag's value is read so the PostHog config can + * migrate to multivariate without a coordinated frontend deploy. + */ +export const isInDataApiRevokeTreatment = (flag: boolean | string | undefined): boolean => { + if (flag === true) return true + if (flag === 'test') return true + return false +} + /** * Controls the default state of the "Automatically expose new tables" * checkbox at project creation. When the flag is on, the checkbox defaults @@ -11,7 +31,7 @@ import { useTrack } from '@/lib/telemetry/track' * to checked (current behaviour — default grants remain). */ export const useDataApiRevokeOnCreateDefaultEnabled = (): boolean => { - const flag = usePHFlag('dataApiRevokeOnCreateDefault') + const flag = usePHFlag('dataApiRevokeOnCreateDefault') // Preserve current behaviour (default grants remain) in tests so existing // E2E flows don't change silently. Tests that need the revoke-default path @@ -20,7 +40,7 @@ export const useDataApiRevokeOnCreateDefaultEnabled = (): boolean => { return false } - return !!flag + return isInDataApiRevokeTreatment(flag) } type DefaultPrivilegesExposureOptions = @@ -48,7 +68,7 @@ type DefaultPrivilegesExposureOptions = */ export const useTrackDefaultPrivilegesExposure = (options: DefaultPrivilegesExposureOptions) => { const track = useTrack() - const flag = usePHFlag('dataApiRevokeOnCreateDefault') + const flag = usePHFlag('dataApiRevokeOnCreateDefault') const hasTracked = useRef(false) const { surface, dataApiDefaultPrivileges, hasUserModified } = options @@ -59,7 +79,8 @@ export const useTrackDefaultPrivilegesExposure = (options: DefaultPrivilegesExpo if (flag === undefined) return if (surface === 'vercel' && !orgSlug) return // Gate on form-flag convergence unless the user explicitly dirtied the field. - if (!hasUserModified && dataApiDefaultPrivileges !== !flag) return + const expectedDefault = !isInDataApiRevokeTreatment(flag) + if (!hasUserModified && dataApiDefaultPrivileges !== expectedDefault) return hasTracked.current = true track( 'project_creation_default_privileges_exposed', diff --git a/apps/studio/pages/integrations/vercel/[slug]/deploy-button/new-project.tsx b/apps/studio/pages/integrations/vercel/[slug]/deploy-button/new-project.tsx index f1cdab561e38d..02a69b9ee899d 100644 --- a/apps/studio/pages/integrations/vercel/[slug]/deploy-button/new-project.tsx +++ b/apps/studio/pages/integrations/vercel/[slug]/deploy-button/new-project.tsx @@ -28,6 +28,7 @@ import { useVercelProjectsQuery } from '@/data/integrations/integrations-vercel- import { useOrganizationsQuery } from '@/data/organizations/organizations-query' import { useProjectCreateMutation } from '@/data/projects/project-create-mutation' import { + isInDataApiRevokeTreatment, useDataApiRevokeOnCreateDefaultEnabled, useTrackDefaultPrivilegesExposure, } from '@/hooks/misc/useDataApiRevokeOnCreateDefault' @@ -82,7 +83,9 @@ const CreateProject = () => { const track = useTrack() const snapshot = useIntegrationInstallationSnapshot() const isDataApiRevokeOnCreateDefault = useDataApiRevokeOnCreateDefaultEnabled() - const dataApiRevokeOnCreateDefaultFlag = usePHFlag('dataApiRevokeOnCreateDefault') + const dataApiRevokeOnCreateDefaultFlag = usePHFlag( + 'dataApiRevokeOnCreateDefault' + ) const [dataApiDefaultPrivileges, setDataApiDefaultPrivileges] = useState( !isDataApiRevokeOnCreateDefault ) @@ -91,7 +94,7 @@ const CreateProject = () => { useEffect(() => { if (dataApiRevokeOnCreateDefaultFlag === undefined) return if (hasUserModifiedDataApiDefaultPrivileges.current) return - setDataApiDefaultPrivileges(!dataApiRevokeOnCreateDefaultFlag) + setDataApiDefaultPrivileges(!isInDataApiRevokeTreatment(dataApiRevokeOnCreateDefaultFlag)) }, [dataApiRevokeOnCreateDefaultFlag]) const { slug, next, currentProjectId: foreignProjectId, externalId } = useParams() diff --git a/apps/studio/pages/new/[slug].tsx b/apps/studio/pages/new/[slug].tsx index fc4287d2b50aa..266d8a0e62dac 100644 --- a/apps/studio/pages/new/[slug].tsx +++ b/apps/studio/pages/new/[slug].tsx @@ -58,7 +58,10 @@ import { import { useCustomContent } from '@/hooks/custom-content/useCustomContent' import { useCheckEntitlements } from '@/hooks/misc/useCheckEntitlements' import { useAsyncCheckPermissions } from '@/hooks/misc/useCheckPermissions' -import { useDataApiRevokeOnCreateDefaultEnabled } from '@/hooks/misc/useDataApiRevokeOnCreateDefault' +import { + isInDataApiRevokeTreatment, + useDataApiRevokeOnCreateDefaultEnabled, +} from '@/hooks/misc/useDataApiRevokeOnCreateDefault' import { useIsFeatureEnabled } from '@/hooks/misc/useIsFeatureEnabled' import { useLocalStorageQuery } from '@/hooks/misc/useLocalStorage' import { useSelectedOrganizationQuery } from '@/hooks/misc/useSelectedOrganization' @@ -108,8 +111,11 @@ const Wizard: NextPageWithLayout = () => { // Read the raw flag for telemetry — coerce-undefined-to-false would record false for // users whose flags haven't loaded yet. The raw value preserves undefined (omitted from - // PostHog) so we only record true/false when the flag is resolved. - const dataApiRevokeOnCreateDefaultFlag = usePHFlag('dataApiRevokeOnCreateDefault') + // PostHog) so we only record an actual value (boolean true/false, or a variant string + // like 'test'/'control' post-multivariate migration) once the flag has resolved. + const dataApiRevokeOnCreateDefaultFlag = usePHFlag( + 'dataApiRevokeOnCreateDefault' + ) const isDataApiRevokeOnCreateDefault = useDataApiRevokeOnCreateDefaultEnabled() const isNotOnHigherPlan = !['team', 'enterprise', 'platform'].includes(currentOrg?.plan.id ?? '') @@ -174,9 +180,13 @@ const Wizard: NextPageWithLayout = () => { useEffect(() => { if (dataApiRevokeOnCreateDefaultFlag === undefined) return if (isDataApiDefaultPrivilegesDirty) return - setValue('dataApiDefaultPrivileges', !dataApiRevokeOnCreateDefaultFlag, { - shouldDirty: false, - }) + setValue( + 'dataApiDefaultPrivileges', + !isInDataApiRevokeTreatment(dataApiRevokeOnCreateDefaultFlag), + { + shouldDirty: false, + } + ) }, [dataApiRevokeOnCreateDefaultFlag, isDataApiDefaultPrivilegesDirty, setValue]) // [Charis] Since the form is updated in a useEffect, there is an edge case diff --git a/packages/common/telemetry-constants.ts b/packages/common/telemetry-constants.ts index 23fd8cebc16ca..5c092e8084fc5 100644 --- a/packages/common/telemetry-constants.ts +++ b/packages/common/telemetry-constants.ts @@ -319,10 +319,13 @@ export interface ProjectCreationDefaultPrivilegesExposedEvent { dataApiDefaultPrivileges: boolean /** * Raw value of the dataApiRevokeOnCreateDefault PostHog flag at exposure time. - * true = revoke cohort (checkbox defaulted to unchecked) - * false = control cohort (checkbox defaulted to checked) + * Accepts boolean (current rollout shape) or string (post-multivariate-migration + * variant name, e.g. 'test' / 'control'). See GROWTH-877 for the migration plan. + * true | 'test' = revoke cohort (checkbox defaulted to unchecked) + * false = outside the rollout (checkbox defaulted to checked) + * 'control' = in-experiment control arm (checkbox defaulted to checked) */ - dataApiRevokeOnCreateDefaultEnabled: boolean + dataApiRevokeOnCreateDefaultEnabled: boolean | string } groups: Omit } @@ -381,14 +384,17 @@ export interface ProjectCreationSimpleVersionSubmittedEvent { */ dataApiDefaultPrivilegesGranted?: boolean /** - * Whether the dataApiRevokeOnCreateDefault PostHog flag was enabled for this user. + * Raw value of the dataApiRevokeOnCreateDefault PostHog flag at submission time. * Controls only the default checkbox state of "Automatically expose new tables and functions" * at project creation. Tracking it lets us correlate flag cohort with user choice. - * true = user is in the staged rollout cohort (checkbox defaulted to unchecked) + * Accepts boolean (current rollout shape) or string (post-multivariate-migration + * variant name, e.g. 'test' / 'control'). See GROWTH-877 for the migration plan. + * true | 'test' = user is in the treatment arm (checkbox defaulted to unchecked) * false = user is outside the rollout (checkbox defaulted to checked) + * 'control' = in-experiment control arm (checkbox defaulted to checked) * omitted = PostHog flags had not loaded at the time of project creation */ - dataApiRevokeOnCreateDefaultEnabled?: boolean + dataApiRevokeOnCreateDefaultEnabled?: boolean | string } groups: TelemetryGroups }