diff --git a/Makefile b/Makefile index f8d5cc4d..3a18b9af 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,7 @@ vpath % src src/compat/$(PGVER) DATA = $(wildcard sql/$(EXTENSION)*--*.sql) SRCS := $(wildcard src/*.c) \ $(wildcard src/compat/$(PGVER)/*.c) + OBJS = $(filter-out src/spock_output.o, $(SRCS:.c=.o)) PG_CPPFLAGS += -I$(libpq_srcdir) \ @@ -64,7 +65,7 @@ REGRESS = preseed infofuncs init_fail init preseed_check basic conflict_secondar row_filter_sampling att_list column_filter apply_delay alter_options \ extended node_origin_cascade multiple_upstreams tuple_origin autoddl \ sync_event sync_table generated_columns spill_transaction read_only \ - resolutions_retention drop + resolutions_retention version_guard drop # The following test cases are disabled while developing. # diff --git a/docs/conflicts.md b/docs/conflicts.md index 554e2673..585005a0 100644 --- a/docs/conflicts.md +++ b/docs/conflicts.md @@ -33,24 +33,47 @@ conflicts with the following logic: Note that on a conflicting transaction, the delta-apply column will be correctly calculated and applied. -To make a column a conflict-free delta-apply column, ensuring that the value -replicated is the delta of the committed changes (the old value plus or -minus any new value) to a given record, you need to apply the following -settings to the column: `log_old_value=true, -delta_apply_function=spock.delta_apply`. For example: +To make a column a conflict-free delta-apply column, ensuring that the +value replicated is the delta of the committed changes (the old value +plus or minus any new value) to a given record, attach a SECURITY LABEL +to the column via the `spock.delta_apply()` helper: ```sql -ALTER TABLE pgbench_accounts ALTER COLUMN abalance - SET (log_old_value=true, delta_apply_function=spock.delta_apply); -ALTER TABLE pgbench_branches ALTER COLUMN bbalance - SET (log_old_value=true, delta_apply_function=spock.delta_apply); -ALTER TABLE pgbench_tellers ALTER COLUMN tbalance - SET (log_old_value=true, delta_apply_function=spock.delta_apply); +SELECT spock.delta_apply('pgbench_accounts'::regclass, 'abalance'); +SELECT spock.delta_apply('pgbench_branches'::regclass, 'bbalance'); +SELECT spock.delta_apply('pgbench_tellers'::regclass, 'tbalance'); ``` -As a special safety-valve feature, if you ever need to re-set a -`log_old_value` column you can temporarily alter the column to -`log_old_value` is `false`. +To remove the marker, pass `to_drop => true`: + +```sql +SELECT spock.delta_apply('pgbench_accounts'::regclass, 'abalance', to_drop => true); +``` + +Under the hood, `spock.delta_apply()` writes a row into `pg_seclabel` +with `provider = 'spock'` and `label = 'spock.delta_apply'`. The +binary-upgrade compatibility shim that translates legacy spock 5.x +reloptions during `pg_upgrade` writes the same canonical label, so +operators can audit the catalog uniformly: + +```sql +SELECT * FROM pg_seclabel + WHERE provider = 'spock' AND label = 'spock.delta_apply'; +``` + +### Upgrading from spock 5.x + +Spock 5.x recorded the same intent as a pair of per-attribute reloptions +(`log_old_value=true, delta_apply_function=spock.delta_apply`). During +`pg_upgrade`, the binary-upgrade compatibility shim translates those +reloptions into the new `SECURITY LABEL` form automatically. Look for +`NOTICE: spock: rewrote ALTER TABLE … ALTER COLUMN … legacy options to +SECURITY LABEL` lines in `pg_upgrade.log` to audit the translation. +After the upgrade, `SELECT * FROM pg_seclabel WHERE provider = 'spock' +AND label = 'spock.delta_apply'` is the authoritative list of delta-apply +columns. See +[Binary-upgrade compatibility shims](internals-doc/binary-upgrade-compat-shim.md) +for the full design. ### Conflict Configuration Options diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index f65f1abc..15c6f491 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -165,8 +165,7 @@ ALTER TABLE table_name ALTER COLUMN column_name SET NOT NULL; Then configure the Delta-Apply column with the following command: ```sql -ALTER TABLE table_name ALTER COLUMN column_name - SET (log_old_value=true, delta_apply_function=spock.delta_apply); +SELECT spock.delta_apply('table_name'::regclass, 'column_name'); ``` ## Configuration Issues diff --git a/include/spock.h b/include/spock.h index 2d96941f..49ffb7d4 100644 --- a/include/spock.h +++ b/include/spock.h @@ -133,4 +133,7 @@ VALGRIND_PRINTF(const char *format,...) extern void spock_init_failover_slot(void); +/* lives in src/spock_bucompat_5x.c -- used only under IsBinaryUpgrade */ +extern void register_spock_compat_5x(void); + #endif /* SPOCK_H */ diff --git a/patches/15/pg15-000-spock-patchset-version.diff b/patches/15/pg15-000-spock-patchset-version.diff new file mode 100644 index 00000000..99f97ab8 --- /dev/null +++ b/patches/15/pg15-000-spock-patchset-version.diff @@ -0,0 +1,32 @@ +Spock core-patchset: export patchset version via miscadmin.h and globals.c. + +Adds SPOCK_CORE_PATCHSET_VERSION (compile-time constant) and +SpockCorePatchsetVersion (runtime global) to the standard places +PostgreSQL already uses for server-wide state. No new files. + +--- a/src/include/miscadmin.h ++++ b/src/include/miscadmin.h +@@ -498,4 +498,11 @@ + /* in executor/nodeHash.c */ + extern size_t get_hash_memory_limit(void); + ++/* ++ * Spock core-patchset identity. Bump the version when the patchset ++ * changes in a way visible to the extension binary. ++ */ ++#define SPOCK_CORE_PATCHSET_VERSION 1 ++extern PGDLLIMPORT int SpockCorePatchsetVersion; ++ + #endif /* MISCADMIN_H */ +--- a/src/backend/utils/init/globals.c ++++ b/src/backend/utils/init/globals.c +@@ -114,6 +114,9 @@ + bool IsBinaryUpgrade = false; + bool IsBackgroundWorker = false; + ++/* Spock core-patchset identity. */ ++int SpockCorePatchsetVersion = SPOCK_CORE_PATCHSET_VERSION; ++ + bool ExitOnAnyError = false; + + int DateStyle = USE_ISO_DATES; diff --git a/patches/16/pg16-000-spock-patchset-version.diff b/patches/16/pg16-000-spock-patchset-version.diff new file mode 100644 index 00000000..224d34d8 --- /dev/null +++ b/patches/16/pg16-000-spock-patchset-version.diff @@ -0,0 +1,32 @@ +Spock core-patchset: export patchset version via miscadmin.h and globals.c. + +Adds SPOCK_CORE_PATCHSET_VERSION (compile-time constant) and +SpockCorePatchsetVersion (runtime global) to the standard places +PostgreSQL already uses for server-wide state. No new files. + +--- a/src/include/miscadmin.h ++++ b/src/include/miscadmin.h +@@ -510,4 +510,11 @@ + /* in executor/nodeHash.c */ + extern size_t get_hash_memory_limit(void); + ++/* ++ * Spock core-patchset identity. Bump the version when the patchset ++ * changes in a way visible to the extension binary. ++ */ ++#define SPOCK_CORE_PATCHSET_VERSION 1 ++extern PGDLLIMPORT int SpockCorePatchsetVersion; ++ + #endif /* MISCADMIN_H */ +--- a/src/backend/utils/init/globals.c ++++ b/src/backend/utils/init/globals.c +@@ -114,6 +114,9 @@ + bool IsBinaryUpgrade = false; + bool IsBackgroundWorker = false; + ++/* Spock core-patchset identity. */ ++int SpockCorePatchsetVersion = SPOCK_CORE_PATCHSET_VERSION; ++ + bool ExitOnAnyError = false; + + int DateStyle = USE_ISO_DATES; diff --git a/patches/17/pg17-000-spock-patchset-version.diff b/patches/17/pg17-000-spock-patchset-version.diff new file mode 100644 index 00000000..0b186060 --- /dev/null +++ b/patches/17/pg17-000-spock-patchset-version.diff @@ -0,0 +1,32 @@ +Spock core-patchset: export patchset version via miscadmin.h and globals.c. + +Adds SPOCK_CORE_PATCHSET_VERSION (compile-time constant) and +SpockCorePatchsetVersion (runtime global) to the standard places +PostgreSQL already uses for server-wide state. No new files. + +--- a/src/include/miscadmin.h ++++ b/src/include/miscadmin.h +@@ -525,4 +525,11 @@ + /* in executor/nodeHash.c */ + extern size_t get_hash_memory_limit(void); + ++/* ++ * Spock core-patchset identity. Bump the version when the patchset ++ * changes in a way visible to the extension binary. ++ */ ++#define SPOCK_CORE_PATCHSET_VERSION 1 ++extern PGDLLIMPORT int SpockCorePatchsetVersion; ++ + #endif /* MISCADMIN_H */ +--- a/src/backend/utils/init/globals.c ++++ b/src/backend/utils/init/globals.c +@@ -117,6 +117,9 @@ + bool IsUnderPostmaster = false; + bool IsBinaryUpgrade = false; + ++/* Spock core-patchset identity. */ ++int SpockCorePatchsetVersion = SPOCK_CORE_PATCHSET_VERSION; ++ + bool ExitOnAnyError = false; + + int DateStyle = USE_ISO_DATES; diff --git a/patches/18/pg18-000-spock-patchset-version.diff b/patches/18/pg18-000-spock-patchset-version.diff new file mode 100644 index 00000000..79bf9aaf --- /dev/null +++ b/patches/18/pg18-000-spock-patchset-version.diff @@ -0,0 +1,32 @@ +Spock core-patchset: export patchset version via miscadmin.h and globals.c. + +Adds SPOCK_CORE_PATCHSET_VERSION (compile-time constant) and +SpockCorePatchsetVersion (runtime global) to the standard places +PostgreSQL already uses for server-wide state. No new files. + +--- a/src/include/miscadmin.h ++++ b/src/include/miscadmin.h +@@ -540,4 +540,11 @@ + /* in executor/nodeHash.c */ + extern size_t get_hash_memory_limit(void); + ++/* ++ * Spock core-patchset identity. Bump the version when the patchset ++ * changes in a way visible to the extension binary. ++ */ ++#define SPOCK_CORE_PATCHSET_VERSION 1 ++extern PGDLLIMPORT int SpockCorePatchsetVersion; ++ + #endif /* MISCADMIN_H */ +--- a/src/backend/utils/init/globals.c ++++ b/src/backend/utils/init/globals.c +@@ -120,6 +120,9 @@ + bool IsUnderPostmaster = false; + bool IsBinaryUpgrade = false; + ++/* Spock core-patchset identity. */ ++int SpockCorePatchsetVersion = SPOCK_CORE_PATCHSET_VERSION; ++ + bool ExitOnAnyError = false; + + int DateStyle = USE_ISO_DATES; diff --git a/sql/spock--5.0.0.sql b/sql/spock--5.0.0.sql new file mode 100644 index 00000000..9ccad9ca --- /dev/null +++ b/sql/spock--5.0.0.sql @@ -0,0 +1,790 @@ +\echo Use "CREATE EXTENSION spock" to load this file. \quit + +CREATE TABLE spock.node ( + node_id oid NOT NULL PRIMARY KEY, + node_name name NOT NULL UNIQUE, + location text, + country text, + info jsonb +) WITH (user_catalog_table=true); + +CREATE TABLE spock.node_interface ( + if_id oid NOT NULL PRIMARY KEY, + if_name name NOT NULL, -- default same as node name + if_nodeid oid REFERENCES node(node_id) ON UPDATE CASCADE, + if_dsn text NOT NULL, + UNIQUE (if_nodeid, if_name) +); + +CREATE TABLE spock.local_node ( + node_id oid PRIMARY KEY REFERENCES node(node_id), + node_local_interface oid NOT NULL REFERENCES node_interface(if_id) +); + +CREATE TABLE spock.subscription ( + sub_id oid NOT NULL PRIMARY KEY, + sub_name name NOT NULL UNIQUE, + sub_origin oid NOT NULL REFERENCES node(node_id) ON UPDATE CASCADE, + sub_target oid NOT NULL REFERENCES node(node_id) ON UPDATE CASCADE, + sub_origin_if oid NOT NULL REFERENCES node_interface(if_id), + sub_target_if oid NOT NULL REFERENCES node_interface(if_id), + sub_enabled boolean NOT NULL DEFAULT true, + sub_slot_name name NOT NULL, + sub_replication_sets text[], + sub_forward_origins text[], + sub_apply_delay interval NOT NULL DEFAULT '0', + sub_force_text_transfer boolean NOT NULL DEFAULT 'f', + sub_skip_lsn pg_lsn NOT NULL DEFAULT '0/0' +); + +CREATE TABLE spock.local_sync_status ( + sync_kind "char" NOT NULL CHECK (sync_kind IN ('i', 's', 'd', 'f')), + sync_subid oid NOT NULL REFERENCES spock.subscription(sub_id), + sync_nspname name, + sync_relname name, + sync_status "char" NOT NULL, + sync_statuslsn pg_lsn NOT NULL, + UNIQUE (sync_subid, sync_nspname, sync_relname) +); + +CREATE TABLE spock.exception_log ( + remote_origin oid NOT NULL, + remote_commit_ts timestamptz NOT NULL, + command_counter integer NOT NULL, + retry_errored_at timestamptz NOT NULL, + remote_xid bigint NOT NULL, + local_origin oid, + local_commit_ts timestamptz, + table_schema text, + table_name text, + operation text, + local_tup jsonb, + remote_old_tup jsonb, + remote_new_tup jsonb, + ddl_statement text, + ddl_user text, + error_message text NOT NULL, + PRIMARY KEY(remote_origin, remote_commit_ts, + command_counter, retry_errored_at) +) WITH (user_catalog_table=true); + +CREATE TABLE spock.exception_status ( + remote_origin oid NOT NULL, + remote_commit_ts timestamptz NOT NULL, + retry_errored_at timestamptz NOT NULL, + remote_xid bigint NOT NULL, + status text NOT NULL, + resolved_at timestamptz, + resolution_details jsonb, + PRIMARY KEY(remote_origin, remote_commit_ts, retry_errored_at) +) WITH (user_catalog_table=true); + +CREATE TABLE spock.exception_status_detail ( + remote_origin oid NOT NULL, + remote_commit_ts timestamptz NOT NULL, + command_counter integer NOT NULL, + retry_errored_at timestamptz NOT NULL, + remote_xid bigint NOT NULL, + status text NOT NULL, + resolved_at timestamptz, + resolution_details jsonb, + PRIMARY KEY(remote_origin, remote_commit_ts, + command_counter, retry_errored_at), + FOREIGN KEY(remote_origin, remote_commit_ts, retry_errored_at) + REFERENCES spock.exception_status +) WITH (user_catalog_table=true); + +CREATE TABLE spock.progress ( + node_id oid NOT NULL, + remote_node_id oid NOT NULL, + remote_commit_ts timestamptz NOT NULL, + remote_lsn pg_lsn NOT NULL, + remote_insert_lsn pg_lsn NOT NULL, + last_updated_ts timestamptz NOT NULL, + updated_by_decode bool NOT NULL, + PRIMARY KEY(node_id, remote_node_id) +) WITH (fillfactor=50); + +CREATE FUNCTION spock.node_create(node_name name, dsn text, + location text DEFAULT NULL, country text DEFAULT NULL, + info jsonb DEFAULT NULL) +RETURNS oid CALLED ON NULL INPUT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_create_node'; +CREATE FUNCTION spock.node_drop(node_name name, ifexists boolean DEFAULT false) +RETURNS boolean STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_drop_node'; + +CREATE FUNCTION spock.node_add_interface(node_name name, interface_name name, dsn text) +RETURNS oid STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_alter_node_add_interface'; +CREATE FUNCTION spock.node_drop_interface(node_name name, interface_name name) +RETURNS boolean STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_alter_node_drop_interface'; + +CREATE FUNCTION spock.sub_create(subscription_name name, provider_dsn text, + replication_sets text[] = '{default,default_insert_only,ddl_sql}', synchronize_structure boolean = false, + synchronize_data boolean = false, forward_origins text[] = '{}', apply_delay interval DEFAULT '0', + force_text_transfer boolean = false, + enabled boolean = true) +RETURNS oid STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_create_subscription'; +CREATE FUNCTION spock.sub_drop(subscription_name name, ifexists boolean DEFAULT false) +RETURNS oid STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_drop_subscription'; + +CREATE FUNCTION spock.sub_alter_interface(subscription_name name, interface_name name) +RETURNS boolean STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_alter_subscription_interface'; + +CREATE FUNCTION spock.sub_disable(subscription_name name, immediate boolean DEFAULT false) +RETURNS boolean STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_alter_subscription_disable'; +CREATE FUNCTION spock.sub_enable(subscription_name name, immediate boolean DEFAULT false) +RETURNS boolean STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_alter_subscription_enable'; + +CREATE FUNCTION spock.sub_add_repset(subscription_name name, replication_set name) +RETURNS boolean STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_alter_subscription_add_replication_set'; +CREATE FUNCTION spock.sub_remove_repset(subscription_name name, replication_set name) +RETURNS boolean STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_alter_subscription_remove_replication_set'; +CREATE FUNCTION spock.sub_alter_skiplsn(subscription_name name, lsn pg_lsn) + RETURNS boolean STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_alter_subscription_skip_lsn'; + +CREATE FUNCTION spock.sub_show_status(subscription_name name DEFAULT NULL, + OUT subscription_name text, OUT status text, OUT provider_node text, + OUT provider_dsn text, OUT slot_name text, OUT replication_sets text[], + OUT forward_origins text[]) +RETURNS SETOF record STABLE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_show_subscription_status'; + +CREATE TABLE spock.replication_set ( + set_id oid NOT NULL PRIMARY KEY, + set_nodeid oid NOT NULL REFERENCES node(node_id) ON UPDATE CASCADE, + set_name name NOT NULL, + replicate_insert boolean NOT NULL DEFAULT true, + replicate_update boolean NOT NULL DEFAULT true, + replicate_delete boolean NOT NULL DEFAULT true, + replicate_truncate boolean NOT NULL DEFAULT true, + UNIQUE (set_nodeid, set_name) +) WITH (user_catalog_table=true); + +CREATE TABLE spock.replication_set_table ( + set_id oid NOT NULL, + set_reloid regclass NOT NULL, + set_att_list text[], + set_row_filter pg_node_tree, + PRIMARY KEY(set_id, set_reloid) +) WITH (user_catalog_table=true); + +CREATE TABLE spock.replication_set_seq ( + set_id oid NOT NULL, + set_seqoid regclass NOT NULL, + PRIMARY KEY(set_id, set_seqoid) +) WITH (user_catalog_table=true); + +CREATE TABLE spock.sequence_state ( + seqoid oid NOT NULL PRIMARY KEY, + cache_size integer NOT NULL, + last_value bigint NOT NULL +) WITH (user_catalog_table=true); + +CREATE TABLE spock.depend ( + classid oid NOT NULL, + objid oid NOT NULL, + objsubid integer NOT NULL, + + refclassid oid NOT NULL, + refobjid oid NOT NULL, + refobjsubid integer NOT NULL, + + deptype "char" NOT NULL +) WITH (user_catalog_table=true); + +CREATE TABLE spock.pii ( + id int generated always as identity, + pii_schema text NOT NULL, + pii_table text NOT NULL, + pii_column text NOT NULL, + PRIMARY KEY(id) +) WITH (user_catalog_table=true); + +CREATE TABLE spock.resolutions ( + id int generated always as identity, + node_name name NOT NULL, + log_time timestamptz NOT NULL, + relname text, + idxname text, + conflict_type text, + conflict_resolution text, + + -- columns for local changes + local_origin int, + local_tuple text, + local_xid xid, + local_timestamp timestamptz, + + -- columns for remote changes + remote_origin int, + remote_tuple text, + remote_xid xid, + remote_timestamp timestamptz, + remote_lsn pg_lsn, + + PRIMARY KEY(id, node_name) +) WITH (user_catalog_table=true); + +CREATE VIEW spock.TABLES AS + WITH set_relations AS ( + SELECT s.set_name, r.set_reloid + FROM spock.replication_set_table r, + spock.replication_set s, + spock.local_node n + WHERE s.set_nodeid = n.node_id + AND s.set_id = r.set_id + ), + user_tables AS ( + SELECT r.oid, n.nspname, r.relname, r.relreplident + FROM pg_catalog.pg_class r, + pg_catalog.pg_namespace n + WHERE r.relkind IN ('r', 'p') + AND r.relpersistence = 'p' + AND n.oid = r.relnamespace + AND n.nspname !~ '^pg_' + AND n.nspname != 'information_schema' + AND n.nspname != 'spock' + ) + SELECT r.oid AS relid, n.nspname, r.relname, s.set_name + FROM pg_catalog.pg_namespace n, + pg_catalog.pg_class r, + set_relations s + WHERE r.relkind IN ('r', 'p') + AND n.oid = r.relnamespace + AND r.oid = s.set_reloid + UNION + SELECT t.oid AS relid, t.nspname, t.relname, NULL + FROM user_tables t + WHERE t.oid NOT IN (SELECT set_reloid FROM set_relations); + +CREATE FUNCTION spock.repset_create(set_name name, + replicate_insert boolean = true, replicate_update boolean = true, + replicate_delete boolean = true, replicate_truncate boolean = true) +RETURNS oid STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_create_replication_set'; +CREATE FUNCTION spock.repset_alter(set_name name, + replicate_insert boolean DEFAULT NULL, replicate_update boolean DEFAULT NULL, + replicate_delete boolean DEFAULT NULL, replicate_truncate boolean DEFAULT NULL) +RETURNS oid CALLED ON NULL INPUT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_alter_replication_set'; +CREATE FUNCTION spock.repset_drop(set_name name, ifexists boolean DEFAULT false) +RETURNS boolean STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_drop_replication_set'; + +CREATE FUNCTION spock.repset_add_table(set_name name, relation regclass, synchronize_data boolean DEFAULT false, + columns text[] DEFAULT NULL, row_filter text DEFAULT NULL, include_partitions boolean default true) +RETURNS boolean CALLED ON NULL INPUT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_replication_set_add_table'; +CREATE FUNCTION spock.repset_add_all_tables(set_name name, schema_names text[], synchronize_data boolean DEFAULT false) +RETURNS boolean STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_replication_set_add_all_tables'; +CREATE FUNCTION spock.repset_remove_table(set_name name, relation regclass, include_partitions boolean default true) +RETURNS boolean STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_replication_set_remove_table'; + +CREATE FUNCTION spock.repset_add_seq(set_name name, relation regclass, synchronize_data boolean DEFAULT false) +RETURNS boolean STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_replication_set_add_sequence'; +CREATE FUNCTION spock.repset_add_all_seqs(set_name name, schema_names text[], synchronize_data boolean DEFAULT false) +RETURNS boolean STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_replication_set_add_all_sequences'; +CREATE FUNCTION spock.repset_remove_seq(set_name name, relation regclass) +RETURNS boolean STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_replication_set_remove_sequence'; + +CREATE FUNCTION spock.repset_add_partition(parent regclass, partition regclass default NULL, + row_filter text default NULL) +RETURNS int CALLED ON NULL INPUT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_replication_set_add_partition'; + +CREATE FUNCTION spock.repset_remove_partition(parent regclass, partition regclass default NULL) +RETURNS int CALLED ON NULL INPUT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_replication_set_remove_partition'; + +CREATE FUNCTION spock.sub_alter_sync(subscription_name name, truncate boolean DEFAULT false) +RETURNS boolean STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_alter_subscription_synchronize'; + +CREATE FUNCTION spock.sub_resync_table(subscription_name name, relation regclass, + truncate boolean DEFAULT true) +RETURNS boolean STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_alter_subscription_resynchronize_table'; + +CREATE FUNCTION spock.sync_seq(relation regclass) +RETURNS boolean STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_synchronize_sequence'; + +CREATE FUNCTION spock.table_data_filtered(reltyp anyelement, relation regclass, repsets text[]) +RETURNS SETOF anyelement CALLED ON NULL INPUT STABLE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_table_data_filtered'; + +CREATE FUNCTION spock.repset_show_table(relation regclass, repsets text[], OUT relid oid, OUT nspname text, + OUT relname text, OUT att_list text[], OUT has_row_filter boolean, OUT relkind "char", OUT relispartition boolean) +RETURNS record STRICT STABLE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_show_repset_table_info'; + +CREATE FUNCTION spock.sub_show_table(subscription_name name, relation regclass, OUT nspname text, OUT relname text, OUT status text) +RETURNS record STRICT STABLE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_show_subscription_table'; + +CREATE TABLE spock.queue ( + queued_at timestamp with time zone NOT NULL, + role name NOT NULL, + replication_sets text[], + message_type "char" NOT NULL, + message json NOT NULL +); + +CREATE FUNCTION spock.replicate_ddl(command text, + replication_sets text[] DEFAULT '{ddl_sql}', + search_path text DEFAULT '', + role text DEFAULT CURRENT_USER) +RETURNS boolean STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_replicate_ddl_command'; + +CREATE FUNCTION spock.replicate_ddl(command text[], + replication_sets text[] DEFAULT '{ddl_sql}', + search_path text DEFAULT current_setting('search_path'), + role text DEFAULT CURRENT_USER) +RETURNS SETOF boolean STRICT VOLATILE LANGUAGE sql AS + 'SELECT spock.replicate_ddl(cmd, $2, $3, $4) FROM (SELECT unnest(command) cmd)'; + +CREATE FUNCTION spock.node_info(OUT node_id oid, OUT node_name text, + OUT sysid text, OUT dbname text, OUT replication_sets text, + OUT location text, OUT country text, OUT info jsonb) +RETURNS record +STABLE STRICT LANGUAGE c AS 'MODULE_PATHNAME', 'spock_node_info'; + +CREATE FUNCTION spock.spock_gen_slot_name(name, name, name) +RETURNS name +IMMUTABLE STRICT LANGUAGE c AS 'MODULE_PATHNAME'; + +CREATE FUNCTION spock_version() RETURNS text +LANGUAGE c AS 'MODULE_PATHNAME'; + +CREATE FUNCTION spock_version_num() RETURNS integer +LANGUAGE c AS 'MODULE_PATHNAME'; + +CREATE FUNCTION spock_max_proto_version() RETURNS integer +LANGUAGE c AS 'MODULE_PATHNAME'; + +CREATE FUNCTION spock_min_proto_version() RETURNS integer +LANGUAGE c AS 'MODULE_PATHNAME'; + +CREATE FUNCTION spock.get_country() RETURNS text +LANGUAGE sql AS +$$ SELECT current_setting('spock.country') $$; + +CREATE FUNCTION +spock.wait_slot_confirm_lsn(slotname name, target pg_lsn) +RETURNS void LANGUAGE c AS 'spock','spock_wait_slot_confirm_lsn'; + +CREATE FUNCTION spock.sub_wait_for_sync(subscription_name name) +RETURNS void RETURNS NULL ON NULL INPUT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_wait_for_subscription_sync_complete'; + +CREATE FUNCTION spock.table_wait_for_sync(subscription_name name, relation regclass) +RETURNS void RETURNS NULL ON NULL INPUT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_wait_for_table_sync_complete'; + +CREATE FUNCTION spock.sync_event() +RETURNS pg_lsn RETURNS NULL ON NULL INPUT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_create_sync_event'; + +CREATE PROCEDURE spock.wait_for_sync_event(OUT result bool, origin_id oid, lsn pg_lsn, timeout int DEFAULT 0) +AS $$ +DECLARE + target_id oid; + elapsed_time numeric := 0; + progress_lsn pg_lsn; +BEGIN + IF origin_id IS NULL THEN + RAISE EXCEPTION 'Origin node ''%'' not found', origin; + END IF; + target_id := node_id FROM spock.node_info(); + + WHILE true LOOP + SELECT INTO progress_lsn remote_lsn + FROM spock.progress + WHERE node_id = target_id AND remote_node_id = origin_id; + IF progress_lsn >= lsn THEN + result = true; + RETURN; + END IF; + elapsed_time := elapsed_time + .2; + IF timeout <> 0 AND elapsed_time >= timeout THEN + result := false; + RETURN; + END IF; + + ROLLBACK; + PERFORM pg_sleep(0.2); + END LOOP; +END; +$$ LANGUAGE plpgsql; + +CREATE PROCEDURE spock.wait_for_sync_event(OUT result bool, origin name, lsn pg_lsn, timeout int DEFAULT 0) +AS $$ +DECLARE + origin_id oid; + target_id oid; + elapsed_time numeric := 0; + progress_lsn pg_lsn; +BEGIN + origin_id := node_id FROM spock.node WHERE node_name = origin; + IF origin_id IS NULL THEN + RAISE EXCEPTION 'Origin node ''%'' not found', origin; + END IF; + target_id := node_id FROM spock.node_info(); + + WHILE true LOOP + SELECT INTO progress_lsn remote_lsn + FROM spock.progress + WHERE node_id = target_id AND remote_node_id = origin_id; + IF progress_lsn >= lsn THEN + result = true; + RETURN; + END IF; + elapsed_time := elapsed_time + .2; + IF timeout <> 0 AND elapsed_time >= timeout THEN + result := false; + RETURN; + END IF; + + ROLLBACK; + PERFORM pg_sleep(0.2); + END LOOP; +END; +$$ LANGUAGE plpgsql; + +CREATE FUNCTION spock.xact_commit_timestamp_origin("xid" xid, OUT "timestamp" timestamptz, OUT "roident" oid) +RETURNS record RETURNS NULL ON NULL INPUT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_xact_commit_timestamp_origin'; + +CREATE FUNCTION spock.get_channel_stats( + OUT subid oid, + OUT relid oid, + OUT n_tup_ins bigint, + OUT n_tup_upd bigint, + OUT n_tup_del bigint, + OUT n_conflict bigint, + OUT n_dca bigint) +RETURNS SETOF record +LANGUAGE c AS 'MODULE_PATHNAME', 'get_channel_stats'; + +CREATE FUNCTION spock.reset_channel_stats() RETURNS void +LANGUAGE c AS 'MODULE_PATHNAME', 'reset_channel_stats'; + +CREATE VIEW spock.channel_table_stats AS + SELECT H.subid, H.relid, + CASE H.subid + WHEN 0 THEN '' + ELSE S.sub_name + END AS sub_name, + pg_catalog.quote_ident(N.nspname) || '.' || pg_catalog.quote_ident(C.relname) AS table_name, + H.n_tup_ins, H.n_tup_upd, H.n_tup_del, + H.n_conflict, H.n_dca + FROM spock.get_channel_stats() AS H + LEFT JOIN spock.subscription AS S ON S.sub_id = H.subid + LEFT JOIN pg_catalog.pg_class AS C ON C.oid = H.relid + LEFT JOIN pg_catalog.pg_namespace AS N ON N.oid = C.relnamespace; + +CREATE VIEW spock.channel_summary_stats AS + SELECT subid, sub_name, + sum(n_tup_ins) AS n_tup_ins, + sum(n_tup_upd) AS n_tup_upd, + sum(n_tup_del) AS n_tup_del, + sum(n_conflict) AS n_conflict, + sum(n_dca) AS n_dca + FROM spock.channel_table_stats + GROUP BY subid, sub_name; + +CREATE VIEW spock.lag_tracker AS + SELECT + origin.node_name AS origin_name, + n.node_name AS receiver_name, + MAX(p.remote_commit_ts) AS commit_timestamp, + MAX(p.remote_lsn) AS last_received_lsn, + MAX(p.remote_insert_lsn) AS remote_insert_lsn, + CASE + WHEN CAST(MAX(CAST(p.updated_by_decode as int)) as bool) THEN pg_wal_lsn_diff(MAX(p.remote_insert_lsn), MAX(p.remote_lsn)) + ELSE 0 + END AS replication_lag_bytes, + CASE + WHEN CAST(MAX(CAST(p.updated_by_decode as int)) as bool) THEN now() - MAX(p.remote_commit_ts) + ELSE now() - MAX(p.last_updated_ts) + END AS replication_lag + FROM spock.progress p + LEFT JOIN spock.subscription sub ON (p.node_id = sub.sub_target and p.remote_node_id = sub.sub_origin) + LEFT JOIN spock.node origin ON sub.sub_origin = origin.node_id + LEFT JOIN spock.node n ON n.node_id = p.node_id + GROUP BY origin.node_name, n.node_name; + +CREATE FUNCTION spock.md5_agg_sfunc(text, anyelement) + RETURNS text + LANGUAGE sql +AS +$$ + SELECT md5($1 || $2::text) +$$; +CREATE AGGREGATE spock.md5_agg (ORDER BY anyelement) +( + STYPE = text, + SFUNC = spock.md5_agg_sfunc, + INITCOND = '' +); + +-- ---------------------------------------------------------------------- +-- Spock Read Only +-- ---------------------------------------------------------------------- +CREATE FUNCTION spock.terminate_active_transactions() RETURNS bool + AS 'MODULE_PATHNAME', 'spockro_terminate_active_transactions' + LANGUAGE C STRICT; + +-- ---------------------------------------------------------------------- +-- We check the PostgreSQL major version number in case a future +-- catalog change forces us to provide different functions for +-- different versions. +-- ---------------------------------------------------------------------- +DO $version_dependent$ +DECLARE + pgmajor integer; +BEGIN + pgmajor = regexp_replace(regexp_replace(version(), '^PostgreSQL ', ''), '[^0-9].*', '')::integer; + + CASE + WHEN pgmajor IN (15, 16, 17, 18) THEN + +-- ---------------------------------------------------------------------- +-- convert_column_to_int8() +-- +-- Change the data type of a column to int8 and recursively alter +-- all columns that reference this one through foreign key constraints. +-- ---------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION spock.convert_column_to_int8(p_rel regclass, p_attnum smallint) +RETURNS integer +SET search_path = pg_catalog +AS $$ +DECLARE + v_attr record; + v_fk record; + v_attidx integer; + v_cmd text; + v_num_altered integer := 0; +BEGIN + -- ---- + -- Get the attribute definition + -- ---- + SELECT * INTO v_attr + FROM pg_namespace N + JOIN pg_class C + ON N.oid = C.relnamespace + JOIN pg_attribute A + ON C.oid = A.attrelid + WHERE A.attrelid = p_rel + AND A.attnum = p_attnum; + + IF NOT FOUND THEN + RAISE EXCEPTION 'Attribute % of reation % not found', p_attnum, p_rel; + END IF; + + -- ---- + -- If the attribute type is not bigint, we change it + -- ---- + IF v_attr.atttypid <> 'int8'::regtype THEN + v_cmd = 'ALTER TABLE ' || + quote_ident(v_attr.nspname) || '.' || + quote_ident(v_attr.relname) || + ' ALTER COLUMN ' || + quote_ident(v_attr.attname) || + ' SET DATA TYPE int8'; + RAISE NOTICE 'EXECUTE %', v_cmd; + EXECUTE v_cmd; + + v_num_altered = v_num_altered + 1; + END IF; + + -- ---- + -- Convert foreign keys referencing this column as well + -- ---- + FOR v_fk IN + SELECT * FROM pg_constraint F + JOIN pg_class C + ON C.oid = F.conrelid + JOIN pg_namespace N + ON N.oid = C.relnamespace + WHERE F.contype = 'f' + AND F.confrelid = v_attr.attrelid + LOOP + -- ---- + -- Lookup the attribute index in the possibly compount FK + -- ---- + v_attidx = array_position(v_fk.confkey, v_attr.attnum); + IF v_attidx IS NULL THEN + CONTINUE; + END IF; + + -- ---- + -- Recurse for the referencing column + -- ---- + v_num_altered = v_num_altered + + spock.convert_column_to_int8(v_fk.conrelid, + v_fk.conkey[v_attidx]); + END LOOP; + RETURN v_num_altered; +END; +$$ LANGUAGE plpgsql; + +-- ---------------------------------------------------------------------- +-- convert_sequence_to_snowflake() +-- +-- Convert the DEFAULT expression for a sequence to snowflake's nextval() +-- function. Eventually change the data type of columns using it +-- to bigint. +-- ---------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION spock.convert_sequence_to_snowflake(p_seqid regclass) +RETURNS integer +SET search_path = pg_catalog +AS $$ +DECLARE + v_attrdef record; + v_attr record; + v_seq record; + v_cmd text; + v_num_altered integer := 0; +BEGIN + -- ---- + -- We are looking for column defaults that use the requested + -- sequence and the function nextval(). + -- ---- + FOR v_attrdef IN + SELECT AD.*, + pg_get_expr(AD.adbin, AD.adrelid, true) adstr + FROM pg_depend D + JOIN pg_attrdef AD + ON D.refclassid = 'pg_class'::regclass + AND AD.adrelid = D.refobjid + AND AD.adnum = D.refobjsubid + WHERE D.classid = 'pg_class'::regclass + AND D.objid = p_seqid + LOOP + IF v_attrdef.adstr NOT LIKE 'nextval(%' THEN + CONTINUE; + END IF; + + -- ---- + -- Get the attribute definition + -- ---- + SELECT * INTO v_attr + FROM pg_namespace N + JOIN pg_class C + ON N.oid = C.relnamespace + JOIN pg_attribute A + ON C.oid = A.attrelid + WHERE A.attrelid = v_attrdef.adrelid + AND A.attnum = v_attrdef.adnum; + + IF NOT FOUND THEN + RAISE EXCEPTION 'Attribute for % not found', v_attrdef.adstr; + END IF; + + -- ---- + -- Get the sequence definition + -- ---- + SELECT * INTO v_seq + FROM pg_namespace N + JOIN pg_class C + ON N.oid = C.relnamespace + WHERE C.oid = p_seqid; + + IF NOT FOUND THEN + RAISE EXCEPTION 'Sequence with Oid % not found', p_seqid; + END IF; + + -- ---- + -- If the attribute type is not bigint, we change it + -- ---- + v_num_altered = v_num_altered + + spock.convert_column_to_int8(v_attr.attrelid, v_attr.attnum); + + -- ---- + -- Now we can change the default to snowflake.nextval() + -- ---- + v_cmd = 'ALTER TABLE ' || + quote_ident(v_attr.nspname) || '.' || + quote_ident(v_attr.relname) || + ' ALTER COLUMN ' || + quote_ident(v_attr.attname) || + ' SET DEFAULT snowflake.nextval(''' || + quote_ident(v_seq.nspname) || '.' || + quote_ident(v_seq.relname) || + '''::regclass)'; + RAISE NOTICE 'EXECUTE %', v_cmd; + EXECUTE v_cmd; + + v_num_altered = v_num_altered + 1; + END LOOP; + RETURN v_num_altered; +END; +$$ LANGUAGE plpgsql; + + -- END pgmajor in (15, 16, 17, 18) + ELSE + RAISE EXCEPTION 'Unsupported PostgreSQL major version %', pgmajor; + END CASE; +-- End of PG major version dependent PL/pgSQL definitions +END; +$version_dependent$ LANGUAGE plpgsql; + +-- ---- +-- Generic delta apply functions for all numeric data types +-- ---- +CREATE FUNCTION spock.delta_apply(int2, int2, int2) +RETURNS int2 LANGUAGE c AS 'MODULE_PATHNAME', 'delta_apply_int2'; +CREATE FUNCTION spock.delta_apply(int4, int4, int4) +RETURNS int4 LANGUAGE c AS 'MODULE_PATHNAME', 'delta_apply_int4'; +CREATE FUNCTION spock.delta_apply(int8, int8, int8) +RETURNS int8 LANGUAGE c AS 'MODULE_PATHNAME', 'delta_apply_int8'; +CREATE FUNCTION spock.delta_apply(float4, float4, float4) +RETURNS float4 LANGUAGE c AS 'MODULE_PATHNAME', 'delta_apply_float4'; +CREATE FUNCTION spock.delta_apply(float8, float8, float8) +RETURNS float8 LANGUAGE c AS 'MODULE_PATHNAME', 'delta_apply_float8'; +CREATE FUNCTION spock.delta_apply(numeric, numeric, numeric) +RETURNS numeric LANGUAGE c AS 'MODULE_PATHNAME', 'delta_apply_numeric'; +CREATE FUNCTION spock.delta_apply(money, money, money) +RETURNS money LANGUAGE c AS 'MODULE_PATHNAME', 'delta_apply_money'; + +-- ---- +-- Function to control REPAIR mode +-- ---- +CREATE FUNCTION spock.repair_mode(enabled bool) +RETURNS pg_catalog.pg_lsn LANGUAGE c +AS 'MODULE_PATHNAME', 'spock_repair_mode'; + +-- ---- +-- Function to determine LSN from commit timestamp +-- ---- +CREATE FUNCTION spock.get_lsn_from_commit_ts(slot_name name, commit_ts timestamptz) +RETURNS pg_lsn STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_get_lsn_from_commit_ts'; + +CREATE OR REPLACE FUNCTION spock.get_apply_worker_status( + OUT worker_pid bigint, -- Changed from int to bigint + OUT worker_dboid int, + OUT worker_subid bigint, + OUT worker_status text +) +RETURNS SETOF record STABLE LANGUAGE c AS 'MODULE_PATHNAME', 'get_apply_worker_status'; + +CREATE FUNCTION spock.wait_for_apply_worker(p_subbid bigint, timeout int DEFAULT 0) +RETURNS boolean +AS $$ +DECLARE + start_time timestamptz := clock_timestamp(); + elapsed_time int := 0; + current_status text; +BEGIN + -- Loop until the timeout is reached or the worker is no longer running + WHILE true LOOP + -- Call spock.get_apply_worker_status to check the worker's status + SELECT worker_status + INTO current_status + FROM spock.get_apply_worker_status() + WHERE worker_subid = p_subbid; + + -- If no row is found, return -1 + IF NOT FOUND THEN + RETURN false; + END IF; + + -- If the worker is no longer running, return 0 + IF current_status IS DISTINCT FROM 'running' THEN + RETURN false; + END IF; + + -- Check if the timeout has been reached + elapsed_time := EXTRACT(EPOCH FROM clock_timestamp() - start_time) * 1000; + IF timeout > 0 AND elapsed_time >= timeout THEN + RETURN true; + END IF; + + -- Sleep for a short interval before checking again + PERFORM pg_sleep(0.2); + END LOOP; +END; +$$ LANGUAGE plpgsql; diff --git a/sql/spock--5.0.6--5.0.7.sql b/sql/spock--5.0.6--5.0.7.sql new file mode 100644 index 00000000..cb8ceb4b --- /dev/null +++ b/sql/spock--5.0.6--5.0.7.sql @@ -0,0 +1,163 @@ + +/* spock--5.0.6--5.0.7.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION spock UPDATE TO '5.0.7'" to load this file. \quit + +CREATE FUNCTION spock.pause_apply_workers() +RETURNS void VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_pause_apply_workers'; + +CREATE FUNCTION spock.resume_apply_workers() +RETURNS void VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_resume_apply_workers'; + +REVOKE EXECUTE ON FUNCTION spock.pause_apply_workers() FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION spock.resume_apply_workers() FROM PUBLIC; + +DROP PROCEDURE IF EXISTS spock.wait_for_sync_event(OUT bool, oid, pg_lsn, int); +DROP PROCEDURE IF EXISTS spock.wait_for_sync_event(OUT bool, oid, pg_lsn, int, bool); +DROP PROCEDURE IF EXISTS spock.wait_for_sync_event(OUT bool, name, pg_lsn, int); +DROP PROCEDURE IF EXISTS spock.wait_for_sync_event(OUT bool, name, pg_lsn, int, bool); +CREATE PROCEDURE spock.wait_for_sync_event( + OUT result bool, + origin_id oid, + lsn pg_lsn, + timeout int DEFAULT 0, + wait_if_disabled bool DEFAULT false +) AS $$ +DECLARE + target_id oid; + start_time timestamptz := clock_timestamp(); + progress_lsn pg_lsn; + sub_is_enabled bool; + sub_slot name; +BEGIN + IF origin_id IS NULL THEN + RAISE EXCEPTION 'Invalid NULL origin_id'; + END IF; + target_id := node_id FROM spock.node_info(); + + -- Upfront existence check is skipped when wait_if_disabled is true because + -- the subscription may not yet exist (e.g. a newly added node whose + -- subscriptions are still initializing). The loop below handles both the + -- not-found and disabled cases gracefully in that mode. + IF NOT wait_if_disabled THEN + SELECT sub_enabled, sub_slot_name INTO sub_is_enabled, sub_slot + FROM spock.subscription + WHERE sub_origin = origin_id AND sub_target = target_id; + + IF NOT FOUND THEN + RAISE EXCEPTION 'No subscription found for replication % => %', + origin_id, target_id; + END IF; + END IF; + + WHILE true LOOP + -- Re-check subscription state each iteration. Also re-fetches + -- sub_slot_name so the loop is self-contained when wait_if_disabled + -- is true and the pre-loop check was skipped. + SELECT sub_enabled, sub_slot_name INTO sub_is_enabled, sub_slot + FROM spock.subscription + WHERE sub_origin = origin_id AND sub_target = target_id; + + IF NOT FOUND THEN + IF NOT wait_if_disabled THEN + RAISE EXCEPTION 'No subscription found for replication % => %', + origin_id, target_id; + END IF; + -- Subscription not yet created; fall through to sleep. + ELSIF NOT sub_is_enabled THEN + IF NOT wait_if_disabled THEN + RAISE EXCEPTION 'Subscription % => % has been disabled', + origin_id, target_id; + END IF; + -- Subscription still initializing; fall through to sleep. + ELSE + -- Subscription is enabled; check LSN progress. + -- Uses PostgreSQL's native origin tracking rather than spock.progress + SELECT remote_lsn INTO progress_lsn + FROM pg_replication_origin_status + WHERE external_id = sub_slot; + + IF progress_lsn IS NOT NULL AND progress_lsn >= lsn THEN + result = true; + RETURN; + END IF; + END IF; + + IF timeout <> 0 AND + EXTRACT(EPOCH FROM (clock_timestamp() - start_time)) >= timeout THEN + result := false; + RETURN; + END IF; + + ROLLBACK; + PERFORM pg_sleep(0.2); + END LOOP; +END; +$$ LANGUAGE plpgsql; + +CREATE PROCEDURE spock.wait_for_sync_event( + OUT result bool, + origin name, + lsn pg_lsn, + timeout int DEFAULT 0, + wait_if_disabled bool DEFAULT false +) AS $$ +DECLARE + origin_id oid; +BEGIN + origin_id := node_id FROM spock.node WHERE node_name = origin; + IF origin_id IS NULL THEN + RAISE EXCEPTION 'Origin node ''%'' not found', origin; + END IF; + CALL spock.wait_for_sync_event(result, origin_id, lsn, timeout, wait_if_disabled); +END; +$$ LANGUAGE plpgsql; + +-- spock.sync_event() gained an optional 'transactional' boolean argument +-- (default false). Drop the old zero-arg signature first so the upgrade +-- doesn't leave behind two overloads with overlapping zero-arg resolution. +DROP FUNCTION IF EXISTS spock.sync_event(); +CREATE FUNCTION spock.sync_event(transactional boolean DEFAULT false) +RETURNS pg_lsn RETURNS NULL ON NULL INPUT +AS 'MODULE_PATHNAME', 'spock_create_sync_event' +LANGUAGE C VOLATILE; + +/* + * Correct the declared type of spock.subscription.sub_skip_schema. + * + * The column was added as text in the 5.0.1--5.0.2 upgrade, but the C code + * has always treated it as text[] on both read and write paths + * (strlist_to_textarray on write, DatumGetArrayTypeP on read). The bytes + * already on disk are therefore a valid ArrayType; only the catalog's type + * label is wrong. ALTER TABLE ... ALTER COLUMN TYPE text[] USING ... is + * not viable here: there is no SQL expression that converts "varlena bytes + * the planner believes are text but are in fact ArrayType internal format" + * back into an ArrayType Datum. Relabel the column in place so SQL-level + * access (SELECT, unnest, etc.) works as users expect, without rewriting + * data. + */ +LOCK TABLE spock.subscription IN ACCESS EXCLUSIVE MODE; + +UPDATE pg_catalog.pg_attribute + SET atttypid = 'text[]'::regtype, + attndims = 1 + WHERE attrelid = 'spock.subscription'::regclass + AND attname = 'sub_skip_schema' + AND atttypid = 'text'::regtype; + +/* + * Drop any pg_statistic rows for the column. Stats sampled when the + * column was labelled text encode varlena bytes with text semantics; + * after the relabel the planner would interpret the same stavalues + * arrays as text[], producing nonsense selectivities (and possibly + * crashing on operators that validate ArrayType structure). ANALYZE + * will repopulate as needed. + */ +DELETE FROM pg_catalog.pg_statistic + WHERE starelid = 'spock.subscription'::regclass + AND staattnum = ( + SELECT attnum + FROM pg_catalog.pg_attribute + WHERE attrelid = 'spock.subscription'::regclass + AND attname = 'sub_skip_schema'); diff --git a/sql/spock--5.0.7--5.0.8.sql b/sql/spock--5.0.7--5.0.8.sql new file mode 100644 index 00000000..85b89849 --- /dev/null +++ b/sql/spock--5.0.7--5.0.8.sql @@ -0,0 +1,6 @@ +/* spock--5.0.7--5.0.8.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION spock UPDATE TO '5.0.8'" to load this file. \quit + +-- No schema changes in 5.0.8. diff --git a/sql/spock--5.0.6--6.0.0-devel.sql b/sql/spock--5.0.8--6.0.0-devel.sql similarity index 72% rename from sql/spock--5.0.6--6.0.0-devel.sql rename to sql/spock--5.0.8--6.0.0-devel.sql index f0078a09..fff13e01 100644 --- a/sql/spock--5.0.6--6.0.0-devel.sql +++ b/sql/spock--5.0.8--6.0.0-devel.sql @@ -1,8 +1,12 @@ -/* spock--5.0.6--6.0.0-devel.sql */ +/* spock--5.0.8--6.0.0-devel.sql */ -- complain if script is sourced in psql, rather than via ALTER EXTENSION \echo Use "ALTER EXTENSION spock UPDATE TO '6.0.0-devel'" to load this file. \quit +-- Note: the spock.subscription.sub_skip_schema text->text[] relabel is now +-- performed in the 5.0.6--5.0.7 upgrade (matching v5_STABLE), so no UPDATE +-- of pg_attribute is needed here. + DROP VIEW IF EXISTS spock.lag_tracker; DROP TABLE IF EXISTS spock.progress; @@ -30,19 +34,8 @@ CREATE VIEW spock.progress AS SELECT oid FROM pg_database WHERE datname = current_database() ); -CREATE FUNCTION spock.pause_apply_workers() -RETURNS void -AS 'MODULE_PATHNAME', 'spock_pause_apply_workers' -LANGUAGE C VOLATILE; - -REVOKE ALL ON FUNCTION spock.pause_apply_workers() FROM PUBLIC; - -CREATE FUNCTION spock.resume_apply_workers() -RETURNS void -AS 'MODULE_PATHNAME', 'spock_resume_apply_workers' -LANGUAGE C VOLATILE; - -REVOKE ALL ON FUNCTION spock.resume_apply_workers() FROM PUBLIC; +-- Note: spock.pause_apply_workers() and spock.resume_apply_workers() were +-- introduced in 5.0.7, so no CREATE statements for them are needed here. -- Read peer progress (ros.remote_lsn) for all peer subscriptions. -- Called while apply workers are paused and the slot's snapshot is imported. @@ -323,115 +316,9 @@ END; $$ LANGUAGE plpgsql STRICT VOLATILE; --- spock.sync_event() gained an optional 'transactional' boolean argument --- (default false). Drop the old zero-arg signature first so the upgrade --- doesn't leave behind two overloads with overlapping zero-arg resolution. -DROP FUNCTION IF EXISTS spock.sync_event(); -CREATE FUNCTION spock.sync_event(transactional boolean DEFAULT false) -RETURNS pg_lsn RETURNS NULL ON NULL INPUT -AS 'MODULE_PATHNAME', 'spock_create_sync_event' -LANGUAGE C VOLATILE; - -DROP PROCEDURE IF EXISTS spock.wait_for_sync_event(OUT bool, oid, pg_lsn, int); -DROP PROCEDURE IF EXISTS spock.wait_for_sync_event(OUT bool, oid, pg_lsn, int, bool); -DROP PROCEDURE IF EXISTS spock.wait_for_sync_event(OUT bool, name, pg_lsn, int); -DROP PROCEDURE IF EXISTS spock.wait_for_sync_event(OUT bool, name, pg_lsn, int, bool); -CREATE PROCEDURE spock.wait_for_sync_event( - OUT result bool, - origin_id oid, - lsn pg_lsn, - timeout int DEFAULT 0, - wait_if_disabled bool DEFAULT false -) AS $$ -DECLARE - target_id oid; - start_time timestamptz := clock_timestamp(); - progress_lsn pg_lsn; - sub_is_enabled bool; - sub_slot name; -BEGIN - IF origin_id IS NULL THEN - RAISE EXCEPTION 'Invalid NULL origin_id'; - END IF; - target_id := node_id FROM spock.node_info(); - - -- Upfront existence check is skipped when wait_if_disabled is true because - -- the subscription may not yet exist (e.g. a newly added node whose - -- subscriptions are still initializing). The loop below handles both the - -- not-found and disabled cases gracefully in that mode. - IF NOT wait_if_disabled THEN - SELECT sub_enabled, sub_slot_name INTO sub_is_enabled, sub_slot - FROM spock.subscription - WHERE sub_origin = origin_id AND sub_target = target_id; - - IF NOT FOUND THEN - RAISE EXCEPTION 'No subscription found for replication % => %', - origin_id, target_id; - END IF; - END IF; - - WHILE true LOOP - -- Re-check subscription state each iteration. Also re-fetches - -- sub_slot_name so the loop is self-contained when wait_if_disabled - -- is true and the pre-loop check was skipped. - SELECT sub_enabled, sub_slot_name INTO sub_is_enabled, sub_slot - FROM spock.subscription - WHERE sub_origin = origin_id AND sub_target = target_id; - - IF NOT FOUND THEN - IF NOT wait_if_disabled THEN - RAISE EXCEPTION 'No subscription found for replication % => %', - origin_id, target_id; - END IF; - -- Subscription not yet created; fall through to sleep. - ELSIF NOT sub_is_enabled THEN - IF NOT wait_if_disabled THEN - RAISE EXCEPTION 'Subscription % => % has been disabled', - origin_id, target_id; - END IF; - -- Subscription still initializing; fall through to sleep. - ELSE - -- Subscription is enabled; check LSN progress. - -- Uses PostgreSQL's native origin tracking rather than spock.progress - SELECT remote_lsn INTO progress_lsn - FROM pg_replication_origin_status - WHERE external_id = sub_slot; - - IF progress_lsn IS NOT NULL AND progress_lsn >= lsn THEN - result = true; - RETURN; - END IF; - END IF; - - IF timeout <> 0 AND - EXTRACT(EPOCH FROM (clock_timestamp() - start_time)) >= timeout THEN - result := false; - RETURN; - END IF; - - ROLLBACK; - PERFORM pg_sleep(0.2); - END LOOP; -END; -$$ LANGUAGE plpgsql; - -CREATE PROCEDURE spock.wait_for_sync_event( - OUT result bool, - origin name, - lsn pg_lsn, - timeout int DEFAULT 0, - wait_if_disabled bool DEFAULT false -) AS $$ -DECLARE - origin_id oid; -BEGIN - origin_id := node_id FROM spock.node WHERE node_name = origin; - IF origin_id IS NULL THEN - RAISE EXCEPTION 'Origin node ''%'' not found', origin; - END IF; - CALL spock.wait_for_sync_event(result, origin_id, lsn, timeout, wait_if_disabled); -END; -$$ LANGUAGE plpgsql; +-- Note: spock.sync_event(transactional boolean) and the wait_if_disabled +-- variants of spock.wait_for_sync_event() were introduced in 5.0.7 (matching +-- v5_STABLE), so no CREATE statements for them are needed here. CREATE FUNCTION spock.sub_alter_options( subscription_name name, @@ -440,3 +327,7 @@ CREATE FUNCTION spock.sub_alter_options( RETURNS boolean AS 'MODULE_PATHNAME', 'spock_alter_subscription_options' LANGUAGE C STRICT VOLATILE; + +ALTER TABLE spock.local_node + ADD COLUMN IF NOT EXISTS node_version int4 NOT NULL DEFAULT 0; +UPDATE spock.local_node SET node_version = spock.spock_version_num(); diff --git a/sql/spock--6.0.0-devel.sql b/sql/spock--6.0.0-devel.sql index 35cbebe1..a610d5e5 100644 --- a/sql/spock--6.0.0-devel.sql +++ b/sql/spock--6.0.0-devel.sql @@ -18,7 +18,8 @@ CREATE TABLE spock.node_interface ( CREATE TABLE spock.local_node ( node_id oid PRIMARY KEY REFERENCES node(node_id), - node_local_interface oid NOT NULL REFERENCES node_interface(if_id) + node_local_interface oid NOT NULL REFERENCES node_interface(if_id), + node_version int4 NOT NULL DEFAULT 0 ); CREATE TABLE spock.subscription ( diff --git a/src/spock.c b/src/spock.c index 8def4d39..646f926c 100644 --- a/src/spock.c +++ b/src/spock.c @@ -53,6 +53,7 @@ #include "pgstat.h" #include "spock_apply.h" + #if PG_VERSION_NUM >= 180000 #include "spock_conflict_stat.h" #endif @@ -927,7 +928,9 @@ spock_object_relabel(const ObjectAddress *object, const char *seclabel) extoid = get_extension_oid(EXTENSION_NAME, true); if (!OidIsValid(extoid)) - elog(ERROR, "spock extension is not created yet"); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("spock extension is not created yet"))); /* * Check: classId must be pg_class, objectId should an existing table and @@ -952,6 +955,20 @@ _PG_init(void) if (!process_shared_preload_libraries_in_progress) elog(ERROR, "spock is not in shared_preload_libraries"); + /* + * Runtime patchset check: if the server binary was built from a + * different patchset generation than this extension, refuse to + * start. An unpatched server never reaches here -- the dynamic + * linker fails on the missing SpockCorePatchsetVersion symbol. + */ + if (SpockCorePatchsetVersion != SPOCK_CORE_PATCHSET_VERSION) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("spock core patchset version mismatch: " + "server has v%d, extension expects v%d", + SpockCorePatchsetVersion, + SPOCK_CORE_PATCHSET_VERSION))); + DefineCustomEnumVariable("spock.conflict_resolution", gettext_noop("Sets method used for conflict resolution for resolvable conflicts."), NULL, @@ -1241,6 +1258,23 @@ _PG_init(void) NULL, NULL); + /* + * Register the spock security label provider BEFORE the + * IsBinaryUpgrade early-return. The binary-upgrade compatibility + * shims synthesise SECURITY LABEL statements during pg_restore; + * those statements need the provider to be registered, otherwise + * ExecSecLabelStmt fails with "security label provider 'spock' is + * not loaded". + */ + register_label_provider(SPOCK_SECLABEL_PROVIDER, spock_object_relabel); + + /* + * Install the 5.x -> 6.x binary-upgrade compatibility ProcessUtility + * hook. Self-gates on IsBinaryUpgrade -- nothing happens outside + * pg_upgrade. + */ + register_spock_compat_5x(); + if (IsBinaryUpgrade) return; @@ -1277,8 +1311,11 @@ _PG_init(void) prev_emit_log_hook = emit_log_hook; emit_log_hook = log_message_filter; - /* Security label provider hook */ - register_label_provider(SPOCK_SECLABEL_PROVIDER, spock_object_relabel); + /* + * Note: the security label provider is registered earlier in + * _PG_init, before the IsBinaryUpgrade early-return, so it is + * available during pg_upgrade for the binary-upgrade compat shims. + */ #if PG_VERSION_NUM >= 180000 /* Spock replication conflict statistics */ diff --git a/src/spock_apply_heap.c b/src/spock_apply_heap.c index 46253d76..d1ed4e84 100644 --- a/src/spock_apply_heap.c +++ b/src/spock_apply_heap.c @@ -510,7 +510,6 @@ physatt_in_attmap(SpockRelation *rel, int attid) return false; } - static void build_delta_tuple(SpockRelation *rel, SpockTupleData *oldtup, SpockTupleData *newtup, @@ -569,7 +568,6 @@ build_delta_tuple(SpockRelation *rel, SpockTupleData *oldtup, } } - /** * This is called when there is a potential conflict that may be able to be resolved * according to resolution rules diff --git a/src/spock_bucompat_5x.c b/src/spock_bucompat_5x.c new file mode 100644 index 00000000..6f3fb371 --- /dev/null +++ b/src/spock_bucompat_5x.c @@ -0,0 +1,449 @@ +/*------------------------------------------------------------------------- + * + * spock_bucompat_5x.c + * Binary-upgrade compatibility: spock 5.x -> 6.x. + * + * During pg_upgrade, pg_dump --binary-upgrade emits the legacy form + * used by spock 5.x: + * + * ALTER TABLE t ALTER COLUMN c SET (log_old_value=true, + * delta_apply_function=spock.delta_apply); + * + * The patches that taught core to recognise those reloption names + * live in patches/attic/. spock 6.x records the same intent as a + * security label with provider 'spock': + * + * SECURITY LABEL FOR spock ON COLUMN t.c IS 'spock.delta_apply'; + * + * On pg_restore against the new cluster we install a ProcessUtility + * hook that intercepts each AlterTableCmd carrying the legacy + * DefElem names, drops them from the cmd, and synthesises the + * equivalent SecLabelStmt. Unrelated DefElems (e.g. fillfactor) + * on the same SET clause survive untouched. + * + * Self-contained: the hook is installed only when IsBinaryUpgrade + * is true at module-load time, so the normal DDL path pays nothing. + * + * Retirement: delete this file and the register_spock_compat_5x() + * call from spock.c's _PG_init(). Two edits. + * + * Copyright (c) 2022-2026, pgEdge, Inc. + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "commands/defrem.h" +#include "miscadmin.h" +#include "nodes/makefuncs.h" +#include "nodes/parsenodes.h" +#include "nodes/plannodes.h" +#include "nodes/value.h" +#include "tcop/utility.h" +#include "utils/builtins.h" + +#include "spock.h" + + +/* Legacy reloption names this shim recognises. */ +#define LEGACY_LOG_OLD_VALUE "log_old_value" +#define LEGACY_DELTA_APPLY_FN "delta_apply_function" + + +/* Saved previous ProcessUtility_hook so we chain correctly. */ +static ProcessUtility_hook_type prev_ProcessUtility_hook = NULL; + +/* Forward declarations. */ +static void spock_compat_5x_ProcessUtility(PlannedStmt *pstmt, + const char *queryString, + bool readOnlyTree, + ProcessUtilityContext context, + ParamListInfo params, + QueryEnvironment *queryEnv, + DestReceiver *dest, + QueryCompletion *qc); +static List *rewrite_5x_attoptions(AlterTableStmt *atstmt); +static bool defelem_is_legacy(DefElem *de); +static bool partition_def_list(List *def_in, List **kept_out, List **claimed_out); +static bool claimed_means_clear(List *claimed, bool is_reset); +static SecLabelStmt *make_label_stmt(AlterTableStmt *atstmt, const char *colname, + bool clear); +static void emit_rewrite_notice(AlterTableStmt *atstmt, const char *colname); + + +/* + * register_spock_compat_5x + * Public entry point invoked from spock.c's _PG_init(). + * + * Install the ProcessUtility hook only when running under + * pg_upgrade -- outside that, the legacy reloption form is not + * being replayed and the hook would only add overhead. + */ +void +register_spock_compat_5x(void) +{ + if (!IsBinaryUpgrade) + return; + + prev_ProcessUtility_hook = ProcessUtility_hook; + ProcessUtility_hook = spock_compat_5x_ProcessUtility; +} + + +/* + * spock_compat_5x_ProcessUtility + * Intercept AlterTableStmt during pg_restore and rewrite legacy + * spock-5.x reloptions into SECURITY LABEL form. + * + * For statements we do not recognise, fall through to the previous + * hook (or standard_ProcessUtility) unchanged. When we rewrite, + * we may either keep the (now-trimmed) original parsetree and run + * it followed by the synthesised SecLabelStmt(s), or drop the + * original entirely (when every cmd was legacy-only) and run only + * the synthesised statements. + * + * Recursion: synthesised statements re-enter ProcessUtility, which + * reaches this hook again. The recognition check declines anything + * that is not an AlterTableStmt, so recursion stops at depth 1. + */ +static void +spock_compat_5x_ProcessUtility(PlannedStmt *pstmt, const char *queryString, + bool readOnlyTree, ProcessUtilityContext context, + ParamListInfo params, QueryEnvironment *queryEnv, + DestReceiver *dest, QueryCompletion *qc) +{ + Node *parsetree; + List *synthetic = NIL; + bool skip_original = false; + ListCell *lc; + + Assert(pstmt != NULL); + + parsetree = pstmt->utilityStmt; + + if (parsetree != NULL && IsA(parsetree, AlterTableStmt)) + { + AlterTableStmt *atstmt = (AlterTableStmt *) parsetree; + + synthetic = rewrite_5x_attoptions(atstmt); + + /* + * Every cmd was a legacy-only SET/RESET that we collapsed. Core + * would Assert on an AlterTableStmt with an empty cmds list, so + * skip the original parsetree and run only the synthesised + * SecLabelStmt(s). + */ + skip_original = (synthetic != NIL && atstmt->cmds == NIL); + } + + if (!skip_original) + { + if (prev_ProcessUtility_hook != NULL) + prev_ProcessUtility_hook(pstmt, queryString, readOnlyTree, + context, params, queryEnv, dest, qc); + else + standard_ProcessUtility(pstmt, queryString, readOnlyTree, + context, params, queryEnv, dest, qc); + } + + foreach(lc, synthetic) + { + Node *synth = (Node *) lfirst(lc); + PlannedStmt *synth_pstmt = makeNode(PlannedStmt); + + synth_pstmt->commandType = CMD_UTILITY; + synth_pstmt->canSetTag = false; + synth_pstmt->utilityStmt = synth; + synth_pstmt->stmt_location = -1; + synth_pstmt->stmt_len = 0; + + /* + * queryString = NULL: synthetic statements are not user input and + * must not leak into pg_stat_statements as a sentinel string. + * + * Run via the public ProcessUtility entry point so any other + * registered hooks see synthetic statements. The recognition + * check at the top of this function declines anything that is + * not an AlterTableStmt, so the recursion is bounded. + */ + ProcessUtility(synth_pstmt, NULL, false, + PROCESS_UTILITY_SUBCOMMAND, params, queryEnv, + dest, NULL); + } +} + + +/* + * rewrite_5x_attoptions + * Walk an AlterTableStmt's cmds list. For each AT_SetOptions / + * AT_ResetOptions cmd carrying at least one legacy DefElem: + * - replace cmd->def with the kept (non-legacy) DefElems; + * - if kept is empty, drop the cmd from the list; + * - synthesise a SecLabelStmt and append to the result list; + * - emit one NOTICE per rewritten column. + * + * Iteration uses a build-new-list pattern. ListCell pointers stay + * stable, and any error mid-walk leaves the original cmds list + * untouched. + * + * Returns the list of synthetic SecLabelStmts (NIL when no rewrite + * happened). On a non-NIL return the caller must inspect + * atstmt->cmds: if NIL, the original parsetree must be skipped + * (every cmd was legacy-only). + */ +static List * +rewrite_5x_attoptions(AlterTableStmt *atstmt) +{ + List *new_cmds = NIL; + List *synthetic = NIL; + ListCell *lc; + + Assert(atstmt != NULL); + + if (atstmt->cmds == NIL) + return NIL; + + foreach(lc, atstmt->cmds) + { + AlterTableCmd *cmd = (AlterTableCmd *) lfirst(lc); + List *kept = NIL; + List *claimed = NIL; + bool is_setopt; + bool is_resetopt; + SecLabelStmt *sl; + bool clear; + + Assert(IsA(cmd, AlterTableCmd)); + + is_setopt = (cmd->subtype == AT_SetOptions); + is_resetopt = (cmd->subtype == AT_ResetOptions); + + /* + * Not relevant to this shim? Preserve verbatim. cmd->def is a + * List of DefElem for these subtypes; the IsA check is defensive + * against a future core change. + */ + if ((!is_setopt && !is_resetopt) || + cmd->def == NULL || !IsA(cmd->def, List) || + !partition_def_list((List *) cmd->def, &kept, &claimed)) + { + new_cmds = lappend(new_cmds, cmd); + continue; + } + + clear = claimed_means_clear(claimed, is_resetopt); + sl = make_label_stmt(atstmt, cmd->name, clear); + synthetic = lappend(synthetic, sl); + + emit_rewrite_notice(atstmt, cmd->name); + + /* + * If unrelated DefElems remain, keep the cmd with a trimmed def + * list. Otherwise drop the cmd -- core would choke on an + * AT_SetOptions cmd with an empty list. + */ + if (kept != NIL) + { + cmd->def = (Node *) kept; + new_cmds = lappend(new_cmds, cmd); + } + /* else: drop the cmd from new_cmds. */ + } + + /* Mutate the original parsetree only if we synthesised something. */ + if (synthetic != NIL) + atstmt->cmds = new_cmds; + + return synthetic; +} + + +/* + * defelem_is_legacy + * True if the DefElem names a legacy spock 5.x reloption. + * + * Case-sensitive: pg_dump emits these names verbatim and core's + * grammar lowercases unquoted identifiers. + */ +static bool +defelem_is_legacy(DefElem *de) +{ + Assert(de != NULL); + Assert(de->defname != NULL); + + return strcmp(de->defname, LEGACY_LOG_OLD_VALUE) == 0 || + strcmp(de->defname, LEGACY_DELTA_APPLY_FN) == 0; +} + + +/* + * partition_def_list + * Split a SET/RESET DefElem list into "kept" (untouched) and + * "claimed" (legacy keys this shim translates). + * + * Builds two new lists rather than mutating the input list while + * iterating it -- the latter is a classic source of subtle bugs + * with PostgreSQL's List API. Returns true iff at least one + * claimed DefElem was found. + */ +static bool +partition_def_list(List *def_in, List **kept_out, List **claimed_out) +{ + ListCell *lc; + List *kept = NIL; + List *claimed = NIL; + + Assert(kept_out != NULL); + Assert(claimed_out != NULL); + + foreach(lc, def_in) + { + DefElem *de = (DefElem *) lfirst(lc); + + Assert(IsA(de, DefElem)); + + if (defelem_is_legacy(de)) + claimed = lappend(claimed, de); + else + kept = lappend(kept, de); + } + + *kept_out = kept; + *claimed_out = claimed; + + return claimed != NIL; +} + + +/* + * claimed_means_clear + * Decide whether the claimed DefElems express an intent to CLEAR + * the delta-apply marker (true) or to SET it (false). + * + * - RESET ( log_old_value, ... ): always clear. + * - SET ( log_old_value=false ) with no other claimed key: clear. + * - Otherwise: set. + */ +static bool +claimed_means_clear(List *claimed, bool is_reset) +{ + ListCell *lc; + bool all_log_old_false = true; + bool saw_log_old_value = false; + + if (is_reset) + return true; + + foreach(lc, claimed) + { + DefElem *de = (DefElem *) lfirst(lc); + + if (strcmp(de->defname, LEGACY_LOG_OLD_VALUE) == 0) + { + saw_log_old_value = true; + /* + * de->arg == NULL means SET (log_old_value) with no value. + * pg_dump never emits this shape; treat it as "set" (matching + * defGetBoolean()'s default of true) without calling + * defGetBoolean on a NULL arg. + */ + if (de->arg == NULL || defGetBoolean(de) != false) + all_log_old_false = false; + } + else + { + /* delta_apply_function present -> intent is to set */ + all_log_old_false = false; + } + } + + return saw_log_old_value && all_log_old_false; +} + + +/* + * make_label_stmt + * Build a SECURITY LABEL FOR spock ON COLUMN . + * IS 'spock.delta_apply' (or IS NULL when clear is true). + * + * Allocations live in CurrentMemoryContext; the per-statement + * message context releases them after the synthesised statement + * runs. + */ +static SecLabelStmt * +make_label_stmt(AlterTableStmt *atstmt, const char *colname, bool clear) +{ + SecLabelStmt *sl; + List *object; + RangeVar *rv; + + Assert(atstmt != NULL); + Assert(colname != NULL); + + rv = atstmt->relation; + Assert(rv != NULL); + Assert(rv->relname != NULL); + + /* + * SECURITY LABEL's object list for OBJECT_COLUMN is + * [ ?, , ] -- see + * get_object_address_attribute() in catalog/objectaddress.c. + */ + object = NIL; + if (rv->schemaname != NULL) + object = lappend(object, makeString(pstrdup(rv->schemaname))); + object = lappend(object, makeString(pstrdup(rv->relname))); + object = lappend(object, makeString(pstrdup(colname))); + + sl = makeNode(SecLabelStmt); + sl->objtype = OBJECT_COLUMN; + sl->object = (Node *) object; + sl->provider = pstrdup(SPOCK_SECLABEL_PROVIDER); + /* + * Canonical label spelling: spock.delta_apply. Must match the string + * written by spock--6.0.0-devel.sql's spock.delta_apply() PL/pgSQL + * helper -- both pg_seclabel rows are read by + * spock_lookup_delta_function() as a function name and resolved via + * FuncnameGetCandidates(). Using the bare "delta_apply" string here + * would silently rely on search_path and produce a different label + * than the documented helper. + */ + sl->label = clear ? NULL : pstrdup("spock.delta_apply"); + + return sl; +} + + +/* + * emit_rewrite_notice + * One NOTICE per rewritten column, naming the table and column. + */ +static void +emit_rewrite_notice(AlterTableStmt *atstmt, const char *colname) +{ + RangeVar *rv = atstmt->relation; + const char *relname = "?"; + char *fullname = NULL; + + Assert(colname != NULL); + + if (rv != NULL && rv->relname != NULL) + { + if (rv->schemaname != NULL) + { + fullname = psprintf("%s.%s", rv->schemaname, rv->relname); + relname = fullname; + } + else + relname = rv->relname; + } + + ereport(NOTICE, + (errcode(ERRCODE_SUCCESSFUL_COMPLETION), + errmsg("spock: rewrote ALTER TABLE %s ALTER COLUMN %s " + "legacy options to SECURITY LABEL", + relname, colname))); + + if (fullname != NULL) + pfree(fullname); +} diff --git a/src/spock_executor.c b/src/spock_executor.c index b5d4f51d..5de98844 100644 --- a/src/spock_executor.c +++ b/src/spock_executor.c @@ -248,6 +248,7 @@ spock_object_access(ObjectAccessType access, { ObjectAccessDrop *drop_arg = (ObjectAccessDrop *) arg; DropBehavior behavior; + bool dropping_spock_extension = false; /* No need to check for internal deletions. */ if ((drop_arg->dropflags & PERFORM_DELETION_INTERNAL) != 0) @@ -257,7 +258,10 @@ spock_object_access(ObjectAccessType access, if (classId == ExtensionRelationId && objectId == get_extension_oid(EXTENSION_NAME, true) && objectId != InvalidOid /* Should not happen but check anyway */ ) + { + dropping_spock_extension = true; dropping_spock_obj = true; + } /* Dropping relation within spock? */ if (classId == RelationRelationId) @@ -278,8 +282,15 @@ spock_object_access(ObjectAccessType access, */ if (dropping_spock_obj) { - /* Need to drop any security labels created by the extension */ - DeleteSecurityLabels(SPOCK_SECLABEL_PROVIDER); + /* + * Wipe spock-provider security labels only when the extension + * itself is being dropped. Dropping individual relations in + * the spock namespace (e.g. internal cleanup during ALTER + * EXTENSION UPDATE that recreates spock.lag_tracker / + * spock.progress) must not touch user-set delta_apply labels. + */ + if (dropping_spock_extension) + DeleteSecurityLabels(SPOCK_SECLABEL_PROVIDER); return; } diff --git a/src/spock_node.c b/src/spock_node.c index 87112f9e..41105260 100644 --- a/src/spock_node.c +++ b/src/spock_node.c @@ -65,9 +65,10 @@ typedef struct NodeTuple #define Anum_node_country 4 #define Anum_node_info 5 -#define Natts_local_node 2 -#define Anum_node_local_id 1 -#define Anum_node_local_node_if 2 +#define Natts_local_node 3 +#define Anum_node_local_id 1 +#define Anum_node_local_node_if 2 +#define Anum_node_local_node_version 3 typedef struct NodeInterfaceTuple { @@ -455,6 +456,7 @@ create_local_node(Oid nodeid, Oid ifid) values[Anum_node_local_id - 1] = ObjectIdGetDatum(nodeid); values[Anum_node_local_node_if - 1] = ObjectIdGetDatum(ifid); + values[Anum_node_local_node_version - 1] = Int32GetDatum(SPOCK_VERSION_NUM); tup = heap_form_tuple(tupDesc, values, nulls); @@ -564,6 +566,63 @@ get_local_node(bool for_update, bool missing_ok) nodeifid = DatumGetObjectId(fastgetattr(tuple, Anum_node_local_node_if, desc, &isnull)); + /* + * Version check. The node_version column was added in Spock 6.0. + * Look up the column by name and verify its type. We cannot rely + * on positional access (Anum constants) because DROP COLUMN leaves + * a gap in the physical layout, and VACUUM FULL renumbers attributes. + * + * Always ERROR regardless of missing_ok -- returning NULL would + * conflate "node not configured" with "node misconfigured", and + * callers are not obliged to check the return value. + */ + { + AttrNumber ver_attnum; + int32 node_version; + + ver_attnum = InvalidAttrNumber; + for (int i = 0; i < desc->natts; i++) + { + Form_pg_attribute att = TupleDescAttr(desc, i); + + if (att->attisdropped) + continue; + if (strcmp(NameStr(att->attname), "node_version") == 0) + { + ver_attnum = att->attnum; + break; + } + } + + if (!AttributeNumberIsValid(ver_attnum)) + { + systable_endscan(scan); + table_close(rel, for_update ? NoLock : RowExclusiveLock); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("spock extension schema outdated"), + errhint("Run ALTER EXTENSION spock UPDATE."))); + } + + Assert(TupleDescAttr(desc, ver_attnum - 1)->atttypid == INT4OID); + + node_version = DatumGetInt32(fastgetattr(tuple, + ver_attnum, + desc, &isnull)); + if (isnull || node_version != SPOCK_VERSION_NUM) + { + systable_endscan(scan); + table_close(rel, for_update ? NoLock : RowExclusiveLock); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("spock version mismatch: " + "node at v%d, binary at v%d", + isnull ? 0 : node_version, + SPOCK_VERSION_NUM), + errhint("Run ALTER EXTENSION spock UPDATE."))); + } + } + systable_endscan(scan); table_close(rel, for_update ? NoLock : RowExclusiveLock); diff --git a/tests/regress/expected/version_guard.out b/tests/regress/expected/version_guard.out new file mode 100644 index 00000000..7d22a587 --- /dev/null +++ b/tests/regress/expected/version_guard.out @@ -0,0 +1,47 @@ +-- +-- Version guard: verify that spock.local_node.node_version protects +-- against binary/schema mismatches. +-- +SELECT * FROM spock_regress_variables() +\gset +\c :provider_dsn +-- The node was created by the regression setup. Verify the +-- node_version column exists and carries the current binary version. +SELECT node_version = spock.spock_version_num() AS version_matches + FROM spock.local_node; + version_matches +----------------- + t +(1 row) + +-- Verify the column has a NOT NULL constraint. +SELECT attnotnull FROM pg_attribute + WHERE attrelid = 'spock.local_node'::regclass AND attname = 'node_version'; + attnotnull +------------ + t +(1 row) + +-- --------------------------------------------------------------- +-- Scenario: version tampered to 0 (simulates stale schema after +-- binary upgrade without ALTER EXTENSION UPDATE). +-- --------------------------------------------------------------- +UPDATE spock.local_node SET node_version = 0; +-- Any operation that calls get_local_node() should fail. +\set VERBOSITY terse +SELECT * FROM spock.node_info(); +ERROR: spock version mismatch: node at v0, binary at v60000 +\set VERBOSITY default +-- Restore before next DDL (autoddl event trigger calls get_local_node). +UPDATE spock.local_node SET node_version = spock.spock_version_num(); +-- --------------------------------------------------------------- +-- Scenario: version set to a future value (simulates binary +-- rollback after schema was already upgraded). +-- --------------------------------------------------------------- +UPDATE spock.local_node SET node_version = 999999; +\set VERBOSITY terse +SELECT * FROM spock.node_info(); +ERROR: spock version mismatch: node at v999999, binary at v60000 +\set VERBOSITY default +-- Restore before DDL. +UPDATE spock.local_node SET node_version = spock.spock_version_num(); diff --git a/tests/regress/sql/version_guard.sql b/tests/regress/sql/version_guard.sql new file mode 100644 index 00000000..072f48e0 --- /dev/null +++ b/tests/regress/sql/version_guard.sql @@ -0,0 +1,43 @@ +-- +-- Version guard: verify that spock.local_node.node_version protects +-- against binary/schema mismatches. +-- +SELECT * FROM spock_regress_variables() +\gset +\c :provider_dsn + +-- The node was created by the regression setup. Verify the +-- node_version column exists and carries the current binary version. +SELECT node_version = spock.spock_version_num() AS version_matches + FROM spock.local_node; + +-- Verify the column has a NOT NULL constraint. +SELECT attnotnull FROM pg_attribute + WHERE attrelid = 'spock.local_node'::regclass AND attname = 'node_version'; + +-- --------------------------------------------------------------- +-- Scenario: version tampered to 0 (simulates stale schema after +-- binary upgrade without ALTER EXTENSION UPDATE). +-- --------------------------------------------------------------- +UPDATE spock.local_node SET node_version = 0; + +-- Any operation that calls get_local_node() should fail. +\set VERBOSITY terse +SELECT * FROM spock.node_info(); +\set VERBOSITY default + +-- Restore before next DDL (autoddl event trigger calls get_local_node). +UPDATE spock.local_node SET node_version = spock.spock_version_num(); + +-- --------------------------------------------------------------- +-- Scenario: version set to a future value (simulates binary +-- rollback after schema was already upgraded). +-- --------------------------------------------------------------- +UPDATE spock.local_node SET node_version = 999999; + +\set VERBOSITY terse +SELECT * FROM spock.node_info(); +\set VERBOSITY default + +-- Restore before DDL. +UPDATE spock.local_node SET node_version = spock.spock_version_num(); diff --git a/tests/tap/schedule b/tests/tap/schedule index a7894dea..4ef1abbf 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -43,4 +43,6 @@ test: 016_sub_disable_missing_relation test: 018_forward_origins test: 018_failover_slots test: 019_stale_fd_epoll_after_conn_death +test: 020_version_safety_net test: 022_rmgr_progress_post_checkpoint_crash +test: 030_pg_upgrade_5x_to_6x diff --git a/tests/tap/t/002_create_subscriber.pl b/tests/tap/t/002_create_subscriber.pl index 78b058fc..159020c4 100755 --- a/tests/tap/t/002_create_subscriber.pl +++ b/tests/tap/t/002_create_subscriber.pl @@ -113,7 +113,10 @@ # Test 13: Insert more data and verify replication system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', "INSERT INTO test_subscription_data (name, value) VALUES ('test3', 300)"; -system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', "SELECT spock.sub_wait_for_sync('test_subscription')"; +my $sync_lsn = `$pg_bin/psql -p $node_ports->[0] -d $dbname -t -A -c "SELECT spock.sync_event()"`; +chomp($sync_lsn); +$sync_lsn =~ s/\s+//g; +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', "CALL spock.wait_for_sync_event(NULL, 'n1', '$sync_lsn'::pg_lsn, 60)"; my $count_subscriber_updated = `$pg_bin/psql -p $node_ports->[1] -d $dbname -t -c "SELECT COUNT(*) FROM test_subscription_data"`; chomp($count_subscriber_updated); diff --git a/tests/tap/t/020_version_safety_net.pl b/tests/tap/t/020_version_safety_net.pl new file mode 100644 index 00000000..dbea9838 --- /dev/null +++ b/tests/tap/t/020_version_safety_net.pl @@ -0,0 +1,130 @@ +use strict; +use warnings; +use Test::More tests => 10; + +use lib '.'; +use lib 't'; +use SpockTest qw( + create_cluster + destroy_cluster + scalar_query + psql_or_bail + get_test_config +); + +# ============================================================================= +# Version safety-net tests +# +# Exercises the node_version column in spock.local_node and the +# corresponding check inside get_local_node(). +# +# IMPORTANT: Spock's autoddl event trigger calls get_local_node() on +# every DDL statement. So we must always restore node_version to the +# correct value BEFORE issuing any DDL (ALTER TABLE, DROP COLUMN, etc.). +# Only set it to a wrong value right before the SELECT that tests the +# check. +# ============================================================================= + +create_cluster(1, 'Version safety-net test cluster'); + +my $cfg = get_test_config(); +my $PG_BIN = $cfg->{pg_bin}; + +# ----------------------------------------------------------------- +# Scenario 1: fresh install -- node_version matches, operations work. +# ----------------------------------------------------------------- +note("Scenario 1: version matches after create_node"); + +my $ver = scalar_query(1, "SELECT node_version FROM spock.local_node"); +isnt($ver, '0', "node_version is non-zero after create_node"); +isnt($ver, '', "node_version is not empty"); + +# spock.node_info() calls get_local_node(false, false) internally. +my $node_name = scalar_query(1, + "SELECT node_name FROM spock.node_info()"); +is($node_name, 'n1', "node_info succeeds with correct version"); + +# ----------------------------------------------------------------- +# Scenario 2: version tampered to 0 -- simulates stale schema. +# ----------------------------------------------------------------- +note("Scenario 2: node_version set to 0 (stale schema)"); + +psql_or_bail(1, "UPDATE spock.local_node SET node_version = 0"); + +my $output = psql_expect_error(1, + "SELECT node_name FROM spock.node_info()"); +like($output, qr/version mismatch/i, + "error mentions version mismatch"); +like($output, qr/ALTER EXTENSION spock UPDATE/, + "error hints to run ALTER EXTENSION UPDATE"); + +# Restore before next scenario (DDL triggers get_local_node via autoddl). +psql_or_bail(1, + "UPDATE spock.local_node SET node_version = spock.spock_version_num()"); + +# ----------------------------------------------------------------- +# Scenario 3: version from the future -- simulates binary rollback. +# ----------------------------------------------------------------- +note("Scenario 3: node_version higher than binary (rollback)"); + +psql_or_bail(1, "UPDATE spock.local_node SET node_version = 999999"); + +$output = psql_expect_error(1, + "SELECT node_name FROM spock.node_info()"); +like($output, qr/version mismatch.*999999/, + "error includes the future version number"); + +# Restore before DDL. +psql_or_bail(1, + "UPDATE spock.local_node SET node_version = spock.spock_version_num()"); + +# ----------------------------------------------------------------- +# Scenario 4: column dropped -- simulates pre-6.0 schema. +# DDL must happen while version is correct (autoddl check). +# ----------------------------------------------------------------- +note("Scenario 4: node_version column dropped (pre-6.0 schema)"); + +psql_or_bail(1, + "ALTER TABLE spock.local_node DROP COLUMN node_version"); + +$output = psql_expect_error(1, + "SELECT node_name FROM spock.node_info()"); +like($output, qr/schema outdated/i, + "error mentions outdated schema"); +like($output, qr/ALTER EXTENSION spock UPDATE/, + "error hint present for missing column"); + +# ----------------------------------------------------------------- +# Scenario 5: column restored -- operations resume. +# We must disable the autoddl event trigger to run DDL when the +# schema is in a broken state (column missing). +# ----------------------------------------------------------------- +note("Scenario 5: column restored with correct version"); + +# Disable autoddl trigger so ALTER TABLE can proceed without +# get_local_node() firing. +psql_or_bail(1, q{ + ALTER EVENT TRIGGER spock_autoddl DISABLE; + ALTER TABLE spock.local_node + ADD COLUMN node_version int4 NOT NULL DEFAULT 0; + UPDATE spock.local_node + SET node_version = spock.spock_version_num(); + ALTER EVENT TRIGGER spock_autoddl ENABLE; +}); + +$node_name = scalar_query(1, + "SELECT node_name FROM spock.node_info()"); +is($node_name, 'n1', + "node_info succeeds after version restored"); + +destroy_cluster('Version safety-net test cleanup'); + +# ============================================================================= +# Run psql expecting a failure; return combined stdout+stderr. +# ============================================================================= +sub psql_expect_error { + my ($node_num, $sql) = @_; + my $port = $cfg->{node_ports}[$node_num - 1]; + my $result = `$PG_BIN/psql -X -p $port -d regression -t -c "$sql" 2>&1`; + return $result; +} diff --git a/tests/tap/t/030_pg_upgrade_5x_to_6x.pl b/tests/tap/t/030_pg_upgrade_5x_to_6x.pl new file mode 100644 index 00000000..53aece9f --- /dev/null +++ b/tests/tap/t/030_pg_upgrade_5x_to_6x.pl @@ -0,0 +1,622 @@ + +# Drive the actual pg_upgrade path for spock 5.x -> 6.x. This is the +# upgrade.sh scenario, restated as a TAP test using the standard +# PostgreSQL::Test framework. +# +# Bootstrap phase (raw shell, since we are literally building PostgreSQL): +# - Discover the local PostgreSQL repo (spock lives in contrib/spock, +# so its parent's parent is the PG source tree). Clone --shared from +# it twice -- one for old, one for new -- so we never touch the +# network and tags are already in scope. +# - Clone the local spock working tree once. Capture HEAD as the "new" +# ref; flip between $OLD_SPOCK_REF (default origin/v5_STABLE) and +# that captured ref via `git checkout` between builds. +# - For each variant: checkout the matching spock ref, apply +# patches/$PG_MAJOR/*.diff onto the matching PG tree, configure, +# build, install. +# +# Scenario phase (standard PostgreSQL::Test idioms): +# - Two PostgreSQL::Test::Cluster nodes with install_path pointing at +# the freshly-built prefixes. +# - Old node: install spock in two databases, mark a column with the +# legacy 5.x reloption form +# ALTER TABLE t ALTER COLUMN c SET (log_old_value=true, +# delta_apply_function=spock.delta_apply) +# - command_ok pg_upgrade old -> new. spock_bucompat_5x.c rewrites the +# legacy reloption to a SECURITY LABEL during pg_restore. +# - New node: ALTER EXTENSION spock UPDATE in each db, then assert +# pg_seclabel has the expected 'spock' provider row with label +# 'spock.delta_apply' on each marked column. +# +# Each run is a clean build: $TEMP_BASE and the per-node data dirs +# from any prior run are wiped at the start. Expect 5-30 minutes per +# run -- caching across runs sounded useful but in practice masked +# failures by carrying corrupted/half-applied state forward. +# +# Run it the same way as every other spock TAP test, via the spock +# Makefile's check_prove target: +# +# make check_prove PROVE_TESTS=t/030_pg_upgrade_5x_to_6x.pl +# +# That target already exports PG_CONFIG, prepends $(PG_CONFIG --bindir) +# to PATH, and adds PG_PROVE_FLAGS so PostgreSQL::Test::Cluster is +# importable. The test auto-resolves everything else: the PostgreSQL +# source repo (via spock/../..), the PG major (via PG_CONFIG --version), +# and the PG ref (via `git describe --tags --abbrev=0 REL__STABLE`, +# so REL_17_9 on a shipped branch, REL_18_BETA3 mid-cycle). +# +# Tunable via env (all optional; empty strings are ignored): +# PG_CONFIG path to the pg_config of the build +# target. Default: `pg_config` on PATH. +# Set by make check_prove already. +# SPOCK_TEST_PG_REPO path or URL of the PostgreSQL repo to +# clone from. Default: discovered local +# repo at ../.. relative to spock (or, as +# a fallback, `$PG_CONFIG --srcdir`). No +# network fetch in the default path. +# SPOCK_TEST_PG_BRANCH PostgreSQL ref to checkout. Override to +# pin a specific ref (REL_15_8, master, +# my-feature-branch). +# SPOCK_TEST_OLD_SPOCK_REF spock ref for OLD cluster (default +# origin/v5_STABLE; must be present in the +# local clone's remote refs). +# SPOCK_TEST_TEMP_BASE bootstrap work dir. Default: +# /tests/tap/tmp_check/030_pg_upgrade +# (already in spock's .gitignore). Wiped +# at the start of every run. +# SPOCK_TEST_PG_CONFIGURE extra ./configure flags. + +use strict; +use warnings FATAL => 'all'; + +use Cwd qw(getcwd abs_path); +use File::Basename qw(basename); +use File::Path qw(make_path remove_tree); + +# Locate PostgreSQL's TAP perl modules via pg_config so the test runs +# under a plain `prove t/030_*.pl` (e.g. via tests/tap/run_tests.sh) +# without the caller having to pass `-I .../src/test/perl`. The spock +# Makefile's `make check_prove` path passes PG_PROVE_FLAGS for us, but +# the shell wrapper and direct prove invocations do not. +BEGIN +{ + my $pgc = $ENV{PG_CONFIG}; + $pgc = 'pg_config' if !defined $pgc or $pgc eq ''; + + my @candidates; + + my $pgxs = qx('$pgc' --pgxs 2>/dev/null); + chomp $pgxs if defined $pgxs; + if (defined $pgxs and $pgxs ne '') + { + (my $p = $pgxs) =~ s{/src/makefiles/pgxs\.mk$}{/src/test/perl}; + push @candidates, $p; + } + + my $srcdir = qx('$pgc' --srcdir 2>/dev/null); + chomp $srcdir if defined $srcdir; + push @candidates, "$srcdir/src/test/perl" + if defined $srcdir and $srcdir ne ''; + + for my $p (@candidates) + { + if (-f "$p/PostgreSQL/Test/Cluster.pm") + { + unshift @INC, $p; + last; + } + } +} + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- +# Treat empty strings the same as undef. GitHub Actions and similar CI +# systems often expand inputs into env vars verbatim, so a workflow that +# does not pass a value for an optional input ends up exporting the var +# as ''. Plain `//` would not fall back in that case. +sub env_or +{ + my ($name, $default) = @_; + my $v = $ENV{$name}; + return (defined $v and $v ne '') ? $v : $default; +} + +# Path to pg_config: the canonical spock env var PG_CONFIG (used by the +# Makefile too) wins, otherwise rely on `pg_config` on PATH (which +# `make check_prove` sets up via the spock Makefile). If PG_CONFIG is +# an absolute path that does not exist, bail with a specific message -- +# a stale export from a prior shell session is the usual cause, and +# silently falling through to a confusing later error is worse UX than +# naming it here. +sub pg_config_bin +{ + my $explicit = env_or('PG_CONFIG', undef); + return 'pg_config' unless defined $explicit; + if ($explicit =~ m{/} and !-x $explicit) + { + BAIL_OUT("PG_CONFIG='$explicit' but no such executable exists. " + . "Likely a stale export from a previous shell session: " + . "run `unset PG_CONFIG` (so the test falls back to " + . "`pg_config` on PATH) or set PG_CONFIG to a real path."); + } + return $explicit; +} + +my $configure_flags = env_or('SPOCK_TEST_PG_CONFIGURE', + '--without-icu --without-readline --without-zlib'); +# $old_spock_ref and $temp_base are computed below, after the local +# spock repo is known. + +# Locate a local PostgreSQL git repo to clone from. +sub discover_local_pg_repo +{ + my ($spock_repo) = @_; + + my @candidates; + push @candidates, $ENV{PG_SRCDIR} if defined $ENV{PG_SRCDIR}; + push @candidates, abs_path("$spock_repo/../.."); + push @candidates, abs_path("$spock_repo/../postgres"); + + for my $c (@candidates) + { + next unless defined $c and $c ne ''; + return $c if -d "$c/.git" and -d "$c/src/backend"; + } + + return undef; +} + +# Major from pg_config. PG_CONFIG env var wins; otherwise we look up +# `pg_config` on PATH (which `make check_prove` sets up via the spock +# Makefile). For an explicit SPOCK_TEST_PG_BRANCH override we recover +# the major from the ref itself. +sub detect_pg_major_from_pg_config +{ + my $pgc = pg_config_bin(); + my $ver = qx('$pgc' --version 2>/dev/null); + return undef if $? != 0; + return $1 if $ver =~ /\bPostgreSQL\s+(\d+)/; + return undef; +} + +# Latest tag reachable from a given branch. Both PG and spock tag +# every release on their respective STABLE branches, so `git describe +# --tags --abbrev=0` lands on the most recent tag automatically -- +# REL_17_9 on a shipped PG, v5.0.7 on origin/v5_STABLE, BETA tags +# mid-cycle. Returns undef if the branch is missing or has no tag +# reachable -- caller falls back to the branch name itself. +sub latest_tag_on_branch +{ + my ($branch, $local_repo) = @_; + my $tag = qx(git -C '$local_repo' describe --tags --abbrev=0 '$branch' 2>/dev/null); + return undef if $? != 0; + chomp $tag; + return $tag eq '' ? undef : $tag; +} + +# Locate the local spock working tree. +my $cwd = getcwd(); +my $local_spock_repo; +if ($cwd =~ m{^(/.+?)/tests/tap/t/?$}) { $local_spock_repo = $1; } +elsif ($cwd =~ m{^(/.+?)/tests/tap/?$}) { $local_spock_repo = $1; } +else { $local_spock_repo = abs_path($cwd); } + +BAIL_OUT("cannot find local spock working tree at '$local_spock_repo' " + . "(no Makefile)") + unless -f "$local_spock_repo/Makefile"; + +# Default work dir: under tests/tap/tmp_check, alongside other spock TAP +# state. tmp_check is in spock's .gitignore and is not in EXTRA_CLEAN, +# so the cache survives `make clean`. +my $temp_base = env_or('SPOCK_TEST_TEMP_BASE', + "$local_spock_repo/tests/tap/tmp_check/030_pg_upgrade"); + +my $old_spock_ref = env_or('SPOCK_TEST_OLD_SPOCK_REF', 'origin/v5_STABLE'); + +# Discover (or accept an override of) the local PostgreSQL repo. +my $pg_repo = env_or('SPOCK_TEST_PG_REPO', undef) + // discover_local_pg_repo($local_spock_repo) + // BAIL_OUT('cannot locate local PostgreSQL source repo. spock is ' + . 'normally cloned under contrib/spock so its parent is the PG ' + . 'tree; if your layout differs, set SPOCK_TEST_PG_REPO to a path ' + . 'or URL.'); + +# Resolve the major (env override -> pg_config) and the ref to checkout +# (env override -> latest stable tag in the local repo -> STABLE branch). +my $env_branch = env_or('SPOCK_TEST_PG_BRANCH', undef); +my $pg_major; +if (defined $env_branch) +{ + ($pg_major) = ($env_branch =~ /^REL_?(\d+)/); + BAIL_OUT("cannot derive PG major version from " + . "SPOCK_TEST_PG_BRANCH='$env_branch'") + unless $pg_major; +} +else +{ + $pg_major = detect_pg_major_from_pg_config() + or BAIL_OUT('cannot determine PG major: pg_config did not return ' + . "a version. Set PG_CONFIG to a valid pg_config path, or set " + . "SPOCK_TEST_PG_BRANCH explicitly (current PG_CONFIG=" + . (env_or('PG_CONFIG', '')) . ")."); +} + +my $pg_branch = $env_branch + // latest_tag_on_branch("REL_${pg_major}_STABLE", $pg_repo) + // latest_tag_on_branch('HEAD', $pg_repo) + // 'HEAD'; + +BAIL_OUT("local spock has no patches/$pg_major (need patches for the " + . "PG major being tested)") + unless -d "$local_spock_repo/patches/$pg_major"; + +# Pre-flight: verify both refs we are about to depend on actually resolve +# in their respective repos. CI runners commonly checkout shallow or with +# limited refs, so origin/v5_STABLE may be absent unless the workflow +# unshallowed or fetched it. Fail here -- with a clear hint -- rather +# than minutes into the build when `git checkout` finally errors out. +sub git_ref_exists +{ + my ($repo, $ref) = @_; + return system( + "git -C '$repo' rev-parse --verify --quiet '$ref' >/dev/null 2>&1") + == 0; +} + +unless (git_ref_exists($local_spock_repo, $old_spock_ref)) +{ + if ($old_spock_ref =~ m{^origin/(.+)$}) + { + my $branch = $1; + note("ref '$old_spock_ref' missing locally; " + . "fetching tip of '$branch' from origin"); + system("git -C '$local_spock_repo' fetch --no-tags --depth=1 " + . "origin '$branch:refs/remotes/origin/$branch' " + . ">/dev/null 2>&1"); + } +} + +unless (git_ref_exists($local_spock_repo, $old_spock_ref)) +{ + my $how_set = $ENV{SPOCK_TEST_OLD_SPOCK_REF} + ? "from SPOCK_TEST_OLD_SPOCK_REF" + : "the default (origin/v5_STABLE)"; + BAIL_OUT("spock ref '$old_spock_ref' ($how_set) not found in " + . "'$local_spock_repo' and could not be fetched. Either the " + . "ref name is wrong (typo?), the clone has no origin remote, " + . "or the runner is offline. The manual recipe is " + . "`git -C $local_spock_repo fetch --no-tags origin " + . "v5_STABLE:refs/remotes/origin/v5_STABLE`."); +} + +unless (git_ref_exists($pg_repo, $pg_branch)) +{ + # The auto-resolution chain ends at 'HEAD' which is always present + # in a non-empty repo, so reaching here means the user explicitly + # named a ref that does not resolve -- treat as a typo and bail. + BAIL_OUT("PostgreSQL ref '$pg_branch' (from SPOCK_TEST_PG_BRANCH) " + . "not found in '$pg_repo'. Either the ref name is wrong " + . "(typo?) or your clone has not fetched it -- a shallow " + . "checkout typically needs " + . "`git -C $pg_repo fetch --tags origin $pg_branch`."); +} + +# Layout under $temp_base. +my $old_pg_src = "$temp_base/old_pg"; +my $new_pg_src = "$temp_base/new_pg"; +my $old_pg_install = "$temp_base/old_pg_install"; +my $new_pg_install = "$temp_base/new_pg_install"; +my $spock_src = "$temp_base/spock"; +my $build_log = "$temp_base/build.log"; + +# Each run starts from scratch. The spock Makefile deliberately keeps +# tmp_check/ between runs (so its other state is preserved), but for +# this test that means a previous failed run can leave both stale build +# artefacts under $temp_base and stale per-node data dirs that initdb +# refuses to overwrite. Clean only the paths owned by this test: +# - $temp_base (build cache) +# - $tap_tmp_check/t___data (Cluster data dirs) +my $testid = basename($0, '.pl'); +my $tap_tmp_check = "$local_spock_repo/tests/tap/tmp_check"; +remove_tree($temp_base) if -d $temp_base; +remove_tree($_) for glob "$tap_tmp_check/t_${testid}_*_data"; +make_path($temp_base); +{ open my $fh, '>', $build_log or die "open $build_log: $!"; close $fh; } + +# --------------------------------------------------------------------------- +# Bootstrap helpers (raw shell - we are building PostgreSQL itself) +# --------------------------------------------------------------------------- +sub run_build +{ + my (@cmd) = @_; + my $cmd_str = join(' ', @cmd); + note("BUILD: $cmd_str"); + my $rc = system("($cmd_str) >>'$build_log' 2>&1"); + if ($rc != 0) + { + diag("build step failed (exit " + . ($rc >> 8) . "): $cmd_str"); + diag("--- last 60 lines of $build_log ---"); + diag(qx(tail -n 60 '$build_log')); + return 0; + } + return 1; +} + +sub ok_or_bail +{ + my ($cond, $name) = @_; + BAIL_OUT("bootstrap step failed: $name") unless ok($cond, $name); +} + +sub clone_shared_if_missing +{ + my ($src, $dest) = @_; + return 1 if -d "$dest/.git"; + return 0 unless run_build("git clone --shared '$src' '$dest'"); + + return run_build("git -C '$dest' fetch --update-shallow '$src' " + . "'+refs/remotes/origin/*:refs/remotes/origin/*'"); +} + +# Clone --shared and checkout a specific ref. Idempotent: a re-run sees +# the existing .git/ and skips both. If the user changes +# SPOCK_TEST_PG_BRANCH between runs they need to wipe $TEMP_BASE -- we +# do not force-checkout, since the working tree carries our applied +# patches as unstaged changes after the first run. +sub clone_shared_and_checkout +{ + my ($src, $dest, $ref) = @_; + return 1 if -d "$dest/.git"; + return run_build("git clone --shared '$src' '$dest'") + && run_build("cd '$dest' && git checkout --quiet $ref"); +} + +sub apply_patches_if_pristine +{ + my ($pg_src, $patch_dir, $label) = @_; + my $marker = "$pg_src/.spock_patches_applied"; + return 1 if -f $marker; + unless (-d $patch_dir) + { + diag("missing patch dir: $patch_dir"); + return 0; + } + opendir(my $dh, $patch_dir) or die "opendir $patch_dir: $!"; + my @patches = sort grep { /\.diff$/ } readdir($dh); + closedir($dh); + for my $p (@patches) + { + note("$label: applying $p"); + # -N -f forces forward-only, never-prompt mode. Without these, + # macOS BSD patch can hit its "Reversed (or previously applied) + # patch detected! Assume -R? [y]" heuristic when a hunk's + # context lives near EOF, auto-answer yes against piped stdin, + # silently skip the hunk, and still return exit 0 -- yielding + # half-applied patchsets where a marker says "applied" but the + # file wasn't touched. -N -f turns that into a clean exit-1. + return 0 + unless run_build( + "cd '$pg_src' && patch -p1 -N -f < '$patch_dir/$p'"); + } + open my $fh, '>', $marker or die "marker $marker: $!"; + close $fh; + return 1; +} + +sub build_pg_if_missing +{ + my ($src, $install) = @_; + return 1 if -x "$install/bin/postgres"; + return run_build( + "cd '$src' && ./configure --prefix='$install' $configure_flags") + && run_build("cd '$src' && make -j4 install"); +} + +sub build_spock_if_missing +{ + my ($pg_install) = @_; + my $pgcfg = "$pg_install/bin/pg_config"; + my $libdir = qx('$pgcfg' --pkglibdir); + chomp $libdir; + return 1 if -f "$libdir/spock.so" or -f "$libdir/spock.dylib"; + + # Force a clean: spock objects from the previous variant's build are + # linked against the other PG. + return run_build("cd '$spock_src' && make clean PG_CONFIG='$pgcfg' || true") + && run_build("cd '$spock_src' && make PG_CONFIG='$pgcfg'") + && run_build("cd '$spock_src' && make install PG_CONFIG='$pgcfg'"); +} + +sub build_variant +{ + my ($variant, $spock_ref, $pg_src, $pg_install) = @_; + + ok_or_bail( + run_build("cd '$spock_src' && git checkout --quiet --force $spock_ref"), + "$variant: checkout spock $spock_ref"); + ok_or_bail( + apply_patches_if_pristine( + $pg_src, "$spock_src/patches/$pg_major", "$variant PG"), + "$variant: apply spock patches to PG"); + ok_or_bail(build_pg_if_missing($pg_src, $pg_install), + "$variant: build+install PostgreSQL"); + ok_or_bail(build_spock_if_missing($pg_install), + "$variant: build+install spock"); +} + +# --------------------------------------------------------------------------- +# Prerequisites +# --------------------------------------------------------------------------- +for my $tool (qw(git patch make)) +{ + plan skip_all => "required tool '$tool' not found in PATH" + unless system("which $tool >/dev/null 2>&1") == 0; +} + +note("PG repo: $pg_repo"); +note("PG ref: $pg_branch (major $pg_major" + . ($env_branch ? ", explicit" : ", auto-resolved") + . ")"); +note("Old spock ref: $old_spock_ref"); +note("Local spock repo: $local_spock_repo"); +note("Temp base: $temp_base"); +note("Build log: $build_log"); + +# --------------------------------------------------------------------------- +# Bootstrap: clone PG once from the local repo, mirror for the new tree, +# clone spock once, then build each variant. +# --------------------------------------------------------------------------- +ok_or_bail(clone_shared_and_checkout($pg_repo, $old_pg_src, $pg_branch), + "clone PostgreSQL from $pg_repo @ $pg_branch"); +ok_or_bail(clone_shared_if_missing($old_pg_src, $new_pg_src), + "mirror PostgreSQL tree for new build (--shared)"); +ok_or_bail(clone_shared_if_missing($local_spock_repo, $spock_src), + "clone local spock working tree"); + +my $new_spock_ref = qx(cd '$spock_src' && git rev-parse HEAD); +chomp $new_spock_ref; +BAIL_OUT("could not capture HEAD of $spock_src") + unless $new_spock_ref =~ /^[0-9a-f]{40}$/; +note("New spock ref: $new_spock_ref (captured from local HEAD)"); + +build_variant('old', $old_spock_ref, $old_pg_src, $old_pg_install); +build_variant('new', $new_spock_ref, $new_pg_src, $new_pg_install); + +# PostgreSQL::Test::Cluster->init() invokes +# `$ENV{PG_REGRESS} --config-auth ` +# to set up pg_hba.conf. The spock Makefile sets PG_REGRESS via +# PG_REGRESS='$(top_builddir)/src/test/regress/pg_regress' +# but for a PGXS extension `top_builddir` resolves to a path that does +# not contain pg_regress, so $ENV{PG_REGRESS} ends up pointing at a +# non-existent file (or empty string), and Cluster's `system_log` on +# that path warns "Use of uninitialized value" and dies. Point it at +# the pg_regress we just built instead -- both variants ship one in +# their source tree after `make install` completes. +my $built_regress = "$new_pg_src/src/test/regress/pg_regress"; +BAIL_OUT("expected pg_regress at '$built_regress' but file is missing") + unless -x $built_regress; +$ENV{PG_REGRESS} = $built_regress; + +# --------------------------------------------------------------------------- +# Set up two clusters via PostgreSQL::Test::Cluster +# --------------------------------------------------------------------------- +my $old_node = PostgreSQL::Test::Cluster->new('spock_old', + install_path => $old_pg_install); +my $new_node = PostgreSQL::Test::Cluster->new('spock_new', + install_path => $new_pg_install); + +# Use a stable locale/encoding so pg_upgrade does not refuse to run. +my @initdb_extra = ('--locale', 'C', '--encoding', 'UTF8'); + +$old_node->init(extra => \@initdb_extra); +$new_node->init(extra => \@initdb_extra); + +my $spock_conf = q{ +wal_level = logical +shared_preload_libraries = 'spock' +track_commit_timestamp = on +max_replication_slots = 10 +max_wal_senders = 10 +max_worker_processes = 20 +}; +$old_node->append_conf('postgresql.conf', $spock_conf); +$new_node->append_conf('postgresql.conf', $spock_conf); + +# --------------------------------------------------------------------------- +# Old cluster: legacy 5.x delta-apply reloption in two databases +# --------------------------------------------------------------------------- +$old_node->start; + +$old_node->safe_psql('postgres', 'CREATE DATABASE regression'); +$old_node->safe_psql('postgres', 'CREATE DATABASE spock_2'); + +my %legacy_marked = (regression => 'test', spock_2 => 'test_1'); +for my $db (sort keys %legacy_marked) +{ + my $tbl = $legacy_marked{$db}; + $old_node->safe_psql($db, 'CREATE EXTENSION spock'); + $old_node->safe_psql($db, "CREATE TABLE $tbl (x serial primary key)"); + $old_node->safe_psql($db, + "ALTER TABLE $tbl ALTER COLUMN x SET " + . "(log_old_value=true, delta_apply_function=spock.delta_apply)"); +} + +my $extver; + +$extver = $old_node->safe_psql('regression', 'SELECT spock.spock_version()'); +like($extver, qr/^5\./, "old cluster runs spock 5.x ($extver)"); + +$old_node->stop; + +# --------------------------------------------------------------------------- +# pg_upgrade old -> new +# --------------------------------------------------------------------------- +command_ok( + [ + "$new_pg_install/bin/pg_upgrade", + '--no-sync', + '-d', $old_node->data_dir, + '-D', $new_node->data_dir, + '-b', "$old_pg_install/bin", + '-B', "$new_pg_install/bin", + '-p', $old_node->port, + '-P', $new_node->port, + ], + 'pg_upgrade old -> new'); + +$new_node->start; + +# --------------------------------------------------------------------------- +# New cluster: ALTER EXTENSION UPDATE; verify the shim left the seclabel +# --------------------------------------------------------------------------- +for my $db (sort keys %legacy_marked) +{ + my $tbl = $legacy_marked{$db}; + + # Spock auto-upgrade should make the UPDATE automatically during the start + $extver = $new_node->safe_psql($db, 'SELECT spock.spock_version()'); + like($extver, qr/^6\./, "$db: spock version is 6.x after upgrade ($extver)"); + + $new_node->safe_psql($db, 'ALTER EXTENSION spock UPDATE'); + + $extver = $new_node->safe_psql($db, 'SELECT spock.spock_version()'); + like($extver, qr/^6\./, "$db: spock version is 6.x after upgrade ($extver)"); + + # Visible signature of spock_bucompat_5x.c: a pg_seclabel row scoped + # to the spock provider on the column we marked in the old cluster. + my $cnt = $new_node->safe_psql( + $db, qq{ + SELECT count(*) + FROM pg_seclabel sl + JOIN pg_class c ON c.oid = sl.objoid + WHERE sl.provider = 'spock' + AND c.relname = '$tbl' + }); + cmp_ok($cnt, '>=', 1, + "$db: legacy reloption on $tbl rewritten as spock security label"); + + my $label = $new_node->safe_psql( + $db, qq{ + SELECT label + FROM pg_seclabel sl + JOIN pg_class c ON c.oid = sl.objoid + WHERE sl.provider = 'spock' + AND c.relname = '$tbl' + LIMIT 1 + }); + is($label, 'spock.delta_apply', + "$db: seclabel value is the canonical 'spock.delta_apply'"); +} + +$new_node->stop; + +note("artefacts left under $temp_base for re-runs / inspection"); +note("delete $temp_base to force a clean rebuild"); + +done_testing();