diff --git a/mysql-test/suite/rpl/r/parallel_backup.result b/mysql-test/suite/rpl/r/parallel_backup.result index 361838927e91e..87b7d5965ea94 100644 --- a/mysql-test/suite/rpl/r/parallel_backup.result +++ b/mysql-test/suite/rpl/r/parallel_backup.result @@ -29,9 +29,9 @@ BACKUP STAGE START; BACKUP STAGE BLOCK_COMMIT; connection aux_slave; ROLLBACK; +connection slave; connection backup_slave; BACKUP STAGE END; -connection slave; include/diff_tables.inc [master:t1,slave:t1] # MDEV-30423: dealock XA COMMIT vs BACKUP # @@ -64,6 +64,7 @@ BACKUP STAGE START; BACKUP STAGE BLOCK_COMMIT; connection aux_slave; ROLLBACK; +connection aux_slave; connection backup_slave; BACKUP STAGE END; connection slave; @@ -99,6 +100,7 @@ BACKUP STAGE START; BACKUP STAGE BLOCK_COMMIT; connection aux_slave; ROLLBACK; +connection aux_slave; connection backup_slave; BACKUP STAGE END; connection slave; @@ -140,8 +142,8 @@ connection aux_slave; ROLLBACK; connection backup_slave; BACKUP STAGE END; -connection slave; include/stop_slave.inc +connection slave; SET @@global.innodb_lock_wait_timeout = @sav_innodb_lock_wait_timeout; SET @@global.slave_transaction_retries= @sav_slave_transaction_retries; connection slave; @@ -183,13 +185,112 @@ connection aux_slave; ROLLBACK; connection backup_slave; BACKUP STAGE END; +include/stop_slave.inc +connection slave; +SET @@global.innodb_lock_wait_timeout = @sav_innodb_lock_wait_timeout; +SET @@global.slave_transaction_retries= @sav_slave_transaction_retries; +connection slave; +include/start_slave.inc +include/sync_with_master_gtid.inc +# Normal XA COMMIT +connection slave; +include/stop_slave.inc +connection master; +connection aux_slave; +BEGIN; +INSERT INTO t1 VALUES (110); +connection master1; +INSERT INTO t1 VALUES (110); +include/save_master_gtid.inc +connection master; +XA START '1'; +INSERT INTO t1 VALUES (109); +XA END '1'; +XA PREPARE '1'; +connection master1; +include/save_master_gtid.inc +INSERT INTO t1 VALUES (110 + 1); +connection master; +XA COMMIT '1'; +include/save_master_gtid.inc +connection slave; +SET @sav_innodb_lock_wait_timeout = @@global.innodb_lock_wait_timeout; +SET @sav_slave_transaction_retries = @@global.slave_transaction_retries; +SET @@global.innodb_lock_wait_timeout =2; +SET @@global.slave_transaction_retries=100; +include/start_slave.inc +connection aux_slave; +# Xid '1' must be *not* yet in the output: +XA RECOVER; +formatID gtrid_length bqual_length data +connection backup_slave; +BACKUP STAGE START; +BACKUP STAGE BLOCK_COMMIT; +connection aux_slave; +ROLLBACK; +connection aux_slave; +select @@global.gtid_slave_pos = "0-1-17"; +@@global.gtid_slave_pos = "0-1-17" +1 +connection backup_slave; +BACKUP STAGE END; connection slave; +include/sync_with_master_gtid.inc include/stop_slave.inc +connection slave; SET @@global.innodb_lock_wait_timeout = @sav_innodb_lock_wait_timeout; SET @@global.slave_transaction_retries= @sav_slave_transaction_retries; +# Normal XA ROLLBACK +connection slave; +include/stop_slave.inc +connection master; +connection aux_slave; +BEGIN; +INSERT INTO t1 VALUES (113); +connection master1; +INSERT INTO t1 VALUES (113); +include/save_master_gtid.inc +connection master; +XA START '1'; +INSERT INTO t1 VALUES (112); +XA END '1'; +XA PREPARE '1'; +connection master1; +include/save_master_gtid.inc +INSERT INTO t1 VALUES (113 + 1); +connection master; +XA ROLLBACK '1'; +include/save_master_gtid.inc connection slave; +SET @sav_innodb_lock_wait_timeout = @@global.innodb_lock_wait_timeout; +SET @sav_slave_transaction_retries = @@global.slave_transaction_retries; +SET @@global.innodb_lock_wait_timeout =2; +SET @@global.slave_transaction_retries=100; include/start_slave.inc +connection aux_slave; +# Xid '1' must be *not* yet in the output: +XA RECOVER; +formatID gtrid_length bqual_length data +connection backup_slave; +BACKUP STAGE START; +BACKUP STAGE BLOCK_COMMIT; +connection aux_slave; +ROLLBACK; +connection aux_slave; +select @@global.gtid_slave_pos = "0-1-21"; +@@global.gtid_slave_pos = "0-1-21" +1 +connection backup_slave; +BACKUP STAGE END; +connection slave; include/sync_with_master_gtid.inc +include/stop_slave.inc +connection slave; +SET @@global.innodb_lock_wait_timeout = @sav_innodb_lock_wait_timeout; +SET @@global.slave_transaction_retries= @sav_slave_transaction_retries; +include/stop_slave.inc +SET @@global.slave_parallel_threads= 3; +include/start_slave.inc connection slave; include/stop_slave.inc SET @@global.slave_parallel_threads= @old_parallel_threads; diff --git a/mysql-test/suite/rpl/r/parallel_backup_lsu_off.result b/mysql-test/suite/rpl/r/parallel_backup_lsu_off.result index 08955a772911a..4b57b6889c941 100644 --- a/mysql-test/suite/rpl/r/parallel_backup_lsu_off.result +++ b/mysql-test/suite/rpl/r/parallel_backup_lsu_off.result @@ -32,9 +32,9 @@ BACKUP STAGE START; BACKUP STAGE BLOCK_COMMIT; connection aux_slave; ROLLBACK; +connection slave; connection backup_slave; BACKUP STAGE END; -connection slave; include/diff_tables.inc [master:t1,slave:t1] # MDEV-30423: dealock XA COMMIT vs BACKUP # @@ -67,6 +67,7 @@ BACKUP STAGE START; BACKUP STAGE BLOCK_COMMIT; connection aux_slave; ROLLBACK; +connection aux_slave; connection backup_slave; BACKUP STAGE END; connection slave; @@ -102,6 +103,7 @@ BACKUP STAGE START; BACKUP STAGE BLOCK_COMMIT; connection aux_slave; ROLLBACK; +connection aux_slave; connection backup_slave; BACKUP STAGE END; connection slave; @@ -143,8 +145,8 @@ connection aux_slave; ROLLBACK; connection backup_slave; BACKUP STAGE END; -connection slave; include/stop_slave.inc +connection slave; SET @@global.innodb_lock_wait_timeout = @sav_innodb_lock_wait_timeout; SET @@global.slave_transaction_retries= @sav_slave_transaction_retries; connection slave; @@ -186,13 +188,108 @@ connection aux_slave; ROLLBACK; connection backup_slave; BACKUP STAGE END; +include/stop_slave.inc +connection slave; +SET @@global.innodb_lock_wait_timeout = @sav_innodb_lock_wait_timeout; +SET @@global.slave_transaction_retries= @sav_slave_transaction_retries; +connection slave; +include/start_slave.inc +include/sync_with_master_gtid.inc +include/stop_slave.inc +SET @@global.slave_parallel_threads= 3; +include/start_slave.inc +# Normal XA COMMIT +connection slave; +include/stop_slave.inc +connection master; +connection aux_slave; +BEGIN; +INSERT INTO t1 VALUES (110); +connection master1; +INSERT INTO t1 VALUES (110); +include/save_master_gtid.inc +connection master; +XA START '1'; +INSERT INTO t1 VALUES (109); +XA END '1'; +XA PREPARE '1'; +connection master1; +include/save_master_gtid.inc +INSERT INTO t1 VALUES (110 + 1); +connection master; +XA COMMIT '1'; +include/save_master_gtid.inc +connection slave; +SET @sav_innodb_lock_wait_timeout = @@global.innodb_lock_wait_timeout; +SET @sav_slave_transaction_retries = @@global.slave_transaction_retries; +SET @@global.innodb_lock_wait_timeout =2; +SET @@global.slave_transaction_retries=100; +include/start_slave.inc +connection aux_slave; +# Xid '1' must be *not* yet in the output: +XA RECOVER; +formatID gtrid_length bqual_length data +connection backup_slave; +BACKUP STAGE START; +BACKUP STAGE BLOCK_COMMIT; +connection aux_slave; +ROLLBACK; +connection aux_slave; +include/sync_with_master_gtid.inc +connection backup_slave; +BACKUP STAGE END; connection slave; +include/sync_with_master_gtid.inc include/stop_slave.inc +connection slave; SET @@global.innodb_lock_wait_timeout = @sav_innodb_lock_wait_timeout; SET @@global.slave_transaction_retries= @sav_slave_transaction_retries; +# Normal XA ROLLBACK connection slave; +include/stop_slave.inc +connection master; +connection aux_slave; +BEGIN; +INSERT INTO t1 VALUES (113); +connection master1; +INSERT INTO t1 VALUES (113); +include/save_master_gtid.inc +connection master; +XA START '1'; +INSERT INTO t1 VALUES (112); +XA END '1'; +XA PREPARE '1'; +connection master1; +include/save_master_gtid.inc +INSERT INTO t1 VALUES (113 + 1); +connection master; +XA ROLLBACK '1'; +include/save_master_gtid.inc +connection slave; +SET @sav_innodb_lock_wait_timeout = @@global.innodb_lock_wait_timeout; +SET @sav_slave_transaction_retries = @@global.slave_transaction_retries; +SET @@global.innodb_lock_wait_timeout =2; +SET @@global.slave_transaction_retries=100; include/start_slave.inc +connection aux_slave; +# Xid '1' must be *not* yet in the output: +XA RECOVER; +formatID gtrid_length bqual_length data +connection backup_slave; +BACKUP STAGE START; +BACKUP STAGE BLOCK_COMMIT; +connection aux_slave; +ROLLBACK; +connection aux_slave; +include/sync_with_master_gtid.inc +connection backup_slave; +BACKUP STAGE END; +connection slave; include/sync_with_master_gtid.inc +include/stop_slave.inc +connection slave; +SET @@global.innodb_lock_wait_timeout = @sav_innodb_lock_wait_timeout; +SET @@global.slave_transaction_retries= @sav_slave_transaction_retries; connection slave; include/stop_slave.inc SET @@global.slave_parallel_threads= @old_parallel_threads; diff --git a/mysql-test/suite/rpl/r/parallel_backup_slave_binlog_off.result b/mysql-test/suite/rpl/r/parallel_backup_slave_binlog_off.result index bb00bf9cbf56f..8c89ca3ce07b8 100644 --- a/mysql-test/suite/rpl/r/parallel_backup_slave_binlog_off.result +++ b/mysql-test/suite/rpl/r/parallel_backup_slave_binlog_off.result @@ -32,9 +32,9 @@ BACKUP STAGE START; BACKUP STAGE BLOCK_COMMIT; connection aux_slave; ROLLBACK; +connection slave; connection backup_slave; BACKUP STAGE END; -connection slave; include/diff_tables.inc [master:t1,slave:t1] # MDEV-30423: dealock XA COMMIT vs BACKUP # @@ -67,6 +67,7 @@ BACKUP STAGE START; BACKUP STAGE BLOCK_COMMIT; connection aux_slave; ROLLBACK; +connection aux_slave; connection backup_slave; BACKUP STAGE END; connection slave; @@ -102,6 +103,7 @@ BACKUP STAGE START; BACKUP STAGE BLOCK_COMMIT; connection aux_slave; ROLLBACK; +connection aux_slave; connection backup_slave; BACKUP STAGE END; connection slave; @@ -143,8 +145,8 @@ connection aux_slave; ROLLBACK; connection backup_slave; BACKUP STAGE END; -connection slave; include/stop_slave.inc +connection slave; SET @@global.innodb_lock_wait_timeout = @sav_innodb_lock_wait_timeout; SET @@global.slave_transaction_retries= @sav_slave_transaction_retries; connection slave; @@ -186,13 +188,108 @@ connection aux_slave; ROLLBACK; connection backup_slave; BACKUP STAGE END; +include/stop_slave.inc +connection slave; +SET @@global.innodb_lock_wait_timeout = @sav_innodb_lock_wait_timeout; +SET @@global.slave_transaction_retries= @sav_slave_transaction_retries; +connection slave; +include/start_slave.inc +include/sync_with_master_gtid.inc +include/stop_slave.inc +SET @@global.slave_parallel_threads= 3; +include/start_slave.inc +# Normal XA COMMIT +connection slave; +include/stop_slave.inc +connection master; +connection aux_slave; +BEGIN; +INSERT INTO t1 VALUES (110); +connection master1; +INSERT INTO t1 VALUES (110); +include/save_master_gtid.inc +connection master; +XA START '1'; +INSERT INTO t1 VALUES (109); +XA END '1'; +XA PREPARE '1'; +connection master1; +include/save_master_gtid.inc +INSERT INTO t1 VALUES (110 + 1); +connection master; +XA COMMIT '1'; +include/save_master_gtid.inc +connection slave; +SET @sav_innodb_lock_wait_timeout = @@global.innodb_lock_wait_timeout; +SET @sav_slave_transaction_retries = @@global.slave_transaction_retries; +SET @@global.innodb_lock_wait_timeout =2; +SET @@global.slave_transaction_retries=100; +include/start_slave.inc +connection aux_slave; +# Xid '1' must be *not* yet in the output: +XA RECOVER; +formatID gtrid_length bqual_length data +connection backup_slave; +BACKUP STAGE START; +BACKUP STAGE BLOCK_COMMIT; +connection aux_slave; +ROLLBACK; +connection aux_slave; +include/sync_with_master_gtid.inc +connection backup_slave; +BACKUP STAGE END; connection slave; +include/sync_with_master_gtid.inc include/stop_slave.inc +connection slave; SET @@global.innodb_lock_wait_timeout = @sav_innodb_lock_wait_timeout; SET @@global.slave_transaction_retries= @sav_slave_transaction_retries; +# Normal XA ROLLBACK connection slave; +include/stop_slave.inc +connection master; +connection aux_slave; +BEGIN; +INSERT INTO t1 VALUES (113); +connection master1; +INSERT INTO t1 VALUES (113); +include/save_master_gtid.inc +connection master; +XA START '1'; +INSERT INTO t1 VALUES (112); +XA END '1'; +XA PREPARE '1'; +connection master1; +include/save_master_gtid.inc +INSERT INTO t1 VALUES (113 + 1); +connection master; +XA ROLLBACK '1'; +include/save_master_gtid.inc +connection slave; +SET @sav_innodb_lock_wait_timeout = @@global.innodb_lock_wait_timeout; +SET @sav_slave_transaction_retries = @@global.slave_transaction_retries; +SET @@global.innodb_lock_wait_timeout =2; +SET @@global.slave_transaction_retries=100; include/start_slave.inc +connection aux_slave; +# Xid '1' must be *not* yet in the output: +XA RECOVER; +formatID gtrid_length bqual_length data +connection backup_slave; +BACKUP STAGE START; +BACKUP STAGE BLOCK_COMMIT; +connection aux_slave; +ROLLBACK; +connection aux_slave; +include/sync_with_master_gtid.inc +connection backup_slave; +BACKUP STAGE END; +connection slave; include/sync_with_master_gtid.inc +include/stop_slave.inc +connection slave; +SET @@global.innodb_lock_wait_timeout = @sav_innodb_lock_wait_timeout; +SET @@global.slave_transaction_retries= @sav_slave_transaction_retries; connection slave; include/stop_slave.inc SET @@global.slave_parallel_threads= @old_parallel_threads; diff --git a/mysql-test/suite/rpl/r/parallel_backup_xa_debug.result b/mysql-test/suite/rpl/r/parallel_backup_xa_debug.result index cd3b335b540e0..ffd7050268a3c 100644 --- a/mysql-test/suite/rpl/r/parallel_backup_xa_debug.result +++ b/mysql-test/suite/rpl/r/parallel_backup_xa_debug.result @@ -26,6 +26,12 @@ backup stage start; backup stage block_commit; connection slave; SET debug_sync = 'now SIGNAL continue_worker'; +select * from t where a=1; +a +1 +xa recover; +formatID gtrid_length bqual_length data +1 1 0 x SET debug_sync = RESET; connection slave1; backup stage end; diff --git a/mysql-test/suite/rpl/r/rpl_parallel_backup_waits_worker_retry.result b/mysql-test/suite/rpl/r/rpl_parallel_backup_waits_worker_retry.result new file mode 100644 index 0000000000000..0e69075fba431 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_parallel_backup_waits_worker_retry.result @@ -0,0 +1,154 @@ +include/master-slave.inc +[connection master] +# +# MDEV-36025 backup taken from parallel replica may contain prepared transactions +# +connection master; +CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE = innodb; +INSERT INTO t1 VALUES (1, 0); +INSERT INTO t1 VALUES (2, 0); +connection slave; +include/stop_slave.inc +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +SET @old_parallel_threads = @@GLOBAL.slave_parallel_threads; +SET @old_parallel_mode = @@GLOBAL.slave_parallel_mode; +SET @@global.slave_parallel_threads= 2; +SET @@global.slave_parallel_mode = 'optimistic'; +connection master; +begin /* trx1/L1 */; +delete from t1 where a = 1; +update t1 set b = 1 where a = 2; +commit; +begin /* trx2/F2 */; +delete from t1 where a = 2; +commit; +connect aux_slave,127.0.0.1,root,,test,$SLAVE_MYPORT,; +BEGIN; +DELETE FROM t1 WHERE a = 1; +connection slave; +include/start_slave.inc +connection aux_slave; +connect backup_slave,127.0.0.1,root,,test,$SLAVE_MYPORT,; +BACKUP STAGE START; +BACKUP STAGE BLOCK_COMMIT; +connection aux_slave; +ROLLBACK; +connection aux_slave; +connection aux_slave; +connection backup_slave; +BACKUP STAGE END; +connection slave; +include/diff_tables.inc [master:t1,slave:t1] +connection slave; +include/stop_slave.inc +SET @@global.slave_parallel_threads= 3; +include/start_slave.inc +connection master; +INSERT INTO t1 VALUES (11,0), (12,0), (13,0), (14,0); +connection master; +connection slave; +include/stop_slave.inc +connection aux_slave; +BEGIN; +DELETE FROM t1 WHERE a = 11; +connection master; +begin /* trx1/L1 */; +delete from t1 where a = 11; +update t1 set b = 1 where a = 12; +commit; +begin /* trx2/F2 */; +delete from t1 where a = 12; +update t1 set b=2 where a=14; +commit; +begin /* trx3/F3 */; +delete from t1 where a = 13; +commit; +connection slave; +include/start_slave.inc +connection aux_slave; +connection backup_slave; +BACKUP STAGE START; +BACKUP STAGE BLOCK_COMMIT; +connection aux_slave; +connect aux_slave2,127.0.0.1,root,,test,$SLAVE_MYPORT,; +BEGIN; +UPDATE t1 SET b=2 WHERE a=14; +connection slave; +connection aux_slave; +ROLLBACK; +connection aux_slave; +connection aux_slave2; +ROLLBACK; +connection backup_slave; +BACKUP STAGE END; +connection master; +connection slave; +include/diff_tables.inc [master:t1,slave:t1] +connection slave; +include/stop_slave.inc +SET @@global.slave_parallel_threads= 4; +include/start_slave.inc +connection master; +INSERT INTO t1 VALUES (21,0), (22,0), (23,0), (24,0); +connection master; +connection slave; +include/stop_slave.inc +connection aux_slave; +BEGIN; +DELETE FROM t1 WHERE a = 21; +connection master; +begin /* trx1/L1 */; +delete from t1 where a = 21; +update t1 set b = 1 where a = 22; +commit; +begin /* trx2/F2 */; +delete from t1 where a = 22; +update t1 set b=2 where a=24; +commit; +begin /* trx3/F3 */; +delete from t1 where a = 23; +commit; +connection slave; +include/start_slave.inc +connection aux_slave; +connection backup_slave; +BACKUP STAGE START; +BACKUP STAGE BLOCK_COMMIT; +connection aux_slave; +connection aux_slave2; +BEGIN; +UPDATE t1 SET b=1 WHERE a=24; +connection slave; +connection aux_slave; +ROLLBACK; +# prove the MDL lock state is about to become /*granted*/ S2,S5(F2); /* waitng */ X3 +connection aux_slave; +connection master; +INSERT INTO t1 VALUES (25,0) /* L4 third slave group commit leader */; +connection slave; +connection aux_slave2; +ROLLBACK; +connection backup_slave; +L4's S6 is waiting +connection slave; +INSERT(25) is backup-locked out +select * from t1 where a = 25; +a b +connection backup_slave; +BACKUP STAGE END; +connection master; +connection slave; +INSERT(25) is done +select * from t1 where a = 25; +a b +25 0 +include/diff_tables.inc [master:t1,slave:t1] +connection slave; +include/stop_slave.inc +SET @@global.slave_parallel_threads= @old_parallel_threads; +SET @@global.slave_parallel_mode = @old_parallel_mode; +include/start_slave.inc +connection server_1; +DROP TABLE t1; +include/rpl_end.inc +# End of the tests diff --git a/mysql-test/suite/rpl/r/rpl_parallel_backup_worker_retry.result b/mysql-test/suite/rpl/r/rpl_parallel_backup_worker_retry.result index a07f8dd68fb56..42d9758432435 100644 --- a/mysql-test/suite/rpl/r/rpl_parallel_backup_worker_retry.result +++ b/mysql-test/suite/rpl/r/rpl_parallel_backup_worker_retry.result @@ -35,6 +35,8 @@ connection aux_slave; ROLLBACK; connection aux_slave; connection backup_slave; +connection aux_slave; +connection backup_slave; BACKUP STAGE END; connection slave; include/diff_tables.inc [master:t1,slave:t1] diff --git a/mysql-test/suite/rpl/t/parallel_backup.test b/mysql-test/suite/rpl/t/parallel_backup.test index 4f5ffa9f29107..16184a5d86110 100644 --- a/mysql-test/suite/rpl/t/parallel_backup.test +++ b/mysql-test/suite/rpl/t/parallel_backup.test @@ -3,6 +3,13 @@ --source include/have_binlog_format_mixed.inc --source include/master-slave.inc +# +# The test for MDEV-21953 is adapted to MDEV-36025 fixes including +# some comments. +# Find in rpl.rpl_parallel_backup_worker_retry more +# of the parallel slave concurrent with BACKUP scenarios. +# + --echo # --echo # MDEV-21953: deadlock between BACKUP STAGE BLOCK_COMMIT and parallel --echo # replication @@ -45,24 +52,30 @@ INSERT INTO t1 VALUES (1); --connection aux_slave --let $wait_condition= SELECT COUNT(*) > 0 FROM information_schema.processlist WHERE state = "Waiting for prior transaction to commit" ---source include/wait_condition.inc +source include/wait_condition.inc; # While the 1st worker is locked out run backup --connect (backup_slave,127.0.0.1,root,,test,$SLAVE_MYPORT,) BACKUP STAGE START; --send BACKUP STAGE BLOCK_COMMIT -# release the 1st work --connection aux_slave ---sleep 1 +# BACKUP is waiting +--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for backup lock" +source include/wait_condition.inc; +--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Update" +source include/wait_condition.inc; +# go trx1,trx2 ROLLBACK; +# .. so the group of two will commit. This proves MDEV-36025 efficacy. +--connection slave +--sync_with_master + --connection backup_slave --reap BACKUP STAGE END; ---connection slave ---sync_with_master --let $diff_tables= master:t1,slave:t1 --source include/diff_tables.inc @@ -93,6 +106,28 @@ BACKUP STAGE END; --source include/start_slave.inc --source include/sync_with_master_gtid.inc +if (!$parallel_backup_without_binlog) +{ +# Test with optimistic retry. +--let $complete = COMMIT +--source parallel_backup_xa2.inc +--let $complete = ROLLBACK +--source parallel_backup_xa2.inc +} + +source include/stop_slave.inc; +SET @@global.slave_parallel_threads= 3; +source include/start_slave.inc; + +if ($parallel_backup_without_binlog) +{ +# Test with optimistic retry. +# --skip-log-bin and --log-slave-updates=0 tests have a bit different scenario. +--let $complete = COMMIT +--source parallel_backup_xa3.inc +--let $complete = ROLLBACK +--source parallel_backup_xa3.inc +} # Clean up. --connection slave diff --git a/mysql-test/suite/rpl/t/parallel_backup_lsu_off.test b/mysql-test/suite/rpl/t/parallel_backup_lsu_off.test index 8e2663077b29e..aaaa71f1022f8 100644 --- a/mysql-test/suite/rpl/t/parallel_backup_lsu_off.test +++ b/mysql-test/suite/rpl/t/parallel_backup_lsu_off.test @@ -4,4 +4,5 @@ --echo # MDEV-21953: deadlock between BACKUP STAGE BLOCK_COMMIT and parallel --echo # MDEV-30423: dealock XA COMMIT vs BACKUP --let $rpl_skip_reset_master_and_slave = 1 +--let $parallel_backup_without_binlog=1 --source parallel_backup.test diff --git a/mysql-test/suite/rpl/t/parallel_backup_slave_binlog_off.test b/mysql-test/suite/rpl/t/parallel_backup_slave_binlog_off.test index 7cefbdb3ddd62..142c87dbcc624 100644 --- a/mysql-test/suite/rpl/t/parallel_backup_slave_binlog_off.test +++ b/mysql-test/suite/rpl/t/parallel_backup_slave_binlog_off.test @@ -4,4 +4,5 @@ --echo # MDEV-21953: deadlock between BACKUP STAGE BLOCK_COMMIT and parallel --echo # MDEV-30423: dealock XA COMMIT vs BACKUP --let $rpl_server_skip_log_bin= 1 +--let $parallel_backup_without_binlog=1 --source parallel_backup.test diff --git a/mysql-test/suite/rpl/t/parallel_backup_xa.inc b/mysql-test/suite/rpl/t/parallel_backup_xa.inc index 5f287aa0ca6f8..cade2324ed06a 100644 --- a/mysql-test/suite/rpl/t/parallel_backup_xa.inc +++ b/mysql-test/suite/rpl/t/parallel_backup_xa.inc @@ -62,20 +62,28 @@ XA RECOVER; --let $rpl_allow_error=1 } ROLLBACK; ---let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for backup lock" ---source include/wait_condition.inc ---connection backup_slave + +if (!$slave_ooo_error) +{ + --connection aux_slave + --let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for backup lock" and command = "Slave_worker" + source include/wait_condition.inc; +} + --connection backup_slave --reap BACKUP STAGE END; ---connection slave + if (!$slave_ooo_error) { + --connection slave --source include/sync_with_master_gtid.inc } + --let $rpl_only_running_threads= 1 --source include/stop_slave.inc if ($slave_ooo_error) { + --connection slave SET @@global.innodb_lock_wait_timeout = @sav_innodb_lock_wait_timeout; SET @@global.slave_transaction_retries= @sav_slave_transaction_retries; } diff --git a/mysql-test/suite/rpl/t/parallel_backup_xa2.inc b/mysql-test/suite/rpl/t/parallel_backup_xa2.inc new file mode 100644 index 0000000000000..bd83dbb6ce6ec --- /dev/null +++ b/mysql-test/suite/rpl/t/parallel_backup_xa2.inc @@ -0,0 +1,103 @@ +# Invoked from parallel_backup.test +# Parameters: +# $complete = COMMIT or ROLLBACK + +# This test conducts T1,XAP2,T3,XAC4 parallel execution with retries +# having a BACKUP command interjecting with its BLOCK_COMMIT prior to +# T1 gets out of its retry and T3 reaches its commit stage. +--let $kind = Normal +--echo # $kind XA $complete + +--connection slave +--source include/stop_slave.inc + +--connection master +# val_0 is the first value to insert on master in prepared xa +# val_1 is the next one to insert which is the value to block on slave +--let $val_0 = `SELECT max(a)+1 FROM t1` +--let $val_1 = $val_0 +--inc $val_1 + +--connection aux_slave +BEGIN; +--eval INSERT INTO t1 VALUES ($val_1) + +# the slave group commit leader transaction will be harassed by above into retry +connection master1; +# conflicting value + --eval INSERT INTO t1 VALUES ($val_1) +--source include/save_master_gtid.inc +--let $save_gtid_1=$master_pos + +--connection master +XA START '1'; +--eval INSERT INTO t1 VALUES ($val_0) +XA END '1'; +XA PREPARE '1'; + +--connection master1 +--source include/save_master_gtid.inc +--let $save_gtid_2=$master_pos +--eval INSERT INTO t1 VALUES ($val_1 + 1) + +--connection master +--eval XA $complete '1' +--source include/save_master_gtid.inc +--let $save_gtid_all=$master_pos + +--connection slave +SET @sav_innodb_lock_wait_timeout = @@global.innodb_lock_wait_timeout; +SET @sav_slave_transaction_retries = @@global.slave_transaction_retries; +SET @@global.innodb_lock_wait_timeout =2; +SET @@global.slave_transaction_retries=100; + +--source include/start_slave.inc +--connection aux_slave +--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for prior transaction to commit" +--source include/wait_condition.inc +--echo # Xid '1' must be *not* yet in the output: +XA RECOVER; + +--connection backup_slave + BACKUP STAGE START; +--send BACKUP STAGE BLOCK_COMMIT + +--connection aux_slave + let $status_var= Slave_retried_transactions; + let $status_var_value= query_get_value(SHOW STATUS LIKE '$status_var', Value, 1); + # wait for T1's re-try get stuck in repeating wait-for-lock.. + --let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Update" + source include/wait_condition.inc; + + let $status_var_comparsion= >; + source include/wait_for_status_var.inc; + # Note T3 is not yet scheduled as each worker is busy, and BLOCK_COMMIT is + # being waited. + --let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for backup lock" and command = "Query" + + ROLLBACK; + + --connection aux_slave + --let $wait_condition= SELECT COUNT(*) = 2 FROM information_schema.processlist WHERE state = "Waiting for backup lock" and command = "Slave_worker" + source include/wait_condition.inc; + # This is MDEV-21469 XA recovery related. The XAP has a post-prepare + # action to insert its gtid and that is blocked. The two waiters for Backup + # are the XAP and its follower INSERT($val_1 + 1) trx. So instead of + #--let $master_pos=$save_gtid_2 + #--source include/sync_with_master_gtid.inc + # the check must hold + --eval select @@global.gtid_slave_pos = "$save_gtid_1" + + --connection backup_slave + --reap + BACKUP STAGE END; + + --connection slave + --let $master_pos=$save_gtid_all + --source include/sync_with_master_gtid.inc + +--source include/stop_slave.inc + + --connection slave + SET @@global.innodb_lock_wait_timeout = @sav_innodb_lock_wait_timeout; + SET @@global.slave_transaction_retries= @sav_slave_transaction_retries; diff --git a/mysql-test/suite/rpl/t/parallel_backup_xa3.inc b/mysql-test/suite/rpl/t/parallel_backup_xa3.inc new file mode 100644 index 0000000000000..88ecd0e9cdd56 --- /dev/null +++ b/mysql-test/suite/rpl/t/parallel_backup_xa3.inc @@ -0,0 +1,108 @@ +# Invoked from parallel_backup.test +# Parameters: +# $complete = COMMIT or ROLLBACK + +# This test conducts T1,XAP2,T3,XAC4 parallel execution with retries +# having a BACKUP command interjecting with its BLOCK_COMMIT prior to +# T1 gets out of its retry and T3 has acquire MDL lock. +# We prove the BACKUP command would not prevent 1,2,3 seq-no team group-commit +# as well as itself complete leaving the 4th seq-no transaction waiting. +--let $kind = Normal +--echo # $kind XA $complete + +--connection slave +--source include/stop_slave.inc + +--connection master +# val_0 is the first value to insert on master in prepared xa +# val_1 is the next one to insert which is the value to block on slave +--let $val_0 = `SELECT max(a)+1 FROM t1` +--let $val_1 = $val_0 +--inc $val_1 + +--connection aux_slave +BEGIN; +--eval INSERT INTO t1 VALUES ($val_1) + +# the slave group commit leader transaction will be harassed by above into retry +connection master1; +# conflicting value + --eval INSERT INTO t1 VALUES ($val_1) +--source include/save_master_gtid.inc +--let $save_gtid_1=$master_pos + +--connection master +XA START '1'; +--eval INSERT INTO t1 VALUES ($val_0) +XA END '1'; +XA PREPARE '1'; + +--connection master1 +--source include/save_master_gtid.inc +--let $save_gtid_2=$master_pos +--eval INSERT INTO t1 VALUES ($val_1 + 1) + +--connection master +--eval XA $complete '1' +--source include/save_master_gtid.inc +--let $save_gtid_all=$master_pos + +--connection slave +SET @sav_innodb_lock_wait_timeout = @@global.innodb_lock_wait_timeout; +SET @sav_slave_transaction_retries = @@global.slave_transaction_retries; +SET @@global.innodb_lock_wait_timeout =2; +SET @@global.slave_transaction_retries=100; + +--source include/start_slave.inc +# XAP has not acquired BACKUP MDL at waiting +--connection aux_slave +--let $wait_condition= SELECT COUNT(*) = 2 FROM information_schema.processlist WHERE state = "Waiting for prior transaction to commit" +--source include/wait_condition.inc +--echo # Xid '1' must be *not* yet in the output: +XA RECOVER; + +--connection backup_slave + BACKUP STAGE START; +--send BACKUP STAGE BLOCK_COMMIT + +--connection aux_slave + let $status_var= Slave_retried_transactions; + let $status_var_value= query_get_value(SHOW STATUS LIKE '$status_var', Value, 1); + # wait for T1's re-try get stuck in repeating wait-for-lock.. + --let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Update" + source include/wait_condition.inc; + + let $status_var_comparsion= >; + source include/wait_for_status_var.inc; + # T3 does not let the backup's BLOCK_COMMIT, so the latter is waiting + --let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for backup lock" and command = "Query" + + # release T1 which is going to + # - leapfrog the backup thread waiting strong lock at retry + # - commit itself, notify XAP2 and T3 that both will share MDL and + # group-commit. + ROLLBACK; + + # Backup will finally get the lock; XAC4 is waiting + --connection aux_slave + --let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for backup lock" and command = "Slave_worker" + source include/wait_condition.inc; + # Unlike --log-bin setup, thanks to T3 timeing to thake the shared MD, + # XAP2 will also acquire the BACKUP MDL and complete with its gtid_seq_no + # insertion part. + --let $master_pos=$save_gtid_2 + --source include/sync_with_master_gtid.inc + + --connection backup_slave + --reap + BACKUP STAGE END; + + --connection slave + --let $master_pos=$save_gtid_all + --source include/sync_with_master_gtid.inc + +--source include/stop_slave.inc + + --connection slave + SET @@global.innodb_lock_wait_timeout = @sav_innodb_lock_wait_timeout; + SET @@global.slave_transaction_retries= @sav_slave_transaction_retries; diff --git a/mysql-test/suite/rpl/t/parallel_backup_xa_debug.test b/mysql-test/suite/rpl/t/parallel_backup_xa_debug.test index e3df0ac69a1d0..b138a3e2154d1 100644 --- a/mysql-test/suite/rpl/t/parallel_backup_xa_debug.test +++ b/mysql-test/suite/rpl/t/parallel_backup_xa_debug.test @@ -26,7 +26,27 @@ xa start 'x'; xa end 'x'; xa prepare 'x'; +# Affects of MDEV-36025 to the user XA and backup. +# In the following master logs two trx:s the 1st is Leader which will be +# delayed into the group commmit and the 2nd is XA PREPAREd Follower that will +# acquire BACKUP MDL share lock before a backup thread will make a stronger +# request it. It will wait. +# Here we prove that the delayed arrival of the Leader to the group commit +# will pass throuh the wating backup X. +# The gtid 100,101 group will complete and the backup thread will be unhang +# upon that. +# HOWEVER, the XAP completion in the slave group commit can't be full. +# Becose of MDEV-21469 specific slave's sub-case that incorporates +# gtid_slave_pos to the user XA prepared replicated trx the XAP might not +# be able to acquire an S BACKUP MDL on this extra statement. That's why +# we prove the group commit with showing data affects +# by Leader and XAP state rather than syncing with master. +# +# Despite of the necessary workaround, we prove the delayed Leader leapfrogs +# a stronger waiting request, the same as it happens in the normal trx case. + --connection slave +# Leader is delayed, Follower awaits it. SET @@global.debug_dbug="+d,hold_worker_on_schedule"; start slave; SET debug_sync = 'now WAIT_FOR reached_pause'; @@ -35,15 +55,33 @@ SET debug_sync = 'now WAIT_FOR reached_pause'; --connection slave1 backup stage start; -backup stage block_commit; +--send backup stage block_commit --connection slave + +# Backup is waiting which means the MDL query: S(g101); X(backup) --let $wait_condition= SELECT count(*) = 1 FROM information_schema.processlist WHERE state LIKE "Waiting for backup lock" -SET debug_sync = 'now SIGNAL continue_worker'; --source include/wait_condition.inc + +# Leader is released to group commit with g101 +SET debug_sync = 'now SIGNAL continue_worker'; + +#--connection slave +# This is a workaround MDEV-21469 relate issue. +let $show_statement = xa recover; +let $field = data; +let $condition = = 'x'; +let $wait_timeout = 30; +--source include/wait_show_condition.inc +select * from t where a=1; +--eval $show_statement +# :sweat-smile: SET debug_sync = RESET; +#--sync_slave_with_master + --connection slave1 +--reap backup stage end; --connection master diff --git a/mysql-test/suite/rpl/t/rpl_parallel_backup_waits_worker_retry.test b/mysql-test/suite/rpl/t/rpl_parallel_backup_waits_worker_retry.test new file mode 100644 index 0000000000000..974942aa69747 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_parallel_backup_waits_worker_retry.test @@ -0,0 +1,387 @@ +--source include/have_innodb.inc +--source include/have_binlog_format_mixed.inc +--source include/master-slave.inc + +--echo # +--echo # MDEV-36025 backup taken from parallel replica may contain prepared transactions +--echo # + +# The tests proves the following. +# +# 1. The patch replaces MDEV-21953/MDEV-23586 W2(S1),B(X2),W1(S3) deadlock +# and its post-fixes MDEV-37453. +# 2. Applies "ignore-priority" requesting and scheduling of BACKUP MDL +# by parallel slave worker threads. +# A. demonstrate a shared lock request by a parallel worker which +# is being waited for (prior-commit) is granted immediately even +# though there exists a waiting X lock in the way. +# B. show fainess of MDL acquision, that is a waiting X by +# BLOCK_COMMIT of backup thread is granted in due time +# that is all slave workers have gathered a group-committable "team". + + +# Legends +# ------- +# +# Threads notations. +# L or Li, F,F1,F2 - Leader and Follower slave group commit transactions, +# the number index indicates binlog ordering (read as gtid seq_no). +# T are utility transaction to help building trx dependencies on slave and +# W are worker threads/transactions when their F/L role is irrelevant. +# +# Locks notations. +# Si, Xi - typical MDL:s that are acquired by the slave worker and the +# backup thread; i stands for ordering of acquiring the BACKUP MDL; +# Si(Fj or Lk) describes the owner of the MDL lock or request. +# Note a parallel slave thread can re-request S lock to advance the index. +# To present the MDL queue we use `;' to separate the granted head from +# waiters that are ordered by `k` index as below. +# Si,... ; Xk, ... +# E,g S1(F2); X2(B) - reads a shared BACKUP MDL is granted to the slave thread +# and there is a backup thread waiter for X. + +# PROOF 1. +# -------- +# The plot. +# It's about having the MDL requests order as +# +# S1(F2); X2(B), S3(L1) +# +# where the Follower will be waiting for (prior commit of) L leader (2 > 1), +# while the latter is delayed. +# Also F -depends-> L in data part and therefore L is going to kill it. +# Upon retry F2 will find X2 lock in the way of its S4 (one more request at its +# retry) which it won't be allowed to bypass. After L1 has committed +# the MDL queue settles into +# +# /* granted */ X2(B); /* waiting*/ S4(F2). +# +# T auxiliary slave local trx orchestrates necessary timings to conduct the plot. +# +# This all proves the safety of concurrent execution of the parallel slave and +# backup thread in a simple setup, reported in the referreced bugs. +# The most importantly we prove the state of the waiting queue of +# ...; X2,S3(L) +# is resolved into +# /* granted */ S3(L); X2. + + +--connection master +CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE = innodb; +INSERT INTO t1 VALUES (1, 0); +INSERT INTO t1 VALUES (2, 0); +--sync_slave_with_master +--source include/stop_slave.inc +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +SET @old_parallel_threads = @@GLOBAL.slave_parallel_threads; +SET @old_parallel_mode = @@GLOBAL.slave_parallel_mode; +SET @@global.slave_parallel_threads= 2; +SET @@global.slave_parallel_mode = 'optimistic'; + +# Proof 1 + +--connection master +begin /* trx1/L1 */; + delete from t1 where a = 1; + update t1 set b = 1 where a = 2; +commit; +begin /* trx2/F2 */; + delete from t1 where a = 2; +commit; + +--save_master_pos + +--connect (aux_slave,127.0.0.1,root,,test,$SLAVE_MYPORT,) +BEGIN; +# block the 1st worker and wait for the 2nd ready to commit + DELETE FROM t1 WHERE a = 1; + +--connection slave +--source include/start_slave.inc + +--connection aux_slave +--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for prior transaction to commit" +--source include/wait_condition.inc + +# While the 1st worker is locked out run backup +--connect (backup_slave,127.0.0.1,root,,test,$SLAVE_MYPORT,) +BACKUP STAGE START; +--send BACKUP STAGE BLOCK_COMMIT + +--connection aux_slave +# Make sure BACKUP lock is waiting +--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for backup lock" +--source include/wait_condition.inc + +# release the 1st work +let $status_var= Slave_retried_transactions; +let $status_var_value= query_get_value(SHOW STATUS LIKE '$status_var', Value, 1); +--sleep 1 +ROLLBACK; + +# that will kick the 2nd out of the current WfPTtC into next retry one +--connection aux_slave +--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for prior transaction to commit" +--source include/wait_condition.inc + +let $status_var_comparsion= >; +--source include/wait_for_status_var.inc + +--connection aux_slave +# Make sure trx1 is waiting for MDL +--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for backup lock" +--source include/wait_condition.inc + +--connection backup_slave +reap; +BACKUP STAGE END; + +--connection slave +--sync_with_master + +--let $diff_tables= master:t1,slave:t1 +--source include/diff_tables.inc + +# PROOF 2.A +# The plot. +# It's about initially having the MDL requests shape the lock as +# /* granted */ S1(F3),S3(F2); /* waiting */ X3(B) +# by time of S4(L1) request. It must bypass X3 and +# in transaction non-dependent case that would end up with +# /* granted */ S1,S3,S4(L); /* waiting*/ X3 +# However here we complicate the matter with the Proof 1's kind of +# the data dependency of F2 -> L1. +# L1 commits alone and in that process killes F2 into retry, +# the latter's S5 (its 2nd that is) request will see the BACKUP MDL state as +# S1(F3); X3, S5(F2) +# so it will be allowed (3 > 2) to leapfrog X3. The state now +# /* granted*/ S1,S5(F2); /* waiting */ X3 +# and immediately, as F2 will turn into the leader role, L2 and F3 commit +# to leave +# X3(granted). +# +# This all futher validates the safety of Proof 1 under more stringent +# condtion of the paralle slave retry and effective functionality of +# jumping over "ignoring" incompatible BACKUP MDL request in the way. + +--connection slave +--source include/stop_slave.inc +SET @@global.slave_parallel_threads= 3; +--source include/start_slave.inc + +--connection master +# Insert new, non-conflicting rows for the 3 workers. +INSERT INTO t1 VALUES (11,0), (12,0), (13,0), (14,0); + +--connection master +--sync_slave_with_master +--source include/stop_slave.inc + +--connection aux_slave +BEGIN; +# block L1 + DELETE FROM t1 WHERE a = 11; + +--connection master +begin /* trx1/L1 */; + delete from t1 where a = 11; + update t1 set b = 1 where a = 12; +commit; +begin /* trx2/F2 */; ## TODO: delay F2 + delete from t1 where a = 12; + update t1 set b=2 where a=14; +commit; +begin /* trx3/F3 */; + delete from t1 where a = 13; +commit; + +--connection slave +--source include/start_slave.inc + +--connection aux_slave +--let $wait_condition= SELECT COUNT(*) = 2 FROM information_schema.processlist WHERE state = "Waiting for prior transaction to commit" +--source include/wait_condition.inc + +# While the 1st worker is locked out run backup +--connection backup_slave +BACKUP STAGE START; +--send BACKUP STAGE BLOCK_COMMIT + +--connection aux_slave +# Make sure BACKUP lock is waiting +--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for backup lock" +--source include/wait_condition.inc + +# Prepare to hang again F2 now at its retry +--connect (aux_slave2,127.0.0.1,root,,test,$SLAVE_MYPORT,) +BEGIN; + send UPDATE t1 SET b=2 WHERE a=14; + +--connection slave +# wait the aux_slave2 spoiler starts waiting for F2 lock, and then.. +--connection aux_slave +--let $wait_condition= SELECT COUNT(*) = 2 FROM information_schema.INNODB_LOCK_WAITS; +--source include/wait_condition.inc + +# .. release L +ROLLBACK; + +# prove the MDL lock state is now /* granted */ S1(F3); /* waiting */ X3 +--connection aux_slave +--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for prior transaction to commit" +--source include/wait_condition.inc +--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for backup lock" and Command = "Query" +--source include/wait_condition.inc +--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Updating" and Command = "Slave_worker" +--source include/wait_condition.inc + +# Finish L2=F2,F3 group +--connection aux_slave2 +--reap +ROLLBACK; + +--connection backup_slave +reap; +BACKUP STAGE END; + +# Correctness proof +--connection master +--sync_slave_with_master + +--let $diff_tables= master:t1,slave:t1 +--source include/diff_tables.inc + +# PROOF 2.B. +# The plot further extends the scenario of the Proof 2.A to introduce +# one more replicated transaction which can't leapfrog the X of backup. +# It's about initally having the MDL queue state as +# /* granted */ S1(F2),S2(F3); /* waiting */ X3(B), +# Along the way to commit L1 kills F2 and bypasses X3. Past F2 retry +# the BACKUP MDL queue state gets about +# S3(F3),S5(F2); X3 +# Before the 2nd group commits S6(by L4) is requested and it won't be able +# to make to the grantees 'cos 4 > 3 and 2. +# In the following the plot 2.A is largely repeated, and in the end +# the BACKUP MDL will have +# /*granted*/ X3; /*waiting*/ S6 +# which proves the ignore-priority scheduler's fairness. + +--connection slave +--source include/stop_slave.inc +SET @@global.slave_parallel_threads= 4; +--source include/start_slave.inc + +--connection master +# Insert new, non-conflicting rows for the 3 workers. +INSERT INTO t1 VALUES (21,0), (22,0), (23,0), (24,0); + +--connection master +--sync_slave_with_master +--source include/stop_slave.inc + +--connection aux_slave +BEGIN; +# block the 1st follower + DELETE FROM t1 WHERE a = 21; + +--connection master +begin /* trx1/L1 */; + delete from t1 where a = 21; + update t1 set b = 1 where a = 22; +commit; +begin /* trx2/F2 */; + delete from t1 where a = 22; + update t1 set b=2 where a=24; +commit; +begin /* trx3/F3 */; + delete from t1 where a = 23; +commit; + +--connection slave +--source include/start_slave.inc + +--connection aux_slave +--let $wait_condition= SELECT COUNT(*) = 2 FROM information_schema.processlist WHERE state = "Waiting for prior transaction to commit" +--source include/wait_condition.inc + +# While the 1st worker is locked out run backup +--connection backup_slave +BACKUP STAGE START; +--send BACKUP STAGE BLOCK_COMMIT + +--connection aux_slave +# Make sure BACKUP lock is waiting +--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for backup lock" +--source include/wait_condition.inc + +# Prepare to hang F2 at its retry +connection aux_slave2; +BEGIN; + --send UPDATE t1 SET b=1 WHERE a=24 + +--connection slave +# wait the aux_slave2 spoiler starts waiting for F2 lock, and then.. +--connection aux_slave +--let $wait_condition= SELECT COUNT(*) = 2 FROM information_schema.INNODB_LOCK_WAITS; +--source include/wait_condition.inc + +# .. release L +ROLLBACK; + +--echo # prove the MDL lock state is about to become /*granted*/ S2,S5(F2); /* waitng */ X3 +--connection aux_slave +# F3 is waiting for the innodb lock waiting F2 +--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for prior transaction to commit" +--source include/wait_condition.inc +--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for backup lock" and Command = "Query" +--source include/wait_condition.inc + +# Before to finish F2/L2,F3 group conduct Proof 3 specfic actions +connection master; +INSERT INTO t1 VALUES (25,0) /* L4 third slave group commit leader */; + +# L4 can't leapfrog +connection slave; +--let $wait_condition= SELECT COUNT(*) = 2 FROM information_schema.processlist WHERE state = "Waiting for backup lock" +--source include/wait_condition.inc + +# Complete L2,F3 +connection aux_slave2; +--reap +ROLLBACK; + +connection backup_slave; +reap; + +--echo L4's S6 is waiting +connection slave; +--echo INSERT(25) is backup-locked out +select * from t1 where a = 25; +--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for backup lock" +--source include/wait_condition.inc + +connection backup_slave; +BACKUP STAGE END; + +# Correctness proof +connection master; +sync_slave_with_master; +--echo INSERT(25) is done +select * from t1 where a = 25; + +--let $diff_tables= master:t1,slave:t1 +--source include/diff_tables.inc + + +# Clean up. +--connection slave +--source include/stop_slave.inc +SET @@global.slave_parallel_threads= @old_parallel_threads; +SET @@global.slave_parallel_mode = @old_parallel_mode; +--source include/start_slave.inc + +--connection server_1 +DROP TABLE t1; + +--source include/rpl_end.inc +--echo # End of the tests diff --git a/mysql-test/suite/rpl/t/rpl_parallel_backup_worker_retry.test b/mysql-test/suite/rpl/t/rpl_parallel_backup_worker_retry.test index f3b6f84e833ad..2ce0e7d0bc43b 100644 --- a/mysql-test/suite/rpl/t/rpl_parallel_backup_worker_retry.test +++ b/mysql-test/suite/rpl/t/rpl_parallel_backup_worker_retry.test @@ -2,6 +2,11 @@ --source include/have_binlog_format_mixed.inc --source include/master-slave.inc +# +# The test for MDEV-37453 is adapted to MDEV-36025 fixes including +# some comments. The latter bug-fixes extends the current scenario into few, +# find rpl.rpl_parallel_backup_worker_retry. +# --echo # --echo # MDEV-37453 Parallel Replication Crash During Backup --echo # @@ -17,8 +22,9 @@ # 2. At this point issue BACKUP commands of which BLOCK_COMMIT will # later force the 2nd transaction to wait for the BACKUP MDL lock. # 3. Release locks to the 1st transaction which would kick the 2nd out -# of waiting-for-prior-commit only to return negative from the -# BACKUP MDL acquisition attempt (found itself killed). +# of waiting-for-prior-commit. +# MDEV-36025: it won't do +# BACKUP MDL re-acquisition attempt along the way to retry. # 4. Finally, prove of safety: the 2nd transaction retries successfully. --connection master @@ -59,7 +65,9 @@ BEGIN; # While the 1st worker is locked out run backup --connect (backup_slave,127.0.0.1,root,,test,$SLAVE_MYPORT,) BACKUP STAGE START; -BACKUP STAGE BLOCK_COMMIT; +# MDEV-36025 changes the slave worker not to release BACKUP MDL which it +# is holding see $wait_condition above. +--send BACKUP STAGE BLOCK_COMMIT # release the 1st work --connection aux_slave @@ -71,14 +79,29 @@ ROLLBACK; # that will kick the 2nd out of the current WfPTtC into next retry one --connection aux_slave --let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for prior transaction to commit" ---source include/wait_condition.inc +source include/wait_condition.inc; + +# at same time BACKUP's request is still waiting +--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for backup lock" +source include/wait_condition.inc; let $status_var_comparsion= >; --source include/wait_for_status_var.inc +# BACKUP gets its lock granted by trx1 +--connection backup_slave +reap; + +# which makes the retrying trx2 wait for it +--connection aux_slave +--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for backup lock" +source include/wait_condition.inc; + +# release BACKUP MDL --connection backup_slave BACKUP STAGE END; +# and we're done --connection slave --sync_with_master diff --git a/sql/handler.cc b/sql/handler.cc index af27370e26133..bc0e0bf1a3a55 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -1791,7 +1791,7 @@ int ha_commit_trans(THD *thd, bool all) &no_rollback); /* rw_trans is TRUE when we in a transaction changing data */ bool rw_trans= is_real_trans && rw_ha_count > 0; - MDL_request mdl_backup; + MDL_request mdl_request, *mdl_ptr= &mdl_request; DBUG_PRINT("info", ("is_real_trans: %d rw_trans: %d rw_ha_count: %d", is_real_trans, rw_trans, rw_ha_count)); @@ -1811,19 +1811,48 @@ int ha_commit_trans(THD *thd, bool all) We allow the owner of FTWRL to COMMIT; we assume that it knows what it does. */ - MDL_REQUEST_INIT(&mdl_backup, MDL_key::BACKUP, "", "", MDL_BACKUP_COMMIT, - MDL_EXPLICIT); - if (!WSREP(thd)) { - if (thd->mdl_context.acquire_lock(&mdl_backup, + rpl_group_info *rgi= thd->rgi_slave; + bool is_parallel_slave= rgi && rgi->is_parallel_exec && + rgi->gtid_sub_id > 0; + + /* + Allocate the slave worker's MDL_request in the transaction + mem-root: the lock may have to outlive ha_commit_trans()'s frame + because, per MDEV-7458, an errored-out parallel-slave worker + defers rollback (and the paired MDL release) to cleanup_context. + */ + if (is_parallel_slave) + mdl_ptr= new (&thd->transaction->mem_root) MDL_request(); + + /* + Parallel-slave: register and decide atomically. See + Backup_commit_team::join_and_decide() for the rule. The matrix + grants MDL_BACKUP_COMMIT_HIGH_PRIO past a waiting BACKUP X-request. + */ + enum_mdl_type lock_type= MDL_BACKUP_COMMIT; + if (is_parallel_slave) + lock_type= rgi->rli->backup_team.join_and_decide( + rgi->current_gtid.domain_id, rgi->gtid_sub_id); + + MDL_REQUEST_INIT(mdl_ptr, MDL_key::BACKUP, "", "", lock_type, + MDL_EXPLICIT); + + if (thd->mdl_context.acquire_lock(mdl_ptr, thd->variables.lock_wait_timeout)) { + if (is_parallel_slave) + rgi->rli->backup_team.leave(rgi->current_gtid.domain_id, + rgi->gtid_sub_id); my_error(ER_ERROR_DURING_COMMIT, MYF(0), 1); ha_rollback_trans(thd, all); DBUG_RETURN(1); } - thd->backup_commit_lock= &mdl_backup; + if (is_parallel_slave) + rgi->rli->backup_team.mark_granted(rgi->current_gtid.domain_id, + rgi->gtid_sub_id); + thd->backup_commit_lock= mdl_ptr; } DEBUG_SYNC(thd, "ha_commit_trans_after_acquire_commit_lock"); } @@ -2073,18 +2102,26 @@ int ha_commit_trans(THD *thd, bool all) thd->rgi_slave->is_parallel_exec); } end: - // reset the pointer to the ticket when it's stack instantiated - if (thd->backup_commit_lock == &mdl_backup) + if (!(thd->rgi_slave && thd->rgi_slave->is_parallel_exec && error) && + (mdl_ptr && thd->backup_commit_lock == mdl_ptr)) { /* We do not always immediately release transactional locks after ha_commit_trans() (see uses of ha_enable_transaction()), thus we release the commit blocker lock as soon as it's not needed. + Errored-out slave parallel worker needs to keep it BACKUP lock + to be released at its cleanup before the MDEV-7458 deferred rollback. */ - if (mdl_backup.ticket) - thd->mdl_context.release_lock(mdl_backup.ticket); - thd->backup_commit_lock= 0; + if (mdl_ptr->ticket) + thd->mdl_context.release_lock(mdl_ptr->ticket); + thd->backup_commit_lock= nullptr; + /* Paired with backup_team.join() above. */ + if (thd->rgi_slave && thd->rgi_slave->is_parallel_exec && + thd->rgi_slave->gtid_sub_id > 0) + thd->rgi_slave->rli->backup_team.leave( + thd->rgi_slave->current_gtid.domain_id, + thd->rgi_slave->gtid_sub_id); } #ifdef WITH_WSREP if (wsrep_is_active(thd) && is_real_trans && !error && @@ -2238,6 +2275,23 @@ commit_one_phase_2(THD *thd, bool all, THD_TRANS *trans, bool is_real_trans) if (is_real_trans) { thd->has_waiter= false; + if (!error && (thd->rgi_slave && thd->rgi_slave->is_parallel_exec) && + thd->backup_commit_lock) + { + DBUG_ASSERT(thd->backup_commit_lock->ticket); + /* + As parallel slave allocates its BACKUP lock in transaction mem-root + the successful commit by slave must be followed with the lock release + before the mem-root is cleaned. + */ + thd->mdl_context.release_lock(thd->backup_commit_lock->ticket); + thd->backup_commit_lock= nullptr; + /* Paired with backup_team.join() in ha_commit_trans. */ + if (thd->rgi_slave->gtid_sub_id > 0) + thd->rgi_slave->rli->backup_team.leave( + thd->rgi_slave->current_gtid.domain_id, + thd->rgi_slave->gtid_sub_id); + } thd->transaction->cleanup(); if (count >= 2) statistic_increment(transactions_multi_engine, LOCK_status); @@ -2403,6 +2457,23 @@ int ha_rollback_trans(THD *thd, bool all) thd->transaction->xid_state.set_error(thd->get_stmt_da()->sql_errno()); thd->has_waiter= false; + if (thd->rgi_slave && thd->rgi_slave->is_parallel_exec && + thd->backup_commit_lock) + { + /* + MDL BACKUP unlocking by parallel slave is done after the transaction + rolls back. The MDL lock must be released before its memory allocation + in transaction root is cleaned. + */ + thd->mdl_context.release_lock(thd->backup_commit_lock->ticket); + thd->backup_commit_lock= nullptr; + /* Paired with backup_team.join() in ha_commit_trans. */ + if (thd->rgi_slave->gtid_sub_id > 0) + thd->rgi_slave->rli->backup_team.leave( + thd->rgi_slave->current_gtid.domain_id, + thd->rgi_slave->gtid_sub_id); + } + thd->transaction->cleanup(); } if (all) diff --git a/sql/log.cc b/sql/log.cc index 7a28fa4f476c8..227c4c43c1211 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -8212,7 +8212,6 @@ MYSQL_BIN_LOG::queue_for_group_commit(group_commit_entry *orig_entry) group_commit_entry *entry, *orig_queue, *last; wait_for_commit *cur; wait_for_commit *wfc; - bool backup_lock_released= 0; int result= 0; THD *thd= orig_entry->thd; DBUG_ENTER("MYSQL_BIN_LOG::queue_for_group_commit"); @@ -8247,22 +8246,6 @@ MYSQL_BIN_LOG::queue_for_group_commit(group_commit_entry *orig_entry) !loc_waitee->commit_started) { PSI_stage_info old_stage; - - /* - Release MDL_BACKUP_COMMIT LOCK while waiting for other threads to - commit. - This is needed to avoid deadlock between the other threads (which not - yet have the MDL_BACKUP_COMMIT_LOCK) and any threads using - BACKUP LOCK BLOCK_COMMIT. - */ - if (thd->backup_commit_lock && thd->backup_commit_lock->ticket && - !backup_lock_released) - { - backup_lock_released= 1; - thd->mdl_context.release_lock(thd->backup_commit_lock->ticket); - thd->backup_commit_lock->ticket= 0; - } - /* By setting wfc->opaque_pointer to our own entry, we mark that we are ready to commit, but waiting for another transaction to commit before @@ -8523,9 +8506,6 @@ MYSQL_BIN_LOG::queue_for_group_commit(group_commit_entry *orig_entry) (orig_queue == NULL) ? "leader" : "participant")); end: - if (backup_lock_released) - thd->mdl_context.acquire_lock(thd->backup_commit_lock, - thd->variables.lock_wait_timeout); DBUG_RETURN(result); } diff --git a/sql/mdl.cc b/sql/mdl.cc index b870cc551df2b..f72f2bd3f6d09 100644 --- a/sql/mdl.cc +++ b/sql/mdl.cc @@ -139,7 +139,8 @@ static const LEX_STRING backup_lock_types[]= { C_STRING_WITH_LEN("MDL_BACKUP_DDL") }, { C_STRING_WITH_LEN("MDL_BACKUP_BLOCK_DDL") }, { C_STRING_WITH_LEN("MDL_BACKUP_ALTER_COPY") }, - { C_STRING_WITH_LEN("MDL_BACKUP_COMMIT") } + { C_STRING_WITH_LEN("MDL_BACKUP_COMMIT") }, + { C_STRING_WITH_LEN("MDL_BACKUP_COMMIT_HIGH_PRIO") } }; @@ -1613,45 +1614,47 @@ MDL_lock::MDL_object_lock::m_waiting_incompatible[MDL_TYPE_END]= The first array specifies if particular type of request can be satisfied if there is granted backup lock of certain type. - Request | Type of active backup lock | - type | S0 S1 S2 S3 S4 F1 F2 D TD SD DD BL AC C | - ----------+---------------------------------------------------------+ - S0 | - - - - - + + + + + + + + + | - S1 | - + + + + + + + + + + + + + | - S2 | - + + + + + + - + + + + + + | - S3 | - + + + + + + - + + - + + + | - S4 | - + + + + + + - + - - + + - | - FTWRL1 | + + + + + + + - - - - + - + | - FTWRL2 | + + + + + + + - - - - + - - | - D | + - - - - - - + + + + + + + | - TD | + + + + + - - + + + + + + + | - SD | + + + + - - - + + + + + + + | - DDL | + + + - - - - + + + + - + + | - BLOCK_DDL | - + + + + + + + + + - + + + | - ALTER_COP | + + + + + - - + + + + + + + | - COMMIT | + + + + - + - + + + + + + + | + Request | Type of active backup lock | + type | S0 S1 S2 S3 S4 F1 F2 D TD SD DD BL AC C R | + ----------+------------------------------------------------------------+ + S0 | - - - - - + + + + + + + + + + | + S1 | - + + + + + + + + + + + + + + | + S2 | - + + + + + + - + + + + + + + | + S3 | - + + + + + + - + + - + + + + | + S4 | - + + + + + + - + - - + + - + | + FTWRL1 | + + + + + + + - - - - + - + + | + FTWRL2 | + + + + + + + - - - - + - - + | + D | + - - - - - - + + + + + + + + | + TD | + + + + + - - + + + + + + + + | + SD | + + + + - - - + + + + + + + + | + DDL | + + + - - - - + + + + - + + + | + BLOCK_DDL | - + + + + + + + + + - + + + + | + ALTER_COP | + + + + + - - + + + + + + + + | + COMMIT | + + + + - + - + + + + + + + + | + COMMIT_HIGH_PRIO| + + + + - + - + + + + + + + + | The second array specifies if particular type of request can be satisfied if there is already waiting request for the backup lock of certain type. I.e. it specifies what is the priority of different lock types. - Request | Pending backup lock | - type | S0 S1 S2 S3 S4 F1 F2 D TD SD DD BL AC C | - ----------+---------------------------------------------------------+ - S0 | + - - - - + + + + + + + + + | - S1 | + + + + + + + + + + + + + + | - S2 | + + + + + + + + + + + + + + | - S3 | + + + + + + + + + + + + + + | - S4 | + + + + + + + + + + + + + + | - FTWRL1 | + + + + + + + + + + + + + + | - FTWRL2 | + + + + + + + + + + + + + + | - D | + - - - - - - + + + + + + + | - TD | + + + + + - - + + + + + + + | - SD | + + + + - - - + + + + + + + | - DDL | + + + - - - - + + + + - + + | - BLOCK_DDL | + + + + + + + + + + + + + + | - ALTER_COP | + + + + + - - + + + + + + + | - COMMIT | + + + + - + - + + + + + + + | + Request | Pending backup lock | + type | S0 S1 S2 S3 S4 F1 F2 D TD SD DD BL AC C R | + ----------+------------------------------------------------------------+ + S0 | + - - - - + + + + + + + + + + | + S1 | + + + + + + + + + + + + + + + | + S2 | + + + + + + + + + + + + + + + | + S3 | + + + + + + + + + + + + + + + | + S4 | + + + + + + + + + + + + + + + | + FTWRL1 | + + + + + + + + + + + + + + + | + FTWRL2 | + + + + + + + + + + + + + + + | + D | + - - - - - - + + + + + + + + | + TD | + + + + + - - + + + + + + + + | + SD | + + + + - - - + + + + + + + + | + DDL | + + + - - - - + + + + - + + + | + BLOCK_DDL | + + + + + + + + + + + + + + + | + ALTER_COP | + + + + + - - + + + + + + + + | + COMMIT | + + + + - + - + + + + + + + + | + COMMIT_HIGH_PRIO| + + + + + + + + + + + + + + + | Here: "+" -- means that request can be satisfied "-" -- means that request can't be satisfied and should wait @@ -1685,6 +1688,8 @@ MDL_lock::MDL_backup_lock::m_granted_incompatible[MDL_BACKUP_END]= MDL_BIT(MDL_BACKUP_START) | MDL_BIT(MDL_BACKUP_FLUSH) | MDL_BIT(MDL_BACKUP_WAIT_FLUSH) | MDL_BIT(MDL_BACKUP_WAIT_DDL) | MDL_BIT(MDL_BACKUP_WAIT_COMMIT) | MDL_BIT(MDL_BACKUP_BLOCK_DDL) | MDL_BIT(MDL_BACKUP_DDL), MDL_BIT(MDL_BACKUP_FTWRL1) | MDL_BIT(MDL_BACKUP_FTWRL2), /* MDL_BACKUP_COMMIT */ + MDL_BIT(MDL_BACKUP_WAIT_COMMIT) | MDL_BIT(MDL_BACKUP_FTWRL2), + /* MDL_BACKUP_COMMIT_HIGH_PRIO */ MDL_BIT(MDL_BACKUP_WAIT_COMMIT) | MDL_BIT(MDL_BACKUP_FTWRL2) }; @@ -1712,7 +1717,9 @@ MDL_lock::MDL_backup_lock::m_waiting_incompatible[MDL_BACKUP_END]= MDL_BIT(MDL_BACKUP_START), MDL_BIT(MDL_BACKUP_FTWRL1) | MDL_BIT(MDL_BACKUP_FTWRL2), /* MDL_BACKUP_COMMIT */ - MDL_BIT(MDL_BACKUP_WAIT_COMMIT) | MDL_BIT(MDL_BACKUP_FTWRL2) + MDL_BIT(MDL_BACKUP_WAIT_COMMIT) | MDL_BIT(MDL_BACKUP_FTWRL2), + /* MDL_BACKUP_COMMIT_HIGH_PRIO: is highest priority allowing to bypass any waiter */ + 0 }; diff --git a/sql/mdl.h b/sql/mdl.h index 34b30368cfae9..21dfa006dae08 100644 --- a/sql/mdl.h +++ b/sql/mdl.h @@ -324,7 +324,8 @@ enum enum_mdl_type { Must be acquired during commit. */ #define MDL_BACKUP_COMMIT enum_mdl_type(13) -#define MDL_BACKUP_END enum_mdl_type(14) +#define MDL_BACKUP_COMMIT_HIGH_PRIO enum_mdl_type(14) +#define MDL_BACKUP_END enum_mdl_type(15) /** Duration of metadata lock. */ diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 3b49c53c52b26..028b08f7741a1 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -937,6 +937,7 @@ PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_xid_list, key_mutex_slave_reporting_capability_err_lock, key_relay_log_info_data_lock, key_rpl_group_info_sleep_lock, key_relay_log_info_log_space_lock, key_relay_log_info_run_lock, + key_relay_log_info_backup_team_lock, key_structure_guard_mutex, key_TABLE_SHARE_LOCK_ha_data, key_LOCK_error_messages, key_LOCK_start_thread, @@ -1026,6 +1027,7 @@ static PSI_mutex_info all_server_mutexes[]= { &key_relay_log_info_data_lock, "Relay_log_info::data_lock", 0}, { &key_relay_log_info_log_space_lock, "Relay_log_info::log_space_lock", 0}, { &key_relay_log_info_run_lock, "Relay_log_info::run_lock", 0}, + { &key_relay_log_info_backup_team_lock, "Relay_log_info::backup_team_lock", 0}, { &key_rpl_group_info_sleep_lock, "Rpl_group_info::sleep_lock", 0}, { &key_structure_guard_mutex, "Query_cache::structure_guard_mutex", 0}, { &key_TABLE_SHARE_LOCK_ha_data, "TABLE_SHARE::LOCK_ha_data", 0}, diff --git a/sql/mysqld.h b/sql/mysqld.h index 92cb4f639f743..6c7ae890b5580 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -336,6 +336,7 @@ extern PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_xid_list, key_master_info_start_alter_list_lock, key_mutex_slave_reporting_capability_err_lock, key_relay_log_info_data_lock, key_relay_log_info_log_space_lock, key_relay_log_info_run_lock, + key_relay_log_info_backup_team_lock, key_rpl_group_info_sleep_lock, key_structure_guard_mutex, key_TABLE_SHARE_LOCK_ha_data, key_TABLE_SHARE_LOCK_statistics, diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index 9004aea20e41e..9740f6e773c54 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -700,6 +700,11 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, suspended_wfc= thd->suspend_subsequent_commits(); thd->lex->reset_n_backup_query_tables_list(&lex_backup); tlist.init_one_table(&MYSQL_SCHEMA_NAME, >id_pos_table_name, NULL, TL_WRITE); + if (thd->rgi_slave && thd->rgi_slave->is_parallel_exec) + { + tlist.table_options|= TL_OPTION_GTID_TABLE_SLAVE; + } + if ((err= open_and_lock_tables(thd, &tlist, FALSE, 0))) goto end; table_opened= true; diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index 4ed2f55e8317c..139c406fa6c72 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -2286,6 +2286,8 @@ void rpl_group_info::cleanup_context(THD *thd, bool error, bool keep_domain_owne trans_rollback_stmt(thd); // if a "statement transaction" /* trans_rollback() also resets OPTION_GTID_BEGIN */ trans_rollback(thd); // if a "real transaction" + + DBUG_ASSERT(!thd->backup_commit_lock); // trans_rollback must've taken care /* Now that we have rolled back the transaction, make sure we do not erroneously update the GTID position. diff --git a/sql/rpl_rli.h b/sql/rpl_rli.h index 20c58e4aaf7c4..2a6fc4de169d6 100644 --- a/sql/rpl_rli.h +++ b/sql/rpl_rli.h @@ -24,6 +24,8 @@ #include "sql_class.h" /* THD */ #include "log_event.h" #include "rpl_parallel.h" +#include +#include struct RPL_TABLE_LIST; class Master_info; @@ -59,6 +61,161 @@ class Rpl_filter; struct rpl_group_info; struct inuse_relaylog; + +/** + Each parallel-slave worker that takes MDL_BACKUP_COMMIT (or its + high-priority variant MDL_BACKUP_COMMIT_HIGH_PRIO) at commit time first + registers its `gtid_sub_id` here, then queries `has_higher()` to + decide which mode to request: + + join(domain, my_sub_id); + if (has_higher(domain, my_sub_id)) + lock_type= MDL_BACKUP_COMMIT_HIGH_PRIO; + else + lock_type= MDL_BACKUP_COMMIT; + ...acquire_lock + commit... + leave(domain, my_sub_id); + + `has_higher()` returns true iff some other registered worker in the + same domain has a strictly greater `gtid_sub_id`. The strict-greater + rule preserves the "leapfrog only when needed to break the deadlock + topology" invariant and bounds the per-backup-wait bypass count by + `slave_parallel_threads − 1`. +*/ +class Backup_commit_team +{ +public: + Backup_commit_team() + { + mysql_mutex_init(key_relay_log_info_backup_team_lock, + &m_mtx, MY_MUTEX_INIT_FAST); + } + ~Backup_commit_team() + { + DBUG_ASSERT(is_empty()); + mysql_mutex_destroy(&m_mtx); + } + + /* + Register `sub_id` and atomically decide the BACKUP-COMMIT lock type + for this worker. The decision rule: + + Pick MDL_BACKUP_COMMIT_HIGH_PRIO iff + (a) some larger-sub_id teammate is registered in this domain, AND + (b) no teammate with sub_id < mine is currently pending-C (i.e. + picked C but MDL has not granted yet). + + Otherwise pick MDL_BACKUP_COMMIT and mark this worker pending-C. + + Both conditions are evaluated under one critical section so the + (register, decide, mark-pending-C) triple is atomic — no other + worker can observe a half-decided state. + */ + enum_mdl_type join_and_decide(uint32 domain, uint64 sub_id) + { + enum_mdl_type lock_type; + mysql_mutex_lock(&m_mtx); + + auto &ms= m_per_domain[domain]; + ms.insert(sub_id); + + /* (a) any larger teammate registered? */ + bool has_higher= *ms.rbegin() > sub_id; + + /* (b) any smaller teammate currently pending-C? */ + bool has_smaller_pending_c= false; + auto pit= m_pending_c.find(domain); + if (pit != m_pending_c.end() && !pit->second.empty()) + has_smaller_pending_c= *pit->second.begin() < sub_id; + + if (has_higher && !has_smaller_pending_c) + { + lock_type= MDL_BACKUP_COMMIT_HIGH_PRIO; + /* No pending-C marker — we won't be a stranded C. */ + } + else + { + lock_type= MDL_BACKUP_COMMIT; + m_pending_c[domain].insert(sub_id); + } + mysql_mutex_unlock(&m_mtx); + return lock_type; + } + + /* + Called after MDL acquire_lock() returns granted. Removes the + pending-C marker if we had one. No-op for workers that picked H. + */ + void mark_granted(uint32 domain, uint64 sub_id) + { + mysql_mutex_lock(&m_mtx); + auto pit= m_pending_c.find(domain); + if (pit != m_pending_c.end()) + { + auto v= pit->second.find(sub_id); + if (v != pit->second.end()) + { + pit->second.erase(v); + if (pit->second.empty()) + m_pending_c.erase(pit); + } + } + mysql_mutex_unlock(&m_mtx); + } + + /* + Remove my sub_id from the team. Cleans both the in-flight multiset + and the pending-C set; the pending-C cleanup is a no-op if I had + already been mark_granted()-ed or had picked H. + */ + void leave(uint32 domain, uint64 sub_id) + { + mysql_mutex_lock(&m_mtx); + auto it= m_per_domain.find(domain); + DBUG_ASSERT(it != m_per_domain.end()); + if (it != m_per_domain.end()) + { + auto victim= it->second.find(sub_id); + DBUG_ASSERT(victim != it->second.end()); + if (victim != it->second.end()) + { + it->second.erase(victim); + /* Drop empty domain entries so the map doesn't grow without + bound in multi-source / many-domain setups. */ + if (it->second.empty()) + m_per_domain.erase(it); + } + } + auto pit= m_pending_c.find(domain); + if (pit != m_pending_c.end()) + { + auto v= pit->second.find(sub_id); + if (v != pit->second.end()) + { + pit->second.erase(v); + if (pit->second.empty()) + m_pending_c.erase(pit); + } + } + mysql_mutex_unlock(&m_mtx); + } + + bool is_empty() + { + mysql_mutex_lock(&m_mtx); + bool empty= m_per_domain.empty() && m_pending_c.empty(); + mysql_mutex_unlock(&m_mtx); + return empty; + } + +private: + mysql_mutex_t m_mtx; + /* All registered in-flight teammates per GTID domain. */ + std::map > m_per_domain; + /* Subset that picked C and whose MDL hasn't been granted yet. */ + std::map > m_pending_c; +}; + class Relay_log_info : public Slave_reporting_capability { public: @@ -411,6 +568,9 @@ class Relay_log_info : public Slave_reporting_capability */ slave_connection_state restart_gtid_pos; + /* Per-relay in-flight BACKUP-COMMIT holders; see Backup_commit_team. */ + Backup_commit_team backup_team; + Relay_log_info(bool is_slave_recovery, const char* thread_name= "SQL"); ~Relay_log_info(); @@ -1052,7 +1212,6 @@ struct rpl_group_info { finish_event_group_called= value; } - }; diff --git a/sql/sql_base.cc b/sql/sql_base.cc index bfa787107c516..8987346a3b618 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -2383,9 +2383,21 @@ bool open_table(THD *thd, TABLE_LIST *table_list, Open_table_context *ot_ctx) DBUG_RETURN(TRUE); } + if (table_list->table_options & TL_OPTION_GTID_TABLE_SLAVE) + { + /* + This is a request from the committing phase of a slave + transaction, done on behalf of the implicit gtid_slave_pos + insert statement. It must be able to bypass any waiting lock + by the backup process even without yet knowing whether the + transaction is going to be a slave group-commit leader. When + it turns out not to be, the strong MDL_STATEMENT-requested + lock would already have been relinquished. + */ + mdl_type= MDL_BACKUP_COMMIT_HIGH_PRIO; + } MDL_REQUEST_INIT(&protection_request, MDL_key::BACKUP, "", "", mdl_type, MDL_STATEMENT); - /* Install error handler which if possible will convert deadlock error into request to back-off and restart process of opening tables. diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 7b54c1ddc2d7d..2d26573a65cd2 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -8299,20 +8299,6 @@ wait_for_commit::wait_for_prior_commit2(THD *thd, bool allow_kill) { PSI_stage_info old_stage; wait_for_commit *loc_waitee; - bool backup_lock_released= false; - - /* - Release MDL_BACKUP_COMMIT LOCK while waiting for other threads to commit - This is needed to avoid deadlock between the other threads (which not - yet have the MDL_BACKUP_COMMIT_LOCK) and any threads using - BACKUP LOCK BLOCK_COMMIT. - */ - if (thd->backup_commit_lock && thd->backup_commit_lock->ticket) - { - backup_lock_released= true; - thd->mdl_context.release_lock(thd->backup_commit_lock->ticket); - thd->backup_commit_lock->ticket= 0; - } mysql_mutex_lock(&LOCK_wait_commit); DEBUG_SYNC(thd, "wait_for_prior_commit_waiting"); @@ -8366,16 +8352,10 @@ wait_for_commit::wait_for_prior_commit2(THD *thd, bool allow_kill) use within enter_cond/exit_cond. */ DEBUG_SYNC(thd, "wait_for_prior_commit_killed"); - if (unlikely(backup_lock_released)) - thd->mdl_context.acquire_lock(thd->backup_commit_lock, - thd->variables.lock_wait_timeout); return wakeup_error; end: thd->EXIT_COND(&old_stage); - if (unlikely(backup_lock_released)) - thd->mdl_context.acquire_lock(thd->backup_commit_lock, - thd->variables.lock_wait_timeout); return wakeup_error; } diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 8d533b06e813f..2485015de57ac 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -465,6 +465,7 @@ enum enum_drop_mode #define TL_OPTION_ALIAS 8 #define TL_OPTION_SEQUENCE 16 #define TL_OPTION_TABLE_FUNCTION 32 +#define TL_OPTION_GTID_TABLE_SLAVE 64 typedef List List_item; typedef Mem_root_array Group_list_ptrs; diff --git a/sql/xa.cc b/sql/xa.cc index 48cbf65c53a29..f41fe14ca0066 100644 --- a/sql/xa.cc +++ b/sql/xa.cc @@ -22,6 +22,7 @@ #include "my_cpu.h" #include #include +#include "rpl_rli.h" // rgi->is_parallel_exec static bool slave_applier_reset_xa_trans(THD *thd); @@ -522,11 +523,31 @@ bool trans_xa_end(THD *thd) static bool trans_xa_get_backup_lock(THD *thd, MDL_request *mdl_request) { DBUG_ASSERT(thd->backup_commit_lock == 0); - MDL_REQUEST_INIT(mdl_request, MDL_key::BACKUP, "", "", MDL_BACKUP_COMMIT, + rpl_group_info *rgi= thd->rgi_slave; + bool is_parallel_slave= rgi && rgi->is_parallel_exec && rgi->gtid_sub_id > 0; + + /* Parallel-slave: atomic register-and-decide. See ha_commit_trans + and Backup_commit_team::join_and_decide() for the rule. */ + enum_mdl_type lock_type= MDL_BACKUP_COMMIT; + if (is_parallel_slave) + lock_type= rgi->rli->backup_team.join_and_decide( + rgi->current_gtid.domain_id, rgi->gtid_sub_id); + + MDL_REQUEST_INIT(mdl_request, MDL_key::BACKUP, "", "", lock_type, MDL_EXPLICIT); + if (thd->mdl_context.acquire_lock(mdl_request, thd->variables.lock_wait_timeout)) + { + if (is_parallel_slave) + rgi->rli->backup_team.leave(rgi->current_gtid.domain_id, + rgi->gtid_sub_id); return 1; + } + if (is_parallel_slave) + rgi->rli->backup_team.mark_granted(rgi->current_gtid.domain_id, + rgi->gtid_sub_id); + thd->backup_commit_lock= mdl_request; return 0; } @@ -537,6 +558,11 @@ static inline void trans_xa_release_backup_lock(THD *thd) { thd->mdl_context.release_lock(thd->backup_commit_lock->ticket); thd->backup_commit_lock= 0; + /* Paired with join() in trans_xa_get_backup_lock. */ + rpl_group_info *rgi= thd->rgi_slave; + if (rgi && rgi->is_parallel_exec && rgi->gtid_sub_id > 0) + rgi->rli->backup_team.leave(rgi->current_gtid.domain_id, + rgi->gtid_sub_id); } } @@ -1166,6 +1192,7 @@ static bool slave_applier_reset_xa_trans(THD *thd) thd->transaction->all.ha_list= 0; ha_close_connection(thd); + trans_xa_release_backup_lock(thd); thd->transaction->cleanup(); thd->transaction->all.reset();