# large trx crash cases: # 1. leader, large trx write half, crash # 2. leader, large trx write all, waitting to commit, crash # 3. follower, large trx apply half, crash # # leader large trx commit, crash # it is ok, do not test # follower receive half, crash # no apply start, will back to normal after restart --source include/have_debug_new.inc --connect (conn1,127.0.0.1,root,,test,$MYPORT_1) --connection conn1 flush local logs; set global debug="+d,force_large_trx,force_large_trx_single_ev"; --connect (conn2,127.0.0.1,root,,test,$MYPORT_2) --connection conn2 flush local logs; set global debug="+d,force_large_trx,force_large_trx_single_ev"; --connect (conn3,127.0.0.1,root,,test,$MYPORT_3) --connection conn3 flush local logs; set global debug="+d,force_large_trx,force_large_trx_single_ev"; --echo # --------- normal case --------- --connection conn1 use test; create table t1 (id int); insert into t1 values (1), (2), (3); begin; insert into t1 values (4); insert into t1 values (5); commit; begin; insert into t1 values (6); insert into t1 values (7); rollback; --source include/wait_follower_catchup.inc let $cnt1= query_get_value("select count(*) as cnt from t1", cnt, 1); --connection conn2 --source include/wait_apply_catchup.inc let $cnt2= query_get_value("select count(*) as cnt from t1", cnt, 1); --connection conn3 --source include/wait_apply_catchup.inc let $cnt3= query_get_value("select count(*) as cnt from t1", cnt, 1); if ($cnt1 == $cnt2) { if ($cnt2 == $cnt3) { echo consistent!!!!!!!; } } #select * from t1; --echo # --------- case 1 --------- --connection conn1 insert into t1 values (8); set global debug="+d,crash_during_large_trx_binlog_flush"; --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --error 2013 insert into t1 values (9); --source include/wait_until_disconnected.inc --sleep 2 --let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --source include/start_mysqld.inc set global debug="+d,force_large_trx,force_large_trx_single_ev"; --source include/wait_follower_catchup.inc let $cnt1= query_get_value("select count(*) as cnt from t1", cnt, 1); --connection conn2 --enable_reconnect --source include/wait_until_connected_again_old.inc --source include/wait_apply_catchup.inc let $cnt2= query_get_value("select count(*) as cnt from t1", cnt, 1); --connection conn3 --enable_reconnect --source include/wait_until_connected_again_old.inc --source include/wait_apply_catchup.inc let $cnt3= query_get_value("select count(*) as cnt from t1", cnt, 1); if ($cnt1 == $cnt2) { if ($cnt2 == $cnt3) { echo consistent!!!!!!!; } } #select * from t1; --sleep 1 --echo # --------- case 2 --------- --connection conn1 insert into t1 values (10); set global debug="+d,crash_before_large_trx_commit_early"; --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --error 2013 insert into t1 values (11); --source include/wait_until_disconnected.inc --sleep 2 --let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --source include/start_mysqld.inc set global debug="+d,force_large_trx,force_large_trx_single_ev"; --sleep 1 --source include/wait_follower_catchup.inc let $cnt1= query_get_value("select count(*) as cnt from t1", cnt, 1); --connection conn2 --enable_reconnect --source include/wait_until_connected_again_old.inc --source include/wait_apply_catchup.inc let $cnt2= query_get_value("select count(*) as cnt from t1", cnt, 1); --connection conn3 --enable_reconnect --source include/wait_until_connected_again_old.inc --source include/wait_apply_catchup.inc let $cnt3= query_get_value("select count(*) as cnt from t1", cnt, 1); if ($cnt1 == $cnt2) { if ($cnt2 == $cnt3) { echo consistent!!!!!!!; } } #select * from t1; --connection conn1 insert into t1 values (12); set global debug="+d,crash_before_large_trx_commit_late"; --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --error 2013 insert into t1 values (13); --source include/wait_until_disconnected.inc --sleep 2 --let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --source include/start_mysqld.inc set global debug="+d,force_large_trx,force_large_trx_single_ev"; --source include/wait_follower_catchup.inc let $cnt1= query_get_value("select count(*) as cnt from t1", cnt, 1); --connection conn2 --enable_reconnect --source include/wait_until_connected_again_old.inc --source include/wait_apply_catchup.inc let $cnt2= query_get_value("select count(*) as cnt from t1", cnt, 1); --connection conn3 --enable_reconnect --source include/wait_until_connected_again_old.inc --source include/wait_apply_catchup.inc let $cnt3= query_get_value("select count(*) as cnt from t1", cnt, 1); if ($cnt1 == $cnt2) { if ($cnt2 == $cnt3) { echo consistent!!!!!!!; } } #select * from t1; --echo # --------- case 3 --------- --connection conn1 insert into t1 values (14); --connection conn2 set global debug="+d,crash_before_update_pos"; --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.2.expect --connection conn1 insert into t1 values (15); --connection conn2 --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.2.expect --enable_reconnect --source include/wait_until_connected_again_old.inc set global debug="+d,force_large_trx,force_large_trx_single_ev"; --connection conn1 insert into t1 values (16); --source include/wait_follower_catchup.inc let $cnt1= query_get_value("select count(*) as cnt from t1", cnt, 1); --connection conn2 --enable_reconnect --source include/wait_until_connected_again_old.inc --source include/wait_apply_catchup.inc let $cnt2= query_get_value("select count(*) as cnt from t1", cnt, 1); --connection conn3 --enable_reconnect --source include/wait_until_connected_again_old.inc --source include/wait_apply_catchup.inc let $cnt3= query_get_value("select count(*) as cnt from t1", cnt, 1); if ($cnt1 == $cnt2) { if ($cnt2 == $cnt3) { echo consistent!!!!!!!; } } #select * from t1; --echo # --------- case 4 --------- --connection conn1 insert into t1 values (17); --connection conn2 set global debug="+d,crash_in_a_worker"; --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.2.expect --connection conn1 insert into t1 values (18); --connection conn2 --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.2.expect --enable_reconnect --source include/wait_until_connected_again_old.inc set global debug="+d,force_large_trx,force_large_trx_single_ev"; --connection conn1 insert into t1 values (19); --source include/wait_follower_catchup.inc let $cnt1= query_get_value("select count(*) as cnt from t1", cnt, 1); --connection conn2 --enable_reconnect --source include/wait_until_connected_again_old.inc --source include/wait_apply_catchup.inc let $cnt2= query_get_value("select count(*) as cnt from t1", cnt, 1); --connection conn3 --enable_reconnect --source include/wait_until_connected_again_old.inc --source include/wait_apply_catchup.inc let $cnt3= query_get_value("select count(*) as cnt from t1", cnt, 1); if ($cnt1 == $cnt2) { if ($cnt2 == $cnt3) { echo consistent!!!!!!!; } } #select * from t1; --echo # --------- case 5 --------- --connection conn1 insert into t1 values (20); --connection conn2 set global debug="+d,crash_after_commit_and_update_pos"; --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.2.expect --connection conn1 insert into t1 values (21); --connection conn2 --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.2.expect --enable_reconnect --source include/wait_until_connected_again_old.inc set global debug="+d,force_large_trx,force_large_trx_single_ev"; --connection conn1 insert into t1 values (22); --source include/wait_follower_catchup.inc let $cnt1= query_get_value("select count(*) as cnt from t1", cnt, 1); --connection conn2 --enable_reconnect --source include/wait_until_connected_again_old.inc --source include/wait_apply_catchup.inc let $cnt2= query_get_value("select count(*) as cnt from t1", cnt, 1); --connection conn3 --enable_reconnect --source include/wait_until_connected_again_old.inc --source include/wait_apply_catchup.inc let $cnt3= query_get_value("select count(*) as cnt from t1", cnt, 1); if ($cnt1 == $cnt2) { if ($cnt2 == $cnt3) { echo consistent!!!!!!!; } } #select * from t1; --echo # --------- case 6 --------- --connection conn1 insert into t1 values (23); --connection conn2 set global debug="+d,crash_after_update_pos_before_apply"; --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.2.expect --connection conn1 insert into t1 values (24); --connection conn2 --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.2.expect --enable_reconnect --source include/wait_until_connected_again_old.inc set global debug="+d,force_large_trx,force_large_trx_single_ev"; --connection conn1 insert into t1 values (25); --source include/wait_follower_catchup.inc let $cnt1= query_get_value("select count(*) as cnt from t1", cnt, 1); --connection conn2 --enable_reconnect --source include/wait_until_connected_again_old.inc --source include/wait_apply_catchup.inc let $cnt2= query_get_value("select count(*) as cnt from t1", cnt, 1); --connection conn3 --enable_reconnect --source include/wait_until_connected_again_old.inc --source include/wait_apply_catchup.inc let $cnt3= query_get_value("select count(*) as cnt from t1", cnt, 1); if ($cnt1 == $cnt2) { if ($cnt2 == $cnt3) { echo consistent!!!!!!!; } } #select * from t1; --echo # --------- case 7 --------- --connection conn1 insert into t1 values (26); --connection conn2 set global debug="+d,crash_after_apply"; --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.2.expect --connection conn1 insert into t1 values (27); --connection conn2 --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.2.expect --enable_reconnect --source include/wait_until_connected_again_old.inc set global debug="+d,force_large_trx,force_large_trx_single_ev"; --connection conn1 insert into t1 values (28); --source include/wait_follower_catchup.inc let $cnt1= query_get_value("select count(*) as cnt from t1", cnt, 1); --connection conn2 --enable_reconnect --source include/wait_until_connected_again_old.inc --source include/wait_apply_catchup.inc let $cnt2= query_get_value("select count(*) as cnt from t1", cnt, 1); --connection conn3 --enable_reconnect --source include/wait_until_connected_again_old.inc --source include/wait_apply_catchup.inc let $cnt3= query_get_value("select count(*) as cnt from t1", cnt, 1); if ($cnt1 == $cnt2) { if ($cnt2 == $cnt3) { echo consistent!!!!!!!; } } #select * from t1; --echo # --------- case 1.1 --------- --connection conn1 insert into t1 values (29); set global debug="+d,crash_during_large_trx_binlog_flush_slow"; --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --error 2013 insert into t1 values (30); --source include/wait_until_disconnected.inc --sleep 2 --let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --source include/start_mysqld.inc set global debug="+d,force_large_trx,force_large_trx_single_ev"; --source include/wait_follower_catchup.inc let $cnt1= query_get_value("select count(*) as cnt from t1", cnt, 1); --connection conn2 --enable_reconnect --source include/wait_until_connected_again_old.inc --source include/wait_apply_catchup.inc let $cnt2= query_get_value("select count(*) as cnt from t1", cnt, 1); --connection conn3 --enable_reconnect --source include/wait_until_connected_again_old.inc --source include/wait_apply_catchup.inc let $cnt3= query_get_value("select count(*) as cnt from t1", cnt, 1); if ($cnt1 == $cnt2) { if ($cnt2 == $cnt3) { echo consistent!!!!!!!; } } #select * from t1; --echo # --------- case 1.2 --------- --connection conn1 insert into t1 values (31); set global debug="+d,crash_during_large_trx_binlog_flush2"; --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --error 2013 insert into t1 values (32); --source include/wait_until_disconnected.inc --sleep 2 --let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --source include/start_mysqld.inc set global debug="+d,force_large_trx,force_large_trx_single_ev"; --source include/wait_follower_catchup.inc let $cnt1= query_get_value("select count(*) as cnt from t1", cnt, 1); --connection conn2 --enable_reconnect --source include/wait_until_connected_again_old.inc --source include/wait_apply_catchup.inc let $cnt2= query_get_value("select count(*) as cnt from t1", cnt, 1); --connection conn3 --enable_reconnect --source include/wait_until_connected_again_old.inc --source include/wait_apply_catchup.inc let $cnt3= query_get_value("select count(*) as cnt from t1", cnt, 1); if ($cnt1 == $cnt2) { if ($cnt2 == $cnt3) { echo consistent!!!!!!!; } } #select * from t1; --connection conn1 --echo # --------- case large event --------- insert into t1 values (100); set global debug="+d,force_large_trx,force_large_trx_single_ev,force_large_event"; --error 1534 insert into t1 values (101); set global debug="-d,force_large_event"; --echo # --------- case flush --------- SET debug_sync = "large_trx_sync_part WAIT_FOR continue"; --send insert into t1 values (102) --connect (conn1_1,127.0.0.1,root,,test,$MYPORT_1) --connection conn1_1 # will wait prepared trx --send flush logs --connection conn2 # wait replicate --sleep 2 # will fail because last log is in middle of large trx --error ER_CONSENSUS_FOLLOWER_NOT_ALLOWED flush local logs; --connect (conn1_2,127.0.0.1,root,,test,$MYPORT_1) --connection conn1_2 SET debug_sync = "now SIGNAL continue"; --connection conn1 --reap --connection conn1_1 --reap --connection conn1 drop table t1; set global debug="";