diff --git a/bin/pubsub b/bin/pubsub index bd19f38..d043a71 100755 --- a/bin/pubsub +++ b/bin/pubsub @@ -49,7 +49,7 @@ module PubSubDebug handle_errors('Subscriber') do e = ps.next_event(0.01) log "#{role}: recv: #{e.nil? ? 'nil' : e}" - ps.call('ssubscribe', c) if e.first == 'sunsubscribe' + ps.call('ssubscribe', c) if !e.nil? && e.first == 'sunsubscribe' end ensure sleep 1.0 diff --git a/test/cluster_controller.rb b/test/cluster_controller.rb index 0ddb752..69e3697 100644 --- a/test/cluster_controller.rb +++ b/test/cluster_controller.rb @@ -398,9 +398,11 @@ def wait_failover(clients, primary_node_key:, replica_node_key:, max_attempts:) def wait_replication_delay(clients, replica_size:, timeout:) timeout_msec = timeout.to_i * 1000 + server_side_timeout = timeout_msec > 100 ? timeout_msec - 100 : 10 + wait_for_state(clients, max_attempts: clients.size + 1) do |client| swap_timeout(client, timeout: 0.1) do |cli| - cli.blocking_call(timeout, 'WAIT', replica_size, timeout_msec - 100) if primary_client?(cli) + cli.blocking_call(timeout, 'WAIT', replica_size, server_side_timeout) if primary_client?(cli) end true rescue ::RedisClient::ConnectionError diff --git a/test/test_against_cluster_broken.rb b/test/test_against_cluster_broken.rb index 164a7cd..73f32ed 100644 --- a/test/test_against_cluster_broken.rb +++ b/test/test_against_cluster_broken.rb @@ -21,9 +21,7 @@ def setup print "\n" @logger.info('setup: test') prepare_test_data - @clients = Array.new(3) do - build_client.tap { |c| c.call('echo', 'init') } - end + @clients = Array.new(3) { build_client.tap { |c| c.call('echo', 'init') } } end def teardown @@ -32,28 +30,29 @@ def teardown @clients.each(&:close) @controller&.close refute(@captured_commands.count('cluster', 'nodes').zero?, @captured_commands.to_a.map(&:command)) - print "#{@redirect_count.get}, "\ - "ClusterNodesCall: #{@captured_commands.count('cluster', 'nodes')}, "\ - "ClusterDownError: #{@cluster_down_error_count} = " end def test_client_patience + failover_manually + wait_for_cluster_to_be_ready + do_assertions(offset: 0) + # a replica sacrifice_replica = @controller.select_sacrifice_of_replica kill_a_node(sacrifice_replica) wait_for_cluster_to_be_ready(ignore: [sacrifice_replica]) - do_assertions(offset: 0) + do_assertions(offset: 1) # a primary sacrifice_primary = @controller.select_sacrifice_of_primary kill_a_node(sacrifice_primary) wait_for_cluster_to_be_ready(ignore: [sacrifice_replica, sacrifice_primary]) - do_assertions(offset: 1) + do_assertions(offset: 2) # recovery revive_dead_nodes wait_for_cluster_to_be_ready - do_assertions(offset: 2) + do_assertions(offset: 3) end private @@ -122,6 +121,8 @@ def do_assertions(offset:) assert_equal(want, got, 'Case: Transaction: SET') end end + + log_metrics end end @@ -139,6 +140,12 @@ def wait_for_cluster_to_be_ready(ignore: []) end end + def failover_manually + log_info('failover') do + @controller.failover + end + end + def kill_a_node(sacrifice) log_info("kill #{sacrifice.config.host}:#{sacrifice.config.port}") do refute_nil(sacrifice, "#{sacrifice.config.host}:#{sacrifice.config.port}") @@ -171,6 +178,12 @@ def log_info(message) @logger.info(" done: #{message}") end + def log_metrics + print "#{@redirect_count.get}, "\ + "ClusterNodesCall: #{@captured_commands.count('cluster', 'nodes')}, "\ + "ClusterDownError: #{@cluster_down_error_count}\n" + end + def retryable(attempts: MAX_ATTEMPTS, wait_sec: WAIT_SEC) loop do raise MaxRetryExceeded if attempts <= 0 diff --git a/test/test_against_cluster_state.rb b/test/test_against_cluster_state.rb index 5ebdeb2..f7e2c4f 100644 --- a/test/test_against_cluster_state.rb +++ b/test/test_against_cluster_state.rb @@ -29,14 +29,6 @@ def teardown "ClusterNodesCall: #{@captured_commands.count('cluster', 'nodes')} = " end - def test_the_state_of_cluster_failover - @controller.failover - 1000.times { |i| assert_equal('OK', @client.call('SET', "key#{i}", i)) } - wait_for_replication - 1000.times { |i| assert_equal(i.to_s, @client.call('GET', "key#{i}")) } - refute(@redirect_count.zero?, @redirect_count.get) - end - def test_the_state_of_cluster_resharding resharded_keys = nil do_resharding_test do |keys|