/* Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, as published by the Free Software Foundation. This program is also distributed with certain software (including but not limited to OpenSSL) that is licensed under separate terms, as designated in a particular file or component or in included license documentation. The authors of MySQL hereby grant you an additional permission to link the program and your derivative works with the separately licensed software that they have included with MySQL. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include "router_config.h" // defines HAVE_PRLIMIT #ifdef HAVE_PRLIMIT #include // prlimit() #endif #ifndef _WIN32 #include #include #include #include #include #include #include #else #include #include #include #endif #include "mysql_session.h" #include "router_component_test.h" #include "router_test_helpers.h" #include "socket_operations.h" #include "tcp_port_pool.h" using namespace std::chrono_literals; using mysql_harness::SocketOperations; using mysqlrouter::MySQLSession; class RouterRoutingTest : public RouterComponentTest { protected: TcpPortPool port_pool_; }; TEST_F(RouterRoutingTest, RoutingOk) { const auto server_port = port_pool_.get_next_available(); const auto router_port = port_pool_.get_next_available(); // use the json file that adds additional rows to the metadata to increase the // packet size to +10MB to verify routing of the big packets const std::string json_stmts = get_data_dir().join("bootstrap_big_data.js").str(); TempDirectory bootstrap_dir; // launch the server mock for bootstrapping auto &server_mock = launch_mysql_server_mock( json_stmts, server_port, false /*expecting huge data, can't print on the console*/); const std::string routing_section = "[routing:basic]\n" "bind_port = " + std::to_string(router_port) + "\n" "mode = read-write\n" "destinations = 127.0.0.1:" + std::to_string(server_port) + "\n"; TempDirectory conf_dir("conf"); std::string conf_file = create_config_file(conf_dir.name(), routing_section); // launch the router with simple static routing configuration auto &router_static = launch_router({"-c", conf_file}); // wait for both to begin accepting the connections ASSERT_NO_FATAL_FAILURE(check_port_ready(server_mock, server_port)); ASSERT_NO_FATAL_FAILURE(check_port_ready(router_static, router_port)); // launch another router to do the bootstrap connecting to the mock server // via first router instance auto &router_bootstrapping = launch_router({ "--bootstrap=localhost:" + std::to_string(router_port), "--report-host", "dont.query.dns", "-d", bootstrap_dir.name(), }); router_bootstrapping.register_response( "Please enter MySQL password for root: ", "fake-pass\n"); ASSERT_NO_FATAL_FAILURE(check_exit_code(router_bootstrapping, EXIT_SUCCESS)); ASSERT_TRUE(router_bootstrapping.expect_output( "MySQL Router configured for the InnoDB cluster 'test'")) << "bootstrap output: " << router_bootstrapping.get_full_output() << std::endl << "routing log: " << router_bootstrapping.get_full_logfile() << std::endl << "server output: " << server_mock.get_full_output() << std::endl; } TEST_F(RouterRoutingTest, RoutingTooManyConnections) { const auto server_port = port_pool_.get_next_available(); const auto router_port = port_pool_.get_next_available(); // doesn't really matter which file we use here, we are not going to do any // queries const std::string json_stmts = get_data_dir().join("bootstrap_big_data.js").str(); // launch the server mock auto &server_mock = launch_mysql_server_mock(json_stmts, server_port, false); // create a config with routing that has max_connections == 2 const std::string routing_section = "[routing:basic]\n" "bind_port = " + std::to_string(router_port) + "\n" "mode = read-write\n" "max_connections = 2\n" "destinations = 127.0.0.1:" + std::to_string(server_port) + "\n"; TempDirectory conf_dir("conf"); std::string conf_file = create_config_file(conf_dir.name(), routing_section); // launch the router with the created configuration auto &router = launch_router({"-c", conf_file}); // wait for server and router to begin accepting the connections ASSERT_NO_FATAL_FAILURE(check_port_ready(server_mock, server_port)); ASSERT_NO_FATAL_FAILURE(check_port_ready(router, router_port)); // try to create 3 connections, the third should fail // because of the max_connections limit being exceeded mysqlrouter::MySQLSession client1, client2, client3; EXPECT_NO_THROW(client1.connect("127.0.0.1", router_port, "username", "password", "", "")); EXPECT_NO_THROW(client2.connect("127.0.0.1", router_port, "username", "password", "", "")); ASSERT_THROW_LIKE( client3.connect("127.0.0.1", router_port, "username", "password", "", ""), std::runtime_error, "Too many connections to MySQL Router (1040)"); } template ::testing::AssertionResult ThrowsExceptionWith(std::function callable, const char *expected_text) { try { callable(); return ::testing::AssertionFailure() << "Expected exception to throw, but it didn't"; } catch (const T &e) { if (nullptr == ::strstr(e.what(), expected_text)) { return ::testing::AssertionFailure() << "Expected exception-text to contain: " << expected_text << ". Actual: " << e.what(); } return ::testing::AssertionSuccess(); } catch (...) { // as T may be std::exception we can't use it as default case and need to do // this extra round try { throw; } catch (const std::exception &e) { return ::testing::AssertionFailure() << "Expected exception of type " << typeid(T).name() << ". Actual: " << typeid(e).name(); } catch (...) { return ::testing::AssertionFailure() << "Expected exception of type " << typeid(T).name() << ". Actual: non-std exception"; } } } // this test uses OS-specific methods to restrict thread creation #ifdef HAVE_PRLIMIT TEST_F(RouterRoutingTest, RoutingPluginCantSpawnMoreThreads) { const auto server_port = port_pool_.get_next_available(); const auto router_port = port_pool_.get_next_available(); // doesn't really matter which file we use here, we are not going to do any // queries const std::string json_stmts = get_data_dir().join("bootstrap_big_data.js").str(); // launch the server mock auto &server_mock = launch_mysql_server_mock(json_stmts, server_port, false); // create a basic config // // DEBUG is needed to synchronize with 'Running.' from the Loader::main_loop() // to get a stable test. const std::string routing_section = "[logger]\n" "level = DEBUG\n" "[routing:basic]\n" "bind_port = " + std::to_string(router_port) + "\n" "mode = read-write\n" "destinations = 127.0.0.1:" + std::to_string(server_port) + "\n"; TempDirectory conf_dir("conf"); std::string conf_file = create_config_file(conf_dir.name(), routing_section); SCOPED_TRACE("// launch the router with the created configuration"); auto &router_static = launch_router({"-c", conf_file}); SCOPED_TRACE("// capture current NPROC"); pid_t pid = router_static.get_pid(); struct rlimit old_limit; EXPECT_EQ(0, prlimit(pid, RLIMIT_NPROC, nullptr, &old_limit)); SCOPED_TRACE( "// wait for server and router to begin accepting the connections"); ASSERT_NO_FATAL_FAILURE(check_port_ready(server_mock, server_port)); ASSERT_NO_FATAL_FAILURE(check_port_ready(router_static, router_port)); // without waiting we may otherwise hit a race between the // signal-handler-thread and the plugin-start thread and get different // test-scenario which leads to "exit-code 1" SCOPED_TRACE("// wait for Loader::main_loop() to start"); EXPECT_TRUE(find_in_file( get_logging_dir().join("mysqlrouter.log").str(), [](const auto &line) { return pattern_found(line, " Running."); }, 1s)) << router_static.get_full_logfile(); SCOPED_TRACE("// reducing NPROC to 0"); { // how many threads Router process is allowed to have. If this number is // lower than current count, nothing will happen, but new ones will not be // allowed to be created until count comes down below this limit. Thus 0 is // a nice number to ensure nothing gets spawned anymore. rlim_t max_threads = 0; struct rlimit new_limit { .rlim_cur = max_threads, .rlim_max = old_limit.rlim_max }; EXPECT_EQ(0, prlimit(pid, RLIMIT_NPROC, &new_limit, nullptr)); } // try to create a new connection which should fail as: // // - std::thread() in routing plugin will fail to spawn a new thread for this // new connection // ... which returns the client "Router couldn't spawn new threads" // SCOPED_TRACE( "// opening connection which creates a new thread which should fail."); mysqlrouter::MySQLSession client1; EXPECT_TRUE(ThrowsExceptionWith( [&client1, router_port]() { client1.connect("127.0.0.1", router_port, "username", "password", "", ""); }, "Router couldn't spawn a new thread to service new client connection " "(1040)")) << "mock: " << server_mock.get_full_logfile() << "\n" << "router: " << router_static.get_full_logfile(); SCOPED_TRACE("// restoring old NPROC."); // we need to restore the old limit, otherwise ASAN can't spawn the thread // that it needs on shutdown and crashes EXPECT_EQ(0, prlimit(pid, RLIMIT_NPROC, &old_limit, nullptr)); SCOPED_TRACE("// stopping router and wait for shutdown."); EXPECT_EQ(router_static.send_clean_shutdown_event(), std::error_code{}); EXPECT_NO_THROW(EXPECT_EQ(router_static.wait_for_exit(), 0) << router_static.get_full_logfile()) << router_static.get_full_logfile(); SCOPED_TRACE("// check for expected content."); EXPECT_THAT(router_static.get_full_logfile(), ::testing::ContainsRegex("Couldn't spawn a new thread to " "service new client connection")); } #endif // #ifndef HAVE_PRLIMIT #ifndef _WIN32 // named sockets are not supported on Windows; // on Unix, they're implemented using Unix sockets TEST_F(RouterRoutingTest, named_socket_has_right_permissions) { /** * @test Verify that unix socket has the required file permissions so that it * can be connected to by all users. According to man 7 unix, only r+w * permissions are required, but Server sets x as well, so we do the * same. */ // get config dir (we will also stuff our unix socket file there) TempDirectory bootstrap_dir; // launch Router with unix socket const std::string socket_file = bootstrap_dir.name() + "/sockfile"; const std::string routing_section = "[routing:basic]\n" "socket = " + socket_file + "\n" "mode = read-write\n" "destinations = 127.0.0.1:1234\n"; // port can be bogus TempDirectory conf_dir("conf"); const std::string conf_file = create_config_file(conf_dir.name(), routing_section); launch_router({"-c", conf_file}); // loop until socket file appears and has correct permissions auto wait_for_correct_perms = [&socket_file](int timeout_ms) { const mode_t expected_mode = S_IFSOCK | S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IWGRP | S_IXGRP | S_IROTH | S_IWOTH | S_IXOTH; while (timeout_ms > 0) { struct stat info; memset(&info, 0, sizeof(info)); stat(socket_file.c_str(), &info); // silently ignore error when file doesn't exist yet if (info.st_mode == expected_mode) return true; std::this_thread::sleep_for(std::chrono::milliseconds(10)); timeout_ms -= 10; } return false; }; EXPECT_THAT(wait_for_correct_perms(5000), testing::Eq(true)); } #endif TEST_F(RouterRoutingTest, RoutingMaxConnectErrors) { const auto server_port = port_pool_.get_next_available(); const auto router_port = port_pool_.get_next_available(); // json file does not actually matter in this test as we are not going to const std::string json_stmts = get_data_dir().join("bootstrap_big_data.js").str(); TempDirectory bootstrap_dir; // launch the server mock for bootstrapping auto &server_mock = launch_mysql_server_mock( json_stmts, server_port, false /*expecting huge data, can't print on the console*/); const std::string routing_section = "[routing:basic]\n" "bind_port = " + std::to_string(router_port) + "\n" "mode = read-write\n" "destinations = 127.0.0.1:" + std::to_string(server_port) + "\n" "max_connect_errors = 1\n"; TempDirectory conf_dir("conf"); std::string conf_file = create_config_file(conf_dir.name(), routing_section); // launch the router auto &router = launch_router({"-c", conf_file}); // wait for mock server to begin accepting the connections ASSERT_NO_FATAL_FAILURE(check_port_ready(server_mock, server_port)); // wait for router to begin accepting the connections // NOTE: this should cause connection/disconnection which // should be treated as connection error and increment // connection errors counter. This test relies on that. ASSERT_NO_FATAL_FAILURE(check_port_ready(router, router_port)); // wait until blocking client host info appears in the log bool res = find_in_file(get_logging_dir().str() + "/mysqlrouter.log", [](const std::string &line) -> bool { return line.find("blocking client host") != line.npos; }); ASSERT_TRUE(res) << "Did not found expected entry in log file"; // for the next connection attempt we should get an error as the // max_connect_errors was exceeded MySQLSession client; EXPECT_THROW_LIKE( client.connect("127.0.0.1", router_port, "username", "password", "", ""), std::exception, "Too many connection errors"); } // in following functions we use SocketOperations for convenience: it provides // Win and Unix implemenatations where needed, thus saving us some #ifdefs static bool connect_to_host(int &sock, uint16_t port) { SocketOperations *so = SocketOperations::instance(); struct addrinfo hints, *ainfo; memset(&hints, 0, sizeof hints); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_PASSIVE; int status = getaddrinfo("127.0.0.1", std::to_string(port).c_str(), &hints, &ainfo); if (status != 0) throw std::runtime_error(std::string("getaddrinfo() failed: ") + gai_strerror(status)); std::shared_ptr exit_freeaddrinfo(nullptr, [&](void *) { freeaddrinfo(ainfo); }); sock = socket(ainfo->ai_family, ainfo->ai_socktype, ainfo->ai_protocol); if (sock < 0) throw std::runtime_error("socket() failed: " + std::to_string(so->get_errno())); // return connection success return (connect(sock, ainfo->ai_addr, ainfo->ai_addrlen) == 0); } static void read_until_error(int sock) { SocketOperations *so = SocketOperations::instance(); char buf[1024]; while (true) { int bytes_read = so->read(sock, buf, sizeof(buf)); if (bytes_read <= 0) return; } } static void make_bad_connection(uint16_t port) { SocketOperations *so = SocketOperations::instance(); // TCP-level connection phase int sock; if (!connect_to_host(sock, port)) return; // MySQL protocol handshake phase // To simplify code, instead of alternating between reading and writing // protocol packets, we write a lot of garbage upfront, and then read whatever // Router sends back. Router will read what we wrote in chunks, inbetween its // writes, thinking they're replies to its handshake packets. Eventually it // will finish the handshake with error and disconnect. std::vector bogus_data(1024, 0); // '=' is arbitrary bad value if (so->write(sock, bogus_data.data(), bogus_data.size()) == -1) throw std::runtime_error("write() failed: " + std::to_string(errno)); read_until_error(sock); // error triggered by Router disconnecting } /** * @test * This test Verifies that: * 1. Router will block a misbehaving client after consecutive * connection errors * 2. Router will reset its connection error counter if client establishes a * successful connection before threshold is hit */ TEST_F(RouterRoutingTest, test1) { const uint16_t server_port = port_pool_.get_next_available(); const uint16_t router_port = port_pool_.get_next_available(); // doesn't really matter which file we use here, we are not going to do any // queries const std::string json_stmts = get_data_dir().join("bootstrap_big_data.js").str(); // launch the server mock auto &server_mock = launch_mysql_server_mock(json_stmts, server_port, false); // create a config with max_connect_errors == 3 const std::string routing_section = "[routing:basic]\n" "bind_port = " + std::to_string(router_port) + "\n" "mode = read-write\n" "max_connect_errors = 3\n" "destinations = 127.0.0.1:" + std::to_string(server_port) + "\n"; TempDirectory conf_dir("conf"); std::string conf_file = create_config_file(conf_dir.name(), routing_section); // launch the router with the created configuration auto &router = launch_router({"-c", conf_file}); // wait for server and router to begin accepting the connections ASSERT_NO_FATAL_FAILURE(check_port_ready(server_mock, server_port)); ASSERT_NO_FATAL_FAILURE(check_port_ready(router, router_port)); // we loop just for good measure, to additionally test that this behaviour is // repeatable for (int i = 0; i < 5; i++) { // good connection, followed by 2 bad ones. Good one should reset the error // counter mysqlrouter::MySQLSession client; EXPECT_NO_THROW(client.connect("127.0.0.1", router_port, "username", "password", "", "")); make_bad_connection(router_port); make_bad_connection(router_port); } // make a 3rd consecutive bad connection - it should cause Router to start // blocking us make_bad_connection(router_port); // we loop just for good measure, to additionally test that this behaviour is // repeatable for (int i = 0; i < 5; i++) { // now trying to make a good connection should fail due to blockage mysqlrouter::MySQLSession client; EXPECT_THROW_LIKE(client.connect("127.0.0.1", router_port, "username", "password", "", ""), std::exception, "Too many connection errors"); } } int main(int argc, char *argv[]) { init_windows_sockets(); ProcessManager::set_origin(Path(argv[0]).dirname()); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); }